HardenedBSD/sys/dev/gve/gve_qpl.c

/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (c) 2023 Google LLC
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#include <sys/malloc.h>

#include "gve.h"
#include "gve_adminq.h"
#include "gve_dqo.h"

static MALLOC_DEFINE(M_GVE_QPL, "gve qpl", "gve qpl allocations");

static uint32_t
gve_num_tx_qpls(struct gve_priv *priv)
{
	if (!gve_is_qpl(priv))
		return (0);

	return (priv->tx_cfg.max_queues);
}

static uint32_t
gve_num_rx_qpls(struct gve_priv *priv)
{
	if (!gve_is_qpl(priv))
		return (0);

	return (priv->rx_cfg.max_queues);
}

static void
gve_free_qpl(struct gve_priv *priv, uint32_t id)
{
	struct gve_queue_page_list *qpl = &priv->qpls[id];
	int i;

	for (i = 0; i < qpl->num_dmas; i++) {
		gve_dmamap_destroy(&qpl->dmas[i]);
	}

	if (qpl->kva) {
		pmap_qremove(qpl->kva, qpl->num_pages);
		kva_free(qpl->kva, PAGE_SIZE * qpl->num_pages);
	}

	for (i = 0; i < qpl->num_pages; i++) {
		/*
		 * Free the page only if this is the last ref.
		 * Tx pages are known to have no other refs at
		 * this point, but Rx pages might still be in
		 * use by the networking stack, see gve_mextadd_free.
		 */
		if (vm_page_unwire_noq(qpl->pages[i])) {
			if (!qpl->kva) {
				pmap_qremove((vm_offset_t)qpl->dmas[i].cpu_addr, 1);
				kva_free((vm_offset_t)qpl->dmas[i].cpu_addr, PAGE_SIZE);
			}
			vm_page_free(qpl->pages[i]);
		}

		priv->num_registered_pages--;
	}

	if (qpl->pages != NULL)
		free(qpl->pages, M_GVE_QPL);

	if (qpl->dmas != NULL)
		free(qpl->dmas, M_GVE_QPL);
}

static int
gve_alloc_qpl(struct gve_priv *priv, uint32_t id, int npages, bool single_kva)
{
	struct gve_queue_page_list *qpl = &priv->qpls[id];
	int err;
	int i;

	if (npages + priv->num_registered_pages > priv->max_registered_pages) {
		device_printf(priv->dev, "Reached max number of registered pages %ju > %ju\n",
		    (uintmax_t)npages + priv->num_registered_pages,
		    (uintmax_t)priv->max_registered_pages);
		return (EINVAL);
	}

	qpl->id = id;
	qpl->num_pages = 0;
	qpl->num_dmas = 0;

	qpl->dmas = malloc(npages * sizeof(*qpl->dmas), M_GVE_QPL,
	    M_WAITOK | M_ZERO);

	qpl->pages = malloc(npages * sizeof(*qpl->pages), M_GVE_QPL,
	    M_WAITOK | M_ZERO);

	qpl->kva = 0;
	if (single_kva) {
		qpl->kva = kva_alloc(PAGE_SIZE * npages);
		if (!qpl->kva) {
			device_printf(priv->dev, "Failed to create the single kva for QPL %d\n", id);
			err = ENOMEM;
			goto abort;
		}
	}

	for (i = 0; i < npages; i++) {
		qpl->pages[i] = vm_page_alloc_noobj(VM_ALLOC_WIRED |
						    VM_ALLOC_WAITOK |
						    VM_ALLOC_ZERO);

		if (!single_kva) {
			qpl->dmas[i].cpu_addr = (void *)kva_alloc(PAGE_SIZE);
			if (!qpl->dmas[i].cpu_addr) {
				device_printf(priv->dev, "Failed to create kva for page %d in QPL %d", i, id);
				err = ENOMEM;
				goto abort;
			}
			pmap_qenter((vm_offset_t)qpl->dmas[i].cpu_addr, &(qpl->pages[i]), 1);
		} else
			qpl->dmas[i].cpu_addr = (void *)(qpl->kva + (PAGE_SIZE * i));


		qpl->num_pages++;
	}

	if (single_kva)
		pmap_qenter(qpl->kva, qpl->pages, npages);

	for (i = 0; i < npages; i++) {
		err = gve_dmamap_create(priv, /*size=*/PAGE_SIZE, /*align=*/PAGE_SIZE,
		    &qpl->dmas[i]);
		if (err != 0) {
			device_printf(priv->dev, "Failed to dma-map page %d in QPL %d\n", i, id);
			goto abort;
		}

		qpl->num_dmas++;
		priv->num_registered_pages++;
	}

	return (0);

abort:
	gve_free_qpl(priv, id);
	return (err);
}

void
gve_free_qpls(struct gve_priv *priv)
{
	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
	int i;

	if (num_qpls == 0)
		return;

	if (priv->qpls != NULL) {
		for (i = 0; i < num_qpls; i++)
			gve_free_qpl(priv, i);
		free(priv->qpls, M_GVE_QPL);
		priv->qpls = NULL;
	}
}

int gve_alloc_qpls(struct gve_priv *priv)
{
	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
	int num_pages;
	int err;
	int i;

	if (num_qpls == 0)
		return (0);

	priv->qpls = malloc(num_qpls * sizeof(*priv->qpls), M_GVE_QPL,
	    M_WAITOK | M_ZERO);

	num_pages = gve_is_gqi(priv) ?
	    priv->tx_desc_cnt / GVE_QPL_DIVISOR :
	    GVE_TX_NUM_QPL_PAGES_DQO;
	for (i = 0; i < gve_num_tx_qpls(priv); i++) {
		err = gve_alloc_qpl(priv, i, num_pages,
		    /*single_kva=*/true);
		if (err != 0)
			goto abort;
	}

	num_pages = gve_is_gqi(priv) ? priv->rx_desc_cnt : GVE_RX_NUM_QPL_PAGES_DQO;
	for (; i < num_qpls; i++) {
		err = gve_alloc_qpl(priv, i, num_pages, /*single_kva=*/false);
		if (err != 0)
			goto abort;
	}

	return (0);

abort:
	gve_free_qpls(priv);
	return (err);
}

static int
gve_unregister_n_qpls(struct gve_priv *priv, int n)
{
	int err;
	int i;

	for (i = 0; i < n; i++) {
		err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
		if (err != 0) {
			device_printf(priv->dev,
			    "Failed to unregister qpl %d, err: %d\n",
			    priv->qpls[i].id, err);
		}
	}

	if (err != 0)
		return (err);

	return (0);
}

int
gve_register_qpls(struct gve_priv *priv)
{
	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
	int err;
	int i;

	if (gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK))
		return (0);

	for (i = 0; i < num_qpls; i++) {
		err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
		if (err != 0) {
			device_printf(priv->dev,
			    "Failed to register qpl %d, err: %d\n",
			    priv->qpls[i].id, err);
			goto abort;
		}
	}

	gve_set_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
	return (0);

abort:
	gve_unregister_n_qpls(priv, i);
	return (err);
}

int
gve_unregister_qpls(struct gve_priv *priv)
{
	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
	int err;

	if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK))
		return (0);

	err = gve_unregister_n_qpls(priv, num_qpls);
	if (err != 0)
		return (err);

	gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
	return (0);
}

void
gve_mextadd_free(struct mbuf *mbuf)
{
	vm_page_t page = (vm_page_t)mbuf->m_ext.ext_arg1;
	vm_offset_t va = (vm_offset_t)mbuf->m_ext.ext_arg2;

	/*
	 * Free the page only if this is the last ref.
	 * The interface might no longer exist by the time
	 * this callback is called, see gve_free_qpl.
	 */
	if (__predict_false(vm_page_unwire_noq(page))) {
		pmap_qremove(va, 1);
		kva_free(va, PAGE_SIZE);
		vm_page_free(page);
	}
}