drivers/infiniband/hw/ionic/ionic_pgtbl.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */

#include <linux/mman.h>
#include <linux/dma-mapping.h>

#include "ionic_fw.h"
#include "ionic_ibdev.h"

__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va)
{
	u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;
	u64 dma;

	if (!buf->tbl_pages)
		return cpu_to_le64(0);

	if (buf->tbl_pages > 1)
		return cpu_to_le64(buf->tbl_dma);

	if (buf->tbl_buf)
		dma = le64_to_cpu(buf->tbl_buf[0]);
	else
		dma = buf->tbl_dma;

	return cpu_to_le64(dma + (va & pg_mask));
}

__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va)
{
	if (buf->tbl_pages > 1) {
		u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;

		return cpu_to_be64(va & pg_mask);
	}

	return 0;
}

int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma)
{
	if (unlikely(buf->tbl_pages == buf->tbl_limit))
		return -ENOMEM;

	if (buf->tbl_buf)
		buf->tbl_buf[buf->tbl_pages] = cpu_to_le64(dma);
	else
		buf->tbl_dma = dma;

	++buf->tbl_pages;

	return 0;
}

static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev,
			       struct ionic_tbl_buf *buf)
{
	int rc;

	buf->tbl_size = buf->tbl_limit * sizeof(*buf->tbl_buf);
	buf->tbl_buf = kmalloc(buf->tbl_size, GFP_KERNEL);
	if (!buf->tbl_buf)
		return -ENOMEM;

	buf->tbl_dma = dma_map_single(dev->lif_cfg.hwdev, buf->tbl_buf,
				      buf->tbl_size, DMA_TO_DEVICE);
	rc = dma_mapping_error(dev->lif_cfg.hwdev, buf->tbl_dma);
	if (rc) {
		kfree(buf->tbl_buf);
		return rc;
	}

	return 0;
}

static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem)
{
	struct ib_block_iter biter;
	u64 page_dma;
	int rc;

	rdma_umem_for_each_dma_block(umem, &biter, BIT_ULL(buf->page_size_log2)) {
		page_dma = rdma_block_iter_dma_address(&biter);
		rc = ionic_pgtbl_page(buf, page_dma);
		if (rc)
			return rc;
	}

	return 0;
}

void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf)
{
	if (buf->tbl_buf)
		dma_unmap_single(dev->lif_cfg.hwdev, buf->tbl_dma,
				 buf->tbl_size, DMA_TO_DEVICE);

	kfree(buf->tbl_buf);
	memset(buf, 0, sizeof(*buf));
}

int ionic_pgtbl_init(struct ionic_ibdev *dev,
		     struct ionic_tbl_buf *buf,
		     struct ib_umem *umem,
		     dma_addr_t dma,
		     int limit,
		     u64 page_size)
{
	int rc;

	memset(buf, 0, sizeof(*buf));

	if (umem) {
		limit = ib_umem_num_dma_blocks(umem, page_size);
		buf->page_size_log2 = order_base_2(page_size);
	}

	if (limit < 1)
		return -EINVAL;

	buf->tbl_limit = limit;

	/* skip pgtbl if contiguous / direct translation */
	if (limit > 1) {
		rc = ionic_tbl_buf_alloc(dev, buf);
		if (rc)
			return rc;
	}

	if (umem)
		rc = ionic_pgtbl_umem(buf, umem);
	else
		rc = ionic_pgtbl_page(buf, dma);

	if (rc)
		goto err_unbuf;

	return 0;

err_unbuf:
	ionic_pgtbl_unbuf(dev, buf);
	return rc;
}