include/net/netmem.h

/* SPDX-License-Identifier: GPL-2.0
 *
 *	Network memory
 *
 *	Author:	Mina Almasry <almasrymina@google.com>
 */

#ifndef _NET_NETMEM_H
#define _NET_NETMEM_H

#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <net/net_debug.h>

/* These fields in struct page are used by the page_pool and net stack:
 *
 *        struct {
 *                unsigned long pp_magic;
 *                struct page_pool *pp;
 *                unsigned long _pp_mapping_pad;
 *                unsigned long dma_addr;
 *                atomic_long_t pp_ref_count;
 *        };
 *
 * We mirror the page_pool fields here so the page_pool can access these
 * fields without worrying whether the underlying fields belong to a
 * page or netmem_desc.
 *
 * CAUTION: Do not update the fields in netmem_desc without also
 * updating the anonymous aliasing union in struct net_iov.
 */
struct netmem_desc {
	unsigned long _flags;
	unsigned long pp_magic;
	struct page_pool *pp;
	unsigned long _pp_mapping_pad;
	unsigned long dma_addr;
	atomic_long_t pp_ref_count;
};

#define NETMEM_DESC_ASSERT_OFFSET(pg, desc)        \
	static_assert(offsetof(struct page, pg) == \
		      offsetof(struct netmem_desc, desc))
NETMEM_DESC_ASSERT_OFFSET(flags, _flags);
NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic);
NETMEM_DESC_ASSERT_OFFSET(pp, pp);
NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr);
NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
#undef NETMEM_DESC_ASSERT_OFFSET

/*
 * Since struct netmem_desc uses the space in struct page, the size
 * should be checked, until struct netmem_desc has its own instance from
 * slab, to avoid conflicting with other members within struct page.
 */
static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount));

/* net_iov */

DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);

/*  We overload the LSB of the struct page pointer to indicate whether it's
 *  a page or net_iov.
 */
#define NET_IOV 0x01UL

enum net_iov_type {
	NET_IOV_DMABUF,
	NET_IOV_IOURING,
};

/* A memory descriptor representing abstract networking I/O vectors.
 *
 * net_iovs are allocated and used by networking code, and the size of
 * the chunk is PAGE_SIZE.
 *
 * Examples include imported dmabuf memory and imported io_uring memory. See
 * net_iov_type for all the supported types. While current net_iov types are
 * unreadable by the CPU, net_iov has no inherent restrictions and may be
 * CPU-readable or unreadable. New code must not assume net_iov implies
 * unreadable memory (check readability via netmem_address() or
 * skb_frags_readable() instead) and should, as much as possible, generalize
 * existing limitations to match the design principles.
 *
 * @pp_magic:	pp field, similar to the one in struct page/struct
 *		netmem_desc.
 * @pp:		the pp this net_iov belongs to, if any.
 * @dma_addr:	the dma addrs of the net_iov. Needed for the network
 *		card to send/receive this net_iov.
 * @pp_ref_count: the pp ref count of this net_iov, exactly the same
 *		usage as struct page/struct netmem_desc.
 * @owner:	the net_iov_area this net_iov belongs to, if any.
 * @type:	the type of the memory.  Different types of net_iovs are
 *		supported.
 */
struct net_iov {
	struct netmem_desc desc;
	enum net_iov_type type;
	struct net_iov_area *owner;
};

struct net_iov_area {
	/* Array of net_iovs for this area. */
	struct net_iov *niovs;
	size_t num_niovs;
};

static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov)
{
	return niov->owner;
}

static inline unsigned int net_iov_idx(const struct net_iov *niov)
{
	return niov - net_iov_owner(niov)->niovs;
}

/* Initialize a niov: stamp the owning area, the memory provider type.
 */
static inline void net_iov_init(struct net_iov *niov,
				struct net_iov_area *owner,
				enum net_iov_type type)
{
	niov->owner = owner;
	niov->type = type;
}

/* netmem */

/**
 * typedef netmem_ref - a nonexistent type marking a reference to generic
 * network memory.
 *
 * A netmem_ref can be a struct page* or a struct net_iov* underneath.
 * Memory providers (or the default page_pool allocator) allocate struct
 * net_iov or struct page, cast them to netmem_ref, and hand them to
 * page_pool.
 *
 * The page_pool, drivers, and core networking stack should operate on
 * netmem_ref rather than struct page or struct net_iov. Downcasting
 * netmem_ref via netmem_to_page() or netmem_to_net_iov() in callers is
 * not allowed unless a code path strictly requires a specific backing
 * type (e.g., kmap_local_page()). In such cases, add a netmem helper here
 * that handles both page and net_iov cases and returns an error if the
 * underlying type cannot support the operation.
 */
typedef unsigned long __bitwise netmem_ref;

static inline bool netmem_is_net_iov(const netmem_ref netmem)
{
	return (__force unsigned long)netmem & NET_IOV;
}

/**
 * __netmem_to_page - unsafely get pointer to the &page backing @netmem
 * @netmem: netmem reference to convert
 *
 * Unsafe version of netmem_to_page(). When @netmem is always page-backed,
 * e.g. when it's a header buffer, performs faster and generates smaller
 * object code (no check for the LSB, no WARN). When @netmem points to IOV,
 * provokes undefined behaviour.
 *
 * Return: pointer to the &page (garbage if @netmem is not page-backed).
 */
static inline struct page *__netmem_to_page(netmem_ref netmem)
{
	return (__force struct page *)netmem;
}

static inline struct page *netmem_to_page(netmem_ref netmem)
{
	if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))
		return NULL;

	return __netmem_to_page(netmem);
}

static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		return (struct net_iov *)((__force unsigned long)netmem &
					  ~NET_IOV);

	DEBUG_NET_WARN_ON_ONCE(true);
	return NULL;
}

static inline netmem_ref net_iov_to_netmem(struct net_iov *niov)
{
	return (__force netmem_ref)((unsigned long)niov | NET_IOV);
}

#define page_to_netmem(p)	(_Generic((p),			\
	const struct page * :	(__force const netmem_ref)(p),	\
	struct page * :		(__force netmem_ref)(p)))

/**
 * virt_to_netmem - convert virtual memory pointer to a netmem reference
 * @data: host memory pointer to convert
 *
 * Return: netmem reference to the &page backing this virtual address.
 */
static inline netmem_ref virt_to_netmem(const void *data)
{
	return page_to_netmem(virt_to_page(data));
}

static inline int netmem_ref_count(netmem_ref netmem)
{
	/* The non-pp refcount of net_iov is always 1. On net_iov, we only
	 * support pp refcounting which uses the pp_ref_count field.
	 */
	if (netmem_is_net_iov(netmem))
		return 1;

	return page_ref_count(netmem_to_page(netmem));
}

static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		return 0;

	return page_to_pfn(netmem_to_page(netmem));
}

/* XXX: How to extract netmem_desc from page must be changed, once
 * netmem_desc no longer overlays on page and will be allocated through
 * slab.
 */
#define __pp_page_to_nmdesc(p)	(_Generic((p),				\
	const struct page * :	(const struct netmem_desc *)(p),	\
	struct page * :		(struct netmem_desc *)(p)))

/* CAUTION: Check if the page is a pp page before calling this helper or
 * know it's a pp page.
 */
#define pp_page_to_nmdesc(p)						\
({									\
	DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p));		\
	__pp_page_to_nmdesc(p);						\
})

/**
 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing
 * @netmem
 * @netmem: netmem reference to convert
 *
 * Unsafe version that can be used only when @netmem is always backed by
 * system memory, performs faster and generates smaller object code (no
 * check for the LSB, no WARN). When @netmem points to IOV, provokes
 * undefined behaviour.
 *
 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed
 * by system memory).
 */
static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem)
{
	return (__force struct netmem_desc *)netmem;
}

/* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for
 * access to common fields.
 * @netmem: netmem reference to get netmem_desc.
 *
 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same
 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc.
 *
 * Return: the pointer to struct netmem_desc * regardless of its
 * underlying type.
 */
static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem)
{
	void *p = (void *)((__force unsigned long)netmem & ~NET_IOV);

	if (netmem_is_net_iov(netmem))
		return &((struct net_iov *)p)->desc;

	return __pp_page_to_nmdesc((struct page *)p);
}

/**
 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem
 * @netmem: netmem reference to get the pointer from
 *
 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed,
 * e.g. when it's a header buffer, performs faster and generates smaller
 * object code (avoids clearing the LSB). When @netmem points to IOV,
 * provokes invalid memory access.
 *
 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed).
 */
static inline struct page_pool *__netmem_get_pp(netmem_ref netmem)
{
	return __netmem_to_nmdesc(netmem)->pp;
}

static inline struct page_pool *netmem_get_pp(netmem_ref netmem)
{
	return netmem_to_nmdesc(netmem)->pp;
}

static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem)
{
	return &netmem_to_nmdesc(netmem)->pp_ref_count;
}

static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid)
{
	/* NUMA node preference only applies to struct page; net_iovs are
	 * managed by their memory provider.
	 */
	if (netmem_is_net_iov(netmem))
		return true;

	return page_to_nid(netmem_to_page(netmem)) == pref_nid;
}

static inline netmem_ref netmem_compound_head(netmem_ref netmem)
{
	/* niov are never compounded */
	if (netmem_is_net_iov(netmem))
		return netmem;

	return page_to_netmem(compound_head(netmem_to_page(netmem)));
}

/**
 * __netmem_address - unsafely get pointer to the memory backing @netmem
 * @netmem: netmem reference to get the pointer for
 *
 * Unsafe version of netmem_address(). When @netmem is always page-backed,
 * e.g. when it's a header buffer, performs faster and generates smaller
 * object code (no check for the LSB). When @netmem points to IOV, provokes
 * undefined behaviour.
 *
 * Return: pointer to the memory (garbage if @netmem is not page-backed).
 */
static inline void *__netmem_address(netmem_ref netmem)
{
	return page_address(__netmem_to_page(netmem));
}

static inline void *netmem_address(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		return NULL;

	return __netmem_address(netmem);
}

/**
 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure
 * @netmem: netmem reference to check
 *
 * Return: true if @netmem is page-backed and the page was allocated under
 * memory pressure, false otherwise.
 */
static inline bool netmem_is_pfmemalloc(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		return false;

	return page_is_pfmemalloc(netmem_to_page(netmem));
}

#define NETMEM_32BIT_ARCH_WITH_64BIT_DMA	\
	(sizeof(dma_addr_t) > sizeof(unsigned long))

static inline unsigned long netmem_dma_addr_encode(dma_addr_t addr)
{
	if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA)
		addr >>= PAGE_SHIFT;

	return addr;
}

static inline dma_addr_t netmem_dma_addr_decode(unsigned long addr)
{
	if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA)
		return (dma_addr_t)addr << PAGE_SHIFT;

	return addr;
}

static inline bool netmem_dma_addr_fits(dma_addr_t addr)
{
	/* We assume page alignment to shave off bottom bits,
	 * if this "compression" doesn't work we need to drop.
	 */
	return addr == netmem_dma_addr_decode(netmem_dma_addr_encode(addr));
}

static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
{
	return netmem_to_nmdesc(netmem)->dma_addr;
}

#if defined(CONFIG_NET_DEVMEM)
static inline bool net_is_devmem_iov(const struct net_iov *niov)
{
	return niov->type == NET_IOV_DMABUF;
}
#else
static inline bool net_is_devmem_iov(const struct net_iov *niov)
{
	return false;
}
#endif

void net_iov_inc_ref(struct net_iov *niov);
void net_iov_dec_ref(struct net_iov *niov);

/**
 * netmem_inc_ref - acquire a non-page_pool reference on a netmem
 * @netmem: netmem to reference
 *
 * Acquires a non-pp backing reference (page._refcount via get_page() or
 * binding->ref via net_iov_inc_ref()) regardless of page_pool membership.
 * Counterpart to netmem_dec_ref().
 */
static __always_inline void netmem_inc_ref(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		net_iov_inc_ref(netmem_to_net_iov(netmem));
	else
		get_page(netmem_to_page(netmem));
}

/**
 * netmem_dec_ref - release a non-page_pool reference on a netmem
 * @netmem: netmem to unreference
 *
 * Drops a non-pp backing reference (page._refcount via put_page() or
 * binding->ref via net_iov_dec_ref()) regardless of page_pool membership.
 * Counterpart to netmem_inc_ref().
 */
static __always_inline void netmem_dec_ref(netmem_ref netmem)
{
	if (netmem_is_net_iov(netmem))
		net_iov_dec_ref(netmem_to_net_iov(netmem));
	else
		put_page(netmem_to_page(netmem));
}

#define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL)   \
	do {                                                     \
		if (!netmem_is_net_iov(NETMEM))                  \
			dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \
		else                                             \
			dma_unmap_addr_set(PTR, ADDR_NAME, 0);   \
	} while (0)

static inline void netmem_dma_unmap_page_attrs(struct device *dev,
					       dma_addr_t addr, size_t size,
					       enum dma_data_direction dir,
					       unsigned long attrs)
{
	if (!addr)
		return;

	dma_unmap_page_attrs(dev, addr, size, dir, attrs);
}

#endif /* _NET_NETMEM_H */