io_uring/zcrx.h

// SPDX-License-Identifier: GPL-2.0
#ifndef IOU_ZC_RX_H
#define IOU_ZC_RX_H

#include <linux/io_uring_types.h>
#include <linux/dma-buf.h>
#include <linux/socket.h>
#include <net/page_pool/types.h>
#include <net/net_trackers.h>

#define ZCRX_SUPPORTED_REG_FLAGS	(ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
#define ZCRX_FEATURES			(ZCRX_FEATURE_RX_PAGE_SIZE |\
					 ZCRX_FEATURE_EVENT)
#define ZCRX_EVENT_TYPE_MASK		((1U << ZCRX_EVENT_ALLOC_FAIL) |\
					 (1U << ZCRX_EVENT_COPY))

struct io_zcrx_mem {
	unsigned long			size;
	bool				is_dmabuf;

	struct page			**pages;
	unsigned long			nr_folios;
	struct sg_table			page_sg_table;
	unsigned long			account_pages;
	struct sg_table			*sgt;

	struct dma_buf_attachment	*attach;
	struct dma_buf			*dmabuf;
};

struct io_zcrx_area {
	struct net_iov_area	nia;
	struct io_zcrx_ifq	*ifq;
	atomic_t		*user_refs;

	bool			is_mapped;
	u16			area_id;

	/* freelist */
	u32			free_count;
	u32			*freelist;

	struct io_zcrx_mem	mem;
};

struct zcrx_rq_hdr {
	u32		head ____cacheline_aligned_in_smp;
	u32		tail ____cacheline_aligned_in_smp;
};

struct zcrx_rq {
	spinlock_t			lock;
	struct zcrx_rq_hdr		*ring;
	struct io_uring_zcrx_rqe	*rqes;
	u32				cached_head;
	u32				cached_tail;
	u32				nr_entries;
};

struct io_zcrx_ifq {
	/* read-protected by any of: ->pp_lock, ->alloc_lock, ->rq.lock */
	struct io_zcrx_area		**areas;
	unsigned			nr_areas;

	unsigned			niov_shift;
	struct user_struct		*user;
	struct mm_struct		*mm_account;
	bool				kern_readable;

	struct zcrx_rq			rq ____cacheline_aligned_in_smp;
	spinlock_t			alloc_lock ____cacheline_aligned_in_smp;

	u32				if_rxq;
	struct device			*dev;
	struct net_device		*netdev;
	netdevice_tracker		netdev_tracker;
	refcount_t			refs;
	/* counts userspace facing users like io_uring */
	refcount_t			user_refs;

	/*
	 * Page pool and net configuration lock, can be taken deeper in the
	 * net stack.
	 */
	struct mutex			pp_lock;
	struct io_mapped_region		rq_region;

	spinlock_t			ctx_lock;
	struct io_ring_ctx		*master_ctx;
	u32				allowed_notif_mask;
	u32				fired_notifs;
	u64				notif_data;
	struct zcrx_stats		*notif_stats;
};

#if defined(CONFIG_IO_URING_ZCRX)
int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
int io_register_zcrx(struct io_ring_ctx *ctx,
			 struct io_uring_zcrx_ifq_reg __user *arg);
void io_unregister_zcrx(struct io_ring_ctx *ctx);
void io_terminate_zcrx(struct io_ring_ctx *ctx);
int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
		 struct socket *sock, unsigned int flags,
		 unsigned issue_flags, unsigned int *len);
struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
					    unsigned int id);
#else
static inline int io_register_zcrx(struct io_ring_ctx *ctx,
				   struct io_uring_zcrx_ifq_reg __user *arg)
{
	return -EOPNOTSUPP;
}
static inline void io_unregister_zcrx(struct io_ring_ctx *ctx)
{
}
static inline void io_terminate_zcrx(struct io_ring_ctx *ctx)
{
}
static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
			       struct socket *sock, unsigned int flags,
			       unsigned issue_flags, unsigned int *len)
{
	return -EOPNOTSUPP;
}
static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
							  unsigned int id)
{
	return NULL;
}
static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
				void __user *arg, unsigned nr_arg)
{
	return -EOPNOTSUPP;
}
#endif

int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

#endif