io_uring/bpf_filter.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * BPF filter support for io_uring. Supports SQE opcodes for now.
  */
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/io_uring.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
 #include <uapi/linux/io_uring.h>

 #include "io_uring.h"
 #include "bpf_filter.h"
 #include "net.h"
 #include "openclose.h"

 struct io_bpf_filter {
 	refcount_t		refs;
 	struct bpf_prog		*prog;
 	struct io_bpf_filter	*next;
 };

 /* Deny if this is set as the filter */
 static const struct io_bpf_filter dummy_filter;

 static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
 				      struct io_kiocb *req)
 {
 	const struct io_issue_def *def = &io_issue_defs[req->opcode];

 	bctx->opcode = req->opcode;
 	bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
 	bctx->user_data = req->cqe.user_data;
 	/* clear residual, anything from pdu_size and below */
 	memset((void *) bctx + offsetof(struct io_uring_bpf_ctx, pdu_size), 0,
 		sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));

 	/*
 	 * Opcodes can provide a handler for populating more data into bctx,
 	 * for filters to use.
 	 */
 	if (def->filter_pdu_size) {
 		bctx->pdu_size = def->filter_pdu_size;
 		def->filter_populate(bctx, req);
 	}
 }

 /*
  * Run registered filters for a given opcode. For filters, a return of 0 denies
  * execution of the request, a return of 1 allows it. If any filter for an
  * opcode returns 0, filter processing is stopped, and the request is denied.
  * This also stops the processing of filters.
  *
  * __io_uring_run_bpf_filters() returns 0 on success, allow running the
  * request, and -EACCES when a request is denied.
  */
 int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
 			       struct io_kiocb *req)
 {
 	struct io_bpf_filter *filter;
 	struct io_uring_bpf_ctx bpf_ctx;
 	int ret;

 	/* Fast check for existence of filters outside of RCU */
 	if (!rcu_access_pointer(filters[req->opcode]))
 		return 0;

 	/*
 	 * req->opcode has already been validated to be within the range
 	 * of what we expect, io_init_req() does this.
 	 */
 	guard(rcu)();
 	filter = rcu_dereference(filters[req->opcode]);
 	if (!filter)
 		return 0;
 	else if (filter == &dummy_filter)
 		return -EACCES;

 	io_uring_populate_bpf_ctx(&bpf_ctx, req);

 	/*
 	 * Iterate registered filters. The opcode is allowed IFF all filters
 	 * return 1. If any filter returns denied, opcode will be denied.
 	 */
 	do {
 		if (filter == &dummy_filter)
 			return -EACCES;
 		ret = bpf_prog_run_pin_on_cpu(filter->prog, &bpf_ctx);
 		if (!ret)
 			return -EACCES;
 		filter = filter->next;
 	} while (filter);

 	return 0;
 }

 static void io_free_bpf_filters(struct rcu_head *head)
 {
 	struct io_bpf_filter __rcu **filter;
 	struct io_bpf_filters *filters;
 	int i;

 	filters = container_of(head, struct io_bpf_filters, rcu_head);
 	scoped_guard(spinlock, &filters->lock) {
 		filter = filters->filters;
 		if (!filter)
 			return;
 	}

 	for (i = 0; i < IORING_OP_LAST; i++) {
 		struct io_bpf_filter *f;

 		rcu_read_lock();
 		f = rcu_dereference(filter[i]);
 		while (f) {
 			struct io_bpf_filter *next = f->next;

 			/*
 			 * Even if stacked, dummy filter will always be last
 			 * as it can only get installed into an empty spot.
 			 */
 			if (f == &dummy_filter)
 				break;

 			/* Someone still holds a ref, stop iterating. */
 			if (!refcount_dec_and_test(&f->refs))
 				break;

 			bpf_prog_destroy(f->prog);
 			kfree(f);
 			f = next;
 		}
 		rcu_read_unlock();
 	}
 	kfree(filters->filters);
 	kfree(filters);
 }

 static void __io_put_bpf_filters(struct io_bpf_filters *filters)
 {
 	if (refcount_dec_and_test(&filters->refs))
 		call_rcu(&filters->rcu_head, io_free_bpf_filters);
 }

 void io_put_bpf_filters(struct io_restriction *res)
 {
 	if (res->bpf_filters)
 		__io_put_bpf_filters(res->bpf_filters);
 }

 static struct io_bpf_filters *io_new_bpf_filters(void)
 {
 	struct io_bpf_filters *filters __free(kfree) = NULL;

 	filters = kzalloc_obj(*filters, GFP_KERNEL_ACCOUNT);
 	if (!filters)
 		return ERR_PTR(-ENOMEM);

 	filters->filters = kzalloc_objs(struct io_bpf_filter *, IORING_OP_LAST,
 					GFP_KERNEL_ACCOUNT);
 	if (!filters->filters)
 		return ERR_PTR(-ENOMEM);

 	refcount_set(&filters->refs, 1);
 	spin_lock_init(&filters->lock);
 	return no_free_ptr(filters);
 }

 /*
  * Validate classic BPF filter instructions. Only allow a safe subset of
  * operations - no packet data access, just context field loads and basic
  * ALU/jump operations.
  */
 static int io_uring_check_cbpf_filter(struct sock_filter *filter,
 				      unsigned int flen)
 {
 	int pc;

 	for (pc = 0; pc < flen; pc++) {
 		struct sock_filter *ftest = &filter[pc];
 		u16 code = ftest->code;
 		u32 k = ftest->k;

 		switch (code) {
 		case BPF_LD | BPF_W | BPF_ABS:
 			ftest->code = BPF_LDX | BPF_W | BPF_ABS;
 			/* 32-bit aligned and not out of bounds. */
 			if (k >= sizeof(struct io_uring_bpf_ctx) || k & 3)
 				return -EINVAL;
 			continue;
 		case BPF_LD | BPF_W | BPF_LEN:
 			ftest->code = BPF_LD | BPF_IMM;
 			ftest->k = sizeof(struct io_uring_bpf_ctx);
 			continue;
 		case BPF_LDX | BPF_W | BPF_LEN:
 			ftest->code = BPF_LDX | BPF_IMM;
 			ftest->k = sizeof(struct io_uring_bpf_ctx);
 			continue;
 		/* Explicitly include allowed calls. */
 		case BPF_RET | BPF_K:
 		case BPF_RET | BPF_A:
 		case BPF_ALU | BPF_ADD | BPF_K:
 		case BPF_ALU | BPF_ADD | BPF_X:
 		case BPF_ALU | BPF_SUB | BPF_K:
 		case BPF_ALU | BPF_SUB | BPF_X:
 		case BPF_ALU | BPF_MUL | BPF_K:
 		case BPF_ALU | BPF_MUL | BPF_X:
 		case BPF_ALU | BPF_DIV | BPF_K:
 		case BPF_ALU | BPF_DIV | BPF_X:
 		case BPF_ALU | BPF_AND | BPF_K:
 		case BPF_ALU | BPF_AND | BPF_X:
 		case BPF_ALU | BPF_OR | BPF_K:
 		case BPF_ALU | BPF_OR | BPF_X:
 		case BPF_ALU | BPF_XOR | BPF_K:
 		case BPF_ALU | BPF_XOR | BPF_X:
 		case BPF_ALU | BPF_LSH | BPF_K:
 		case BPF_ALU | BPF_LSH | BPF_X:
 		case BPF_ALU | BPF_RSH | BPF_K:
 		case BPF_ALU | BPF_RSH | BPF_X:
 		case BPF_ALU | BPF_NEG:
 		case BPF_LD | BPF_IMM:
 		case BPF_LDX | BPF_IMM:
 		case BPF_MISC | BPF_TAX:
 		case BPF_MISC | BPF_TXA:
 		case BPF_LD | BPF_MEM:
 		case BPF_LDX | BPF_MEM:
 		case BPF_ST:
 		case BPF_STX:
 		case BPF_JMP | BPF_JA:
 		case BPF_JMP | BPF_JEQ | BPF_K:
 		case BPF_JMP | BPF_JEQ | BPF_X:
 		case BPF_JMP | BPF_JGE | BPF_K:
 		case BPF_JMP | BPF_JGE | BPF_X:
 		case BPF_JMP | BPF_JGT | BPF_K:
 		case BPF_JMP | BPF_JGT | BPF_X:
 		case BPF_JMP | BPF_JSET | BPF_K:
 		case BPF_JMP | BPF_JSET | BPF_X:
 			continue;
 		default:
 			return -EINVAL;
 		}
 	}
 	return 0;
 }

 void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src)
 {
 	if (!src->bpf_filters)
 		return;

 	rcu_read_lock();
 	/*
 	 * If the src filter is going away, just ignore it.
 	 */
 	if (refcount_inc_not_zero(&src->bpf_filters->refs)) {
 		dst->bpf_filters = src->bpf_filters;
 		dst->bpf_filters_cow = true;
 	}
 	rcu_read_unlock();
 }

 /*
  * Allocate a new struct io_bpf_filters. Used when a filter is cloned and
  * modifications need to be made.
  */
 static struct io_bpf_filters *io_bpf_filter_cow(struct io_restriction *src)
 {
 	struct io_bpf_filters *filters;
 	struct io_bpf_filter *srcf;
 	int i;

 	filters = io_new_bpf_filters();
 	if (IS_ERR(filters))
 		return filters;

 	/*
 	 * Iterate filters from src and assign in destination. Grabbing
 	 * a reference is enough, we don't need to duplicate the memory.
 	 * This is safe because filters are only ever appended to the
 	 * front of the list, hence the only memory ever touched inside
 	 * a filter is the refcount.
 	 */
 	rcu_read_lock();
 	for (i = 0; i < IORING_OP_LAST; i++) {
 		srcf = rcu_dereference(src->bpf_filters->filters[i]);
 		if (!srcf) {
 			continue;
 		} else if (srcf == &dummy_filter) {
 			rcu_assign_pointer(filters->filters[i], &dummy_filter);
 			continue;
 		}

 		/*
 		 * Getting a ref on the first node is enough, putting the
 		 * filter and iterating nodes to free will stop on the first
 		 * one that doesn't hit zero when dropping.
 		 */
 		if (!refcount_inc_not_zero(&srcf->refs))
 			goto err;
 		rcu_assign_pointer(filters->filters[i], srcf);
 	}
 	rcu_read_unlock();
 	return filters;
 err:
 	rcu_read_unlock();
 	__io_put_bpf_filters(filters);
 	return ERR_PTR(-EBUSY);
 }

 #define IO_URING_BPF_FILTER_FLAGS	(IO_URING_BPF_FILTER_DENY_REST | \
 					 IO_URING_BPF_FILTER_SZ_STRICT)

 static int io_bpf_filter_import(struct io_uring_bpf *reg,
 				struct io_uring_bpf __user *arg)
 {
 	const struct io_issue_def *def;
 	int ret;

 	if (copy_from_user(reg, arg, sizeof(*reg)))
 		return -EFAULT;
 	if (reg->cmd_type != IO_URING_BPF_CMD_FILTER)
 		return -EINVAL;
 	if (reg->cmd_flags || reg->resv)
 		return -EINVAL;

 	if (reg->filter.opcode >= IORING_OP_LAST)
 		return -EINVAL;
 	if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
 		return -EINVAL;
 	if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv)))
 		return -EINVAL;
 	if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2)))
 		return -EINVAL;
 	if (!reg->filter.filter_len || reg->filter.filter_len > BPF_MAXINSNS)
 		return -EINVAL;

 	/* Verify filter size */
 	def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];

 	/* same size, always ok */
 	ret = 0;
 	if (reg->filter.pdu_size == def->filter_pdu_size)
 		;
 	/* size differs, fail in strict mode */
 	else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT)
 		ret = -EMSGSIZE;
 	/* userspace filter is bigger, always disallow */
 	else if (reg->filter.pdu_size > def->filter_pdu_size)
 		ret = -EMSGSIZE;

 	/* copy back kernel filter size */
 	reg->filter.pdu_size = def->filter_pdu_size;
 	if (copy_to_user(&arg->filter, &reg->filter, sizeof(reg->filter)))
 		return -EFAULT;

 	return ret;
 }

 int io_register_bpf_filter(struct io_restriction *res,
 			   struct io_uring_bpf __user *arg)
 {
 	struct io_bpf_filters *filters, *old_filters = NULL;
 	struct io_bpf_filter *filter, *old_filter;
 	struct io_uring_bpf reg;
 	struct bpf_prog *prog;
 	struct sock_fprog fprog;
 	int ret;

 	ret = io_bpf_filter_import(&reg, arg);
 	if (ret)
 		return ret;

 	fprog.len = reg.filter.filter_len;
 	fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);

 	ret = bpf_prog_create_from_user(&prog, &fprog,
 					io_uring_check_cbpf_filter, false);
 	if (ret)
 		return ret;

 	/*
 	 * No existing filters, allocate set.
 	 */
 	filters = res->bpf_filters;
 	if (!filters) {
 		filters = io_new_bpf_filters();
 		if (IS_ERR(filters)) {
 			ret = PTR_ERR(filters);
 			goto err_prog;
 		}
 	} else if (res->bpf_filters_cow) {
 		filters = io_bpf_filter_cow(res);
 		if (IS_ERR(filters)) {
 			ret = PTR_ERR(filters);
 			goto err_prog;
 		}
 		/*
 		 * Stash old filters, we'll put them once we know we'll
 		 * succeed. Until then, res->bpf_filters is left untouched.
 		 */
 		old_filters = res->bpf_filters;
 	}

 	filter = kzalloc_obj(*filter, GFP_KERNEL_ACCOUNT);
 	if (!filter) {
 		ret = -ENOMEM;
 		goto err;
 	}
 	refcount_set(&filter->refs, 1);
 	filter->prog = prog;

 	/*
 	 * Success - install the new filter set now. If we did COW, put
 	 * the old filters as we're replacing them.
 	 */
 	if (old_filters) {
 		__io_put_bpf_filters(old_filters);
 		res->bpf_filters_cow = false;
 	}
 	res->bpf_filters = filters;

 	/*
 	 * Insert filter - if the current opcode already has a filter
 	 * attached, add to the set.
 	 */
 	rcu_read_lock();
 	spin_lock_bh(&filters->lock);
 	old_filter = rcu_dereference(filters->filters[reg.filter.opcode]);
 	if (old_filter)
 		filter->next = old_filter;
 	rcu_assign_pointer(filters->filters[reg.filter.opcode], filter);

 	/*
 	 * If IO_URING_BPF_FILTER_DENY_REST is set, fill any unregistered
 	 * opcode with the dummy filter. That will cause them to be denied.
 	 */
 	if (reg.filter.flags & IO_URING_BPF_FILTER_DENY_REST) {
 		for (int i = 0; i < IORING_OP_LAST; i++) {
 			if (i == reg.filter.opcode)
 				continue;
 			old_filter = rcu_dereference(filters->filters[i]);
 			if (old_filter)
 				continue;
 			rcu_assign_pointer(filters->filters[i], &dummy_filter);
 		}
 	}

 	spin_unlock_bh(&filters->lock);
 	rcu_read_unlock();
 	return 0;
 err:
 	if (filters != res->bpf_filters)
 		__io_put_bpf_filters(filters);
 err_prog:
 	bpf_prog_destroy(prog);
 	return ret;
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* BPF filter support for io_uring. Supports SQE opcodes for now.
	*/
	#include <linux/kernel.h>
	#include <linux/errno.h>
	#include <linux/io_uring.h>
	#include <linux/filter.h>
	#include <linux/bpf.h>
	#include <uapi/linux/io_uring.h>

	#include "io_uring.h"
	#include "bpf_filter.h"
	#include "net.h"
	#include "openclose.h"

	struct io_bpf_filter {
	refcount_t refs;
	struct bpf_prog *prog;
	struct io_bpf_filter *next;
	};

	/* Deny if this is set as the filter */
	static const struct io_bpf_filter dummy_filter;

	static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
	struct io_kiocb *req)
	{
	const struct io_issue_def *def = &io_issue_defs[req->opcode];

	bctx->opcode = req->opcode;
	bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
	bctx->user_data = req->cqe.user_data;
	/* clear residual, anything from pdu_size and below */
	memset((void *) bctx + offsetof(struct io_uring_bpf_ctx, pdu_size), 0,
	sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));

	/*
	* Opcodes can provide a handler for populating more data into bctx,
	* for filters to use.
	*/
	if (def->filter_pdu_size) {
	bctx->pdu_size = def->filter_pdu_size;
	def->filter_populate(bctx, req);
	}
	}

	/*
	* Run registered filters for a given opcode. For filters, a return of 0 denies
	* execution of the request, a return of 1 allows it. If any filter for an
	* opcode returns 0, filter processing is stopped, and the request is denied.
	* This also stops the processing of filters.
	*
	* __io_uring_run_bpf_filters() returns 0 on success, allow running the
	* request, and -EACCES when a request is denied.
	*/
	int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
	struct io_kiocb *req)
	{
	struct io_bpf_filter *filter;
	struct io_uring_bpf_ctx bpf_ctx;
	int ret;

	/* Fast check for existence of filters outside of RCU */
	if (!rcu_access_pointer(filters[req->opcode]))
	return 0;

	/*
	* req->opcode has already been validated to be within the range
	* of what we expect, io_init_req() does this.
	*/
	guard(rcu)();
	filter = rcu_dereference(filters[req->opcode]);
	if (!filter)
	return 0;
	else if (filter == &dummy_filter)
	return -EACCES;

	io_uring_populate_bpf_ctx(&bpf_ctx, req);

	/*
	* Iterate registered filters. The opcode is allowed IFF all filters
	* return 1. If any filter returns denied, opcode will be denied.
	*/
	do {
	if (filter == &dummy_filter)
	return -EACCES;
	ret = bpf_prog_run_pin_on_cpu(filter->prog, &bpf_ctx);
	if (!ret)
	return -EACCES;
	filter = filter->next;
	} while (filter);

	return 0;
	}

	static void io_free_bpf_filters(struct rcu_head *head)
	{
	struct io_bpf_filter __rcu **filter;
	struct io_bpf_filters *filters;
	int i;

	filters = container_of(head, struct io_bpf_filters, rcu_head);
	scoped_guard(spinlock, &filters->lock) {
	filter = filters->filters;
	if (!filter)
	return;
	}

	for (i = 0; i < IORING_OP_LAST; i++) {
	struct io_bpf_filter *f;

	rcu_read_lock();
	f = rcu_dereference(filter[i]);
	while (f) {
	struct io_bpf_filter *next = f->next;

	/*
	* Even if stacked, dummy filter will always be last
	* as it can only get installed into an empty spot.
	*/
	if (f == &dummy_filter)
	break;

	/* Someone still holds a ref, stop iterating. */
	if (!refcount_dec_and_test(&f->refs))
	break;

	bpf_prog_destroy(f->prog);
	kfree(f);
	f = next;
	}
	rcu_read_unlock();
	}
	kfree(filters->filters);
	kfree(filters);
	}

	static void __io_put_bpf_filters(struct io_bpf_filters *filters)
	{
	if (refcount_dec_and_test(&filters->refs))
	call_rcu(&filters->rcu_head, io_free_bpf_filters);
	}

	void io_put_bpf_filters(struct io_restriction *res)
	{
	if (res->bpf_filters)
	__io_put_bpf_filters(res->bpf_filters);
	}

	static struct io_bpf_filters *io_new_bpf_filters(void)
	{
	struct io_bpf_filters *filters __free(kfree) = NULL;

	filters = kzalloc_obj(*filters, GFP_KERNEL_ACCOUNT);
	if (!filters)
	return ERR_PTR(-ENOMEM);

	filters->filters = kzalloc_objs(struct io_bpf_filter *, IORING_OP_LAST,
	GFP_KERNEL_ACCOUNT);
	if (!filters->filters)
	return ERR_PTR(-ENOMEM);

	refcount_set(&filters->refs, 1);
	spin_lock_init(&filters->lock);
	return no_free_ptr(filters);
	}

	/*
	* Validate classic BPF filter instructions. Only allow a safe subset of
	* operations - no packet data access, just context field loads and basic
	* ALU/jump operations.
	*/
	static int io_uring_check_cbpf_filter(struct sock_filter *filter,
	unsigned int flen)
	{
	int pc;

	for (pc = 0; pc < flen; pc++) {
	struct sock_filter *ftest = &filter[pc];
	u16 code = ftest->code;
	u32 k = ftest->k;

	switch (code) {
	case BPF_LD \| BPF_W \| BPF_ABS:
	ftest->code = BPF_LDX \| BPF_W \| BPF_ABS;
	/* 32-bit aligned and not out of bounds. */
	if (k >= sizeof(struct io_uring_bpf_ctx) \|\| k & 3)
	return -EINVAL;
	continue;
	case BPF_LD \| BPF_W \| BPF_LEN:
	ftest->code = BPF_LD \| BPF_IMM;
	ftest->k = sizeof(struct io_uring_bpf_ctx);
	continue;
	case BPF_LDX \| BPF_W \| BPF_LEN:
	ftest->code = BPF_LDX \| BPF_IMM;
	ftest->k = sizeof(struct io_uring_bpf_ctx);
	continue;
	/* Explicitly include allowed calls. */
	case BPF_RET \| BPF_K:
	case BPF_RET \| BPF_A:
	case BPF_ALU \| BPF_ADD \| BPF_K:
	case BPF_ALU \| BPF_ADD \| BPF_X:
	case BPF_ALU \| BPF_SUB \| BPF_K:
	case BPF_ALU \| BPF_SUB \| BPF_X:
	case BPF_ALU \| BPF_MUL \| BPF_K:
	case BPF_ALU \| BPF_MUL \| BPF_X:
	case BPF_ALU \| BPF_DIV \| BPF_K:
	case BPF_ALU \| BPF_DIV \| BPF_X:
	case BPF_ALU \| BPF_AND \| BPF_K:
	case BPF_ALU \| BPF_AND \| BPF_X:
	case BPF_ALU \| BPF_OR \| BPF_K:
	case BPF_ALU \| BPF_OR \| BPF_X:
	case BPF_ALU \| BPF_XOR \| BPF_K:
	case BPF_ALU \| BPF_XOR \| BPF_X:
	case BPF_ALU \| BPF_LSH \| BPF_K:
	case BPF_ALU \| BPF_LSH \| BPF_X:
	case BPF_ALU \| BPF_RSH \| BPF_K:
	case BPF_ALU \| BPF_RSH \| BPF_X:
	case BPF_ALU \| BPF_NEG:
	case BPF_LD \| BPF_IMM:
	case BPF_LDX \| BPF_IMM:
	case BPF_MISC \| BPF_TAX:
	case BPF_MISC \| BPF_TXA:
	case BPF_LD \| BPF_MEM:
	case BPF_LDX \| BPF_MEM:
	case BPF_ST:
	case BPF_STX:
	case BPF_JMP \| BPF_JA:
	case BPF_JMP \| BPF_JEQ \| BPF_K:
	case BPF_JMP \| BPF_JEQ \| BPF_X:
	case BPF_JMP \| BPF_JGE \| BPF_K:
	case BPF_JMP \| BPF_JGE \| BPF_X:
	case BPF_JMP \| BPF_JGT \| BPF_K:
	case BPF_JMP \| BPF_JGT \| BPF_X:
	case BPF_JMP \| BPF_JSET \| BPF_K:
	case BPF_JMP \| BPF_JSET \| BPF_X:
	continue;
	default:
	return -EINVAL;
	}
	}
	return 0;
	}

	void io_bpf_filter_clone(struct io_restriction dst, struct io_restriction src)
	{
	if (!src->bpf_filters)
	return;

	rcu_read_lock();
	/*
	* If the src filter is going away, just ignore it.
	*/
	if (refcount_inc_not_zero(&src->bpf_filters->refs)) {
	dst->bpf_filters = src->bpf_filters;
	dst->bpf_filters_cow = true;
	}
	rcu_read_unlock();
	}

	/*
	* Allocate a new struct io_bpf_filters. Used when a filter is cloned and
	* modifications need to be made.
	*/
	static struct io_bpf_filters io_bpf_filter_cow(struct io_restriction src)
	{
	struct io_bpf_filters *filters;
	struct io_bpf_filter *srcf;
	int i;

	filters = io_new_bpf_filters();
	if (IS_ERR(filters))
	return filters;

	/*
	* Iterate filters from src and assign in destination. Grabbing
	* a reference is enough, we don't need to duplicate the memory.
	* This is safe because filters are only ever appended to the
	* front of the list, hence the only memory ever touched inside
	* a filter is the refcount.
	*/
	rcu_read_lock();
	for (i = 0; i < IORING_OP_LAST; i++) {
	srcf = rcu_dereference(src->bpf_filters->filters[i]);
	if (!srcf) {
	continue;
	} else if (srcf == &dummy_filter) {
	rcu_assign_pointer(filters->filters[i], &dummy_filter);
	continue;
	}

	/*
	* Getting a ref on the first node is enough, putting the
	* filter and iterating nodes to free will stop on the first
	* one that doesn't hit zero when dropping.
	*/
	if (!refcount_inc_not_zero(&srcf->refs))
	goto err;
	rcu_assign_pointer(filters->filters[i], srcf);
	}
	rcu_read_unlock();
	return filters;
	err:
	rcu_read_unlock();
	__io_put_bpf_filters(filters);
	return ERR_PTR(-EBUSY);
	}

	#define IO_URING_BPF_FILTER_FLAGS (IO_URING_BPF_FILTER_DENY_REST \| \
	IO_URING_BPF_FILTER_SZ_STRICT)

	static int io_bpf_filter_import(struct io_uring_bpf *reg,
	struct io_uring_bpf __user *arg)
	{
	const struct io_issue_def *def;
	int ret;

	if (copy_from_user(reg, arg, sizeof(*reg)))
	return -EFAULT;
	if (reg->cmd_type != IO_URING_BPF_CMD_FILTER)
	return -EINVAL;
	if (reg->cmd_flags \|\| reg->resv)
	return -EINVAL;

	if (reg->filter.opcode >= IORING_OP_LAST)
	return -EINVAL;
	if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
	return -EINVAL;
	if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv)))
	return -EINVAL;
	if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2)))
	return -EINVAL;
	if (!reg->filter.filter_len \|\| reg->filter.filter_len > BPF_MAXINSNS)
	return -EINVAL;

	/* Verify filter size */
	def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];

	/* same size, always ok */
	ret = 0;
	if (reg->filter.pdu_size == def->filter_pdu_size)
	;
	/* size differs, fail in strict mode */
	else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT)
	ret = -EMSGSIZE;
	/* userspace filter is bigger, always disallow */
	else if (reg->filter.pdu_size > def->filter_pdu_size)
	ret = -EMSGSIZE;

	/* copy back kernel filter size */
	reg->filter.pdu_size = def->filter_pdu_size;
	if (copy_to_user(&arg->filter, &reg->filter, sizeof(reg->filter)))
	return -EFAULT;

	return ret;
	}

	int io_register_bpf_filter(struct io_restriction *res,
	struct io_uring_bpf __user *arg)
	{
	struct io_bpf_filters filters, old_filters = NULL;
	struct io_bpf_filter filter, old_filter;
	struct io_uring_bpf reg;
	struct bpf_prog *prog;
	struct sock_fprog fprog;
	int ret;

	ret = io_bpf_filter_import(&reg, arg);
	if (ret)
	return ret;

	fprog.len = reg.filter.filter_len;
	fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);

	ret = bpf_prog_create_from_user(&prog, &fprog,
	io_uring_check_cbpf_filter, false);
	if (ret)
	return ret;

	/*
	* No existing filters, allocate set.
	*/
	filters = res->bpf_filters;
	if (!filters) {
	filters = io_new_bpf_filters();
	if (IS_ERR(filters)) {
	ret = PTR_ERR(filters);
	goto err_prog;
	}
	} else if (res->bpf_filters_cow) {
	filters = io_bpf_filter_cow(res);
	if (IS_ERR(filters)) {
	ret = PTR_ERR(filters);
	goto err_prog;
	}
	/*
	* Stash old filters, we'll put them once we know we'll
	* succeed. Until then, res->bpf_filters is left untouched.
	*/
	old_filters = res->bpf_filters;
	}

	filter = kzalloc_obj(*filter, GFP_KERNEL_ACCOUNT);
	if (!filter) {
	ret = -ENOMEM;
	goto err;
	}
	refcount_set(&filter->refs, 1);
	filter->prog = prog;

	/*
	* Success - install the new filter set now. If we did COW, put
	* the old filters as we're replacing them.
	*/
	if (old_filters) {
	__io_put_bpf_filters(old_filters);
	res->bpf_filters_cow = false;
	}
	res->bpf_filters = filters;

	/*
	* Insert filter - if the current opcode already has a filter
	* attached, add to the set.
	*/
	rcu_read_lock();
	spin_lock_bh(&filters->lock);
	old_filter = rcu_dereference(filters->filters[reg.filter.opcode]);
	if (old_filter)
	filter->next = old_filter;
	rcu_assign_pointer(filters->filters[reg.filter.opcode], filter);

	/*
	* If IO_URING_BPF_FILTER_DENY_REST is set, fill any unregistered
	* opcode with the dummy filter. That will cause them to be denied.
	*/
	if (reg.filter.flags & IO_URING_BPF_FILTER_DENY_REST) {
	for (int i = 0; i < IORING_OP_LAST; i++) {
	if (i == reg.filter.opcode)
	continue;
	old_filter = rcu_dereference(filters->filters[i]);
	if (old_filter)
	continue;
	rcu_assign_pointer(filters->filters[i], &dummy_filter);
	}
	}

	spin_unlock_bh(&filters->lock);
	rcu_read_unlock();
	return 0;
	err:
	if (filters != res->bpf_filters)
	__io_put_bpf_filters(filters);
	err_prog:
	bpf_prog_destroy(prog);
	return ret;
	}