tools/testing/selftests/sched_ext/dequeue.bpf.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * A scheduler that validates ops.dequeue() is called correctly:
  * - Tasks dispatched to terminal DSQs (local, global) bypass the BPF
  *   scheduler entirely: no ops.dequeue() should be called
  * - Tasks dispatched to user DSQs from ops.enqueue() enter BPF custody:
  *   ops.dequeue() must be called when they leave custody
  * - Every ops.enqueue() dispatch to non-terminal DSQs is followed by
  *   exactly one ops.dequeue() (validate 1:1 pairing and state machine)
  *
  * Copyright (c) 2026 NVIDIA Corporation.
  */

 #include <scx/common.bpf.h>

 #define SHARED_DSQ	0

 /*
  * BPF internal queue.
  *
  * Tasks are stored here and consumed from ops.dispatch(), validating that
  * tasks on BPF internal structures still get ops.dequeue() when they
  * leave.
  */
 struct {
 	__uint(type, BPF_MAP_TYPE_QUEUE);
 	__uint(max_entries, 32768);
 	__type(value, s32);
 } global_queue SEC(".maps");

 char _license[] SEC("license") = "GPL";

 UEI_DEFINE(uei);

 /*
  * Counters to track the lifecycle of tasks:
  * - enqueue_cnt: Number of times ops.enqueue() was called
  * - dequeue_cnt: Number of times ops.dequeue() was called (any type)
  * - dispatch_dequeue_cnt: Number of regular dispatch dequeues (no flag)
  * - change_dequeue_cnt: Number of property change dequeues
  * - bpf_queue_full: Number of times the BPF internal queue was full
  */
 u64 enqueue_cnt, dequeue_cnt, dispatch_dequeue_cnt, change_dequeue_cnt, bpf_queue_full;

 /*
  * Test scenarios:
  * 0) Dispatch to local DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF
  *    scheduler, no dequeue callbacks)
  * 1) Dispatch to global DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF
  *    scheduler, no dequeue callbacks)
  * 2) Dispatch to shared user DSQ from ops.select_cpu() (enters BPF scheduler,
  *    dequeue callbacks expected)
  * 3) Dispatch to local DSQ from ops.enqueue() (terminal DSQ, bypasses BPF
  *    scheduler, no dequeue callbacks)
  * 4) Dispatch to global DSQ from ops.enqueue() (terminal DSQ, bypasses BPF
  *    scheduler, no dequeue callbacks)
  * 5) Dispatch to shared user DSQ from ops.enqueue() (enters BPF scheduler,
  *    dequeue callbacks expected)
  * 6) BPF internal queue from ops.enqueue(): store task PIDs in ops.enqueue(),
  *    consume in ops.dispatch() and dispatch to local DSQ (validates dequeue
  *    for tasks stored in internal BPF data structures)
  */
 u32 test_scenario;

 /*
  * Per-task state to track lifecycle and validate workflow semantics.
  * State transitions:
  *   NONE -> ENQUEUED (on enqueue)
  *   NONE -> DISPATCHED (on direct dispatch to terminal DSQ)
  *   ENQUEUED -> DISPATCHED (on dispatch dequeue)
  *   DISPATCHED -> NONE (on property change dequeue or re-enqueue)
  *   ENQUEUED -> NONE (on property change dequeue before dispatch)
  */
 enum task_state {
 	TASK_NONE = 0,
 	TASK_ENQUEUED,
 	TASK_DISPATCHED,
 };

 struct task_ctx {
 	enum task_state state; /* Current state in the workflow */
 	u64 enqueue_seq;       /* Sequence number for debugging */
 };

 struct {
 	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
 	__uint(map_flags, BPF_F_NO_PREALLOC);
 	__type(key, int);
 	__type(value, struct task_ctx);
 } task_ctx_stor SEC(".maps");

 static struct task_ctx *try_lookup_task_ctx(struct task_struct *p)
 {
 	return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
 }

 s32 BPF_STRUCT_OPS(dequeue_select_cpu, struct task_struct *p,
 		   s32 prev_cpu, u64 wake_flags)
 {
 	struct task_ctx *tctx;

 	tctx = try_lookup_task_ctx(p);
 	if (!tctx)
 		return prev_cpu;

 	switch (test_scenario) {
 	case 0:
 		/*
 		 * Direct dispatch to the local DSQ.
 		 *
 		 * Task bypasses BPF scheduler entirely: no enqueue
 		 * tracking, no ops.dequeue() callbacks.
 		 */
 		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
 		tctx->state = TASK_DISPATCHED;
 		break;
 	case 1:
 		/*
 		 * Direct dispatch to the global DSQ.
 		 *
 		 * Task bypasses BPF scheduler entirely: no enqueue
 		 * tracking, no ops.dequeue() callbacks.
 		 */
 		scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
 		tctx->state = TASK_DISPATCHED;
 		break;
 	case 2:
 		/*
 		 * Dispatch to a shared user DSQ.
 		 *
 		 * Task enters BPF scheduler management: track
 		 * enqueue/dequeue lifecycle and validate state
 		 * transitions.
 		 */
 		if (tctx->state == TASK_ENQUEUED)
 			scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
 				      p->pid, p->comm, tctx->enqueue_seq);

 		scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, 0);

 		__sync_fetch_and_add(&enqueue_cnt, 1);

 		tctx->state = TASK_ENQUEUED;
 		tctx->enqueue_seq++;
 		break;
 	}

 	return prev_cpu;
 }

 void BPF_STRUCT_OPS(dequeue_enqueue, struct task_struct *p, u64 enq_flags)
 {
 	struct task_ctx *tctx;
 	s32 pid = p->pid;

 	tctx = try_lookup_task_ctx(p);
 	if (!tctx)
 		return;

 	switch (test_scenario) {
 	case 3:
 		/*
 		 * Direct dispatch to the local DSQ.
 		 *
 		 * Task bypasses BPF scheduler entirely: no enqueue
 		 * tracking, no ops.dequeue() callbacks.
 		 */
 		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
 		tctx->state = TASK_DISPATCHED;
 		break;
 	case 4:
 		/*
 		 * Direct dispatch to the global DSQ.
 		 *
 		 * Task bypasses BPF scheduler entirely: no enqueue
 		 * tracking, no ops.dequeue() callbacks.
 		 */
 		scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
 		tctx->state = TASK_DISPATCHED;
 		break;
 	case 5:
 		/*
 		 * Dispatch to shared user DSQ.
 		 *
 		 * Task enters BPF scheduler management: track
 		 * enqueue/dequeue lifecycle and validate state
 		 * transitions.
 		 */
 		if (tctx->state == TASK_ENQUEUED)
 			scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
 				      p->pid, p->comm, tctx->enqueue_seq);

 		scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);

 		__sync_fetch_and_add(&enqueue_cnt, 1);

 		tctx->state = TASK_ENQUEUED;
 		tctx->enqueue_seq++;
 		break;
 	case 6:
 		/*
 		 * Store task in BPF internal queue.
 		 *
 		 * Task enters BPF scheduler management: track
 		 * enqueue/dequeue lifecycle and validate state
 		 * transitions.
 		 */
 		if (tctx->state == TASK_ENQUEUED)
 			scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
 				      p->pid, p->comm, tctx->enqueue_seq);

 		if (bpf_map_push_elem(&global_queue, &pid, 0)) {
 			scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
 			__sync_fetch_and_add(&bpf_queue_full, 1);

 			tctx->state = TASK_DISPATCHED;
 		} else {
 			__sync_fetch_and_add(&enqueue_cnt, 1);

 			tctx->state = TASK_ENQUEUED;
 			tctx->enqueue_seq++;
 		}
 		break;
 	default:
 		/* For all other scenarios, dispatch to the global DSQ */
 		scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
 		tctx->state = TASK_DISPATCHED;
 		break;
 	}

 	scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE);
 }

 void BPF_STRUCT_OPS(dequeue_dequeue, struct task_struct *p, u64 deq_flags)
 {
 	struct task_ctx *tctx;

 	__sync_fetch_and_add(&dequeue_cnt, 1);

 	tctx = try_lookup_task_ctx(p);
 	if (!tctx)
 		return;

 	/*
 	 * For scenarios 0, 1, 3, and 4 (terminal DSQs: local and global),
 	 * ops.dequeue() should never be called because tasks bypass the
 	 * BPF scheduler entirely. If we get here, it's a kernel bug.
 	 */
 	if (test_scenario == 0 || test_scenario == 3) {
 		scx_bpf_error("%d (%s): dequeue called for local DSQ scenario",
 			      p->pid, p->comm);
 		return;
 	}

 	if (test_scenario == 1 || test_scenario == 4) {
 		scx_bpf_error("%d (%s): dequeue called for global DSQ scenario",
 			      p->pid, p->comm);
 		return;
 	}

 	if (deq_flags & SCX_DEQ_SCHED_CHANGE) {
 		/*
 		 * Property change interrupting the workflow. Valid from
 		 * both ENQUEUED and DISPATCHED states. Transitions task
 		 * back to NONE state.
 		 */
 		__sync_fetch_and_add(&change_dequeue_cnt, 1);

 		/* Validate state transition */
 		if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_DISPATCHED)
 			scx_bpf_error("%d (%s): invalid property change dequeue state=%d seq=%llu",
 				      p->pid, p->comm, tctx->state, tctx->enqueue_seq);

 		/*
 		 * Transition back to NONE: task outside scheduler control.
 		 *
 		 * Scenario 6: dispatch() checks tctx->state after popping a
 		 * PID, if the task is in state NONE, it was dequeued by
 		 * property change and must not be dispatched (this
 		 * prevents "target CPU not allowed").
 		 */
 		tctx->state = TASK_NONE;
 	} else {
 		/*
 		 * Regular dispatch dequeue: kernel is moving the task from
 		 * BPF custody to a terminal DSQ. Normally we come from
 		 * ENQUEUED state. We can also see TASK_NONE if the task
 		 * was dequeued by property change (SCX_DEQ_SCHED_CHANGE)
 		 * while it was already on a DSQ (dispatched but not yet
 		 * consumed); in that case we just leave state as NONE.
 		 */
 		__sync_fetch_and_add(&dispatch_dequeue_cnt, 1);

 		/*
 		 * Must be ENQUEUED (normal path) or NONE (already dequeued
 		 * by property change while on a DSQ).
 		 */
 		if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_NONE)
 			scx_bpf_error("%d (%s): dispatch dequeue from state %d seq=%llu",
 				      p->pid, p->comm, tctx->state, tctx->enqueue_seq);

 		if (tctx->state == TASK_ENQUEUED)
 			tctx->state = TASK_DISPATCHED;

 		/* NONE: leave as-is, task was already property-change dequeued */
 	}
 }

 void BPF_STRUCT_OPS(dequeue_dispatch, s32 cpu, struct task_struct *prev)
 {
 	if (test_scenario == 6) {
 		struct task_ctx *tctx;
 		struct task_struct *p;
 		s32 pid;

 		if (bpf_map_pop_elem(&global_queue, &pid))
 			return;

 		p = bpf_task_from_pid(pid);
 		if (!p)
 			return;

 		/*
 		 * If the task was dequeued by property change
 		 * (ops.dequeue() set tctx->state = TASK_NONE), skip
 		 * dispatch.
 		 */
 		tctx = try_lookup_task_ctx(p);
 		if (!tctx || tctx->state == TASK_NONE) {
 			bpf_task_release(p);
 			return;
 		}

 		/*
 		 * Dispatch to this CPU's local DSQ if allowed, otherwise
 		 * fallback to the global DSQ.
 		 */
 		if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr))
 			scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_DFL, 0);
 		else
 			scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);

 		bpf_task_release(p);
 	} else {
 		scx_bpf_dsq_move_to_local(SHARED_DSQ, 0);
 	}
 }

 s32 BPF_STRUCT_OPS(dequeue_init_task, struct task_struct *p,
 		   struct scx_init_task_args *args)
 {
 	struct task_ctx *tctx;

 	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0,
 				   BPF_LOCAL_STORAGE_GET_F_CREATE);
 	if (!tctx)
 		return -ENOMEM;

 	return 0;
 }

 s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_init)
 {
 	s32 ret;

 	ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
 	if (ret)
 		return ret;

 	return 0;
 }

 void BPF_STRUCT_OPS(dequeue_exit, struct scx_exit_info *ei)
 {
 	UEI_RECORD(uei, ei);
 }

 SEC(".struct_ops.link")
 struct sched_ext_ops dequeue_ops = {
 	.select_cpu		= (void *)dequeue_select_cpu,
 	.enqueue		= (void *)dequeue_enqueue,
 	.dequeue		= (void *)dequeue_dequeue,
 	.dispatch		= (void *)dequeue_dispatch,
 	.init_task		= (void *)dequeue_init_task,
 	.init			= (void *)dequeue_init,
 	.exit			= (void *)dequeue_exit,
 	.flags			= SCX_OPS_ENQ_LAST,
 	.name			= "dequeue_test",
 };
	// SPDX-License-Identifier: GPL-2.0
	/*
	* A scheduler that validates ops.dequeue() is called correctly:
	* - Tasks dispatched to terminal DSQs (local, global) bypass the BPF
	* scheduler entirely: no ops.dequeue() should be called
	* - Tasks dispatched to user DSQs from ops.enqueue() enter BPF custody:
	* ops.dequeue() must be called when they leave custody
	* - Every ops.enqueue() dispatch to non-terminal DSQs is followed by
	* exactly one ops.dequeue() (validate 1:1 pairing and state machine)
	*
	* Copyright (c) 2026 NVIDIA Corporation.
	*/

	#include <scx/common.bpf.h>

	#define SHARED_DSQ 0

	/*
	* BPF internal queue.
	*
	* Tasks are stored here and consumed from ops.dispatch(), validating that
	* tasks on BPF internal structures still get ops.dequeue() when they
	* leave.
	*/
	struct {
	__uint(type, BPF_MAP_TYPE_QUEUE);
	__uint(max_entries, 32768);
	__type(value, s32);
	} global_queue SEC(".maps");

	char _license[] SEC("license") = "GPL";

	UEI_DEFINE(uei);

	/*
	* Counters to track the lifecycle of tasks:
	* - enqueue_cnt: Number of times ops.enqueue() was called
	* - dequeue_cnt: Number of times ops.dequeue() was called (any type)
	* - dispatch_dequeue_cnt: Number of regular dispatch dequeues (no flag)
	* - change_dequeue_cnt: Number of property change dequeues
	* - bpf_queue_full: Number of times the BPF internal queue was full
	*/
	u64 enqueue_cnt, dequeue_cnt, dispatch_dequeue_cnt, change_dequeue_cnt, bpf_queue_full;

	/*
	* Test scenarios:
	* 0) Dispatch to local DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF
	* scheduler, no dequeue callbacks)
	* 1) Dispatch to global DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF
	* scheduler, no dequeue callbacks)
	* 2) Dispatch to shared user DSQ from ops.select_cpu() (enters BPF scheduler,
	* dequeue callbacks expected)
	* 3) Dispatch to local DSQ from ops.enqueue() (terminal DSQ, bypasses BPF
	* scheduler, no dequeue callbacks)
	* 4) Dispatch to global DSQ from ops.enqueue() (terminal DSQ, bypasses BPF
	* scheduler, no dequeue callbacks)
	* 5) Dispatch to shared user DSQ from ops.enqueue() (enters BPF scheduler,
	* dequeue callbacks expected)
	* 6) BPF internal queue from ops.enqueue(): store task PIDs in ops.enqueue(),
	* consume in ops.dispatch() and dispatch to local DSQ (validates dequeue
	* for tasks stored in internal BPF data structures)
	*/
	u32 test_scenario;

	/*
	* Per-task state to track lifecycle and validate workflow semantics.
	* State transitions:
	* NONE -> ENQUEUED (on enqueue)
	* NONE -> DISPATCHED (on direct dispatch to terminal DSQ)
	* ENQUEUED -> DISPATCHED (on dispatch dequeue)
	* DISPATCHED -> NONE (on property change dequeue or re-enqueue)
	* ENQUEUED -> NONE (on property change dequeue before dispatch)
	*/
	enum task_state {
	TASK_NONE = 0,
	TASK_ENQUEUED,
	TASK_DISPATCHED,
	};

	struct task_ctx {
	enum task_state state; /* Current state in the workflow */
	u64 enqueue_seq; /* Sequence number for debugging */
	};

	struct {
	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
	__uint(map_flags, BPF_F_NO_PREALLOC);
	__type(key, int);
	__type(value, struct task_ctx);
	} task_ctx_stor SEC(".maps");

	static struct task_ctx try_lookup_task_ctx(struct task_struct p)
	{
	return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
	}

	s32 BPF_STRUCT_OPS(dequeue_select_cpu, struct task_struct *p,
	s32 prev_cpu, u64 wake_flags)
	{
	struct task_ctx *tctx;

	tctx = try_lookup_task_ctx(p);
	if (!tctx)
	return prev_cpu;

	switch (test_scenario) {
	case 0:
	/*
	* Direct dispatch to the local DSQ.
	*
	* Task bypasses BPF scheduler entirely: no enqueue
	* tracking, no ops.dequeue() callbacks.
	*/
	scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
	tctx->state = TASK_DISPATCHED;
	break;
	case 1:
	/*
	* Direct dispatch to the global DSQ.
	*
	* Task bypasses BPF scheduler entirely: no enqueue
	* tracking, no ops.dequeue() callbacks.
	*/
	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
	tctx->state = TASK_DISPATCHED;
	break;
	case 2:
	/*
	* Dispatch to a shared user DSQ.
	*
	* Task enters BPF scheduler management: track
	* enqueue/dequeue lifecycle and validate state
	* transitions.
	*/
	if (tctx->state == TASK_ENQUEUED)
	scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
	p->pid, p->comm, tctx->enqueue_seq);

	scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, 0);

	__sync_fetch_and_add(&enqueue_cnt, 1);

	tctx->state = TASK_ENQUEUED;
	tctx->enqueue_seq++;
	break;
	}

	return prev_cpu;
	}

	void BPF_STRUCT_OPS(dequeue_enqueue, struct task_struct *p, u64 enq_flags)
	{
	struct task_ctx *tctx;
	s32 pid = p->pid;

	tctx = try_lookup_task_ctx(p);
	if (!tctx)
	return;

	switch (test_scenario) {
	case 3:
	/*
	* Direct dispatch to the local DSQ.
	*
	* Task bypasses BPF scheduler entirely: no enqueue
	* tracking, no ops.dequeue() callbacks.
	*/
	scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
	tctx->state = TASK_DISPATCHED;
	break;
	case 4:
	/*
	* Direct dispatch to the global DSQ.
	*
	* Task bypasses BPF scheduler entirely: no enqueue
	* tracking, no ops.dequeue() callbacks.
	*/
	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
	tctx->state = TASK_DISPATCHED;
	break;
	case 5:
	/*
	* Dispatch to shared user DSQ.
	*
	* Task enters BPF scheduler management: track
	* enqueue/dequeue lifecycle and validate state
	* transitions.
	*/
	if (tctx->state == TASK_ENQUEUED)
	scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
	p->pid, p->comm, tctx->enqueue_seq);

	scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);

	__sync_fetch_and_add(&enqueue_cnt, 1);

	tctx->state = TASK_ENQUEUED;
	tctx->enqueue_seq++;
	break;
	case 6:
	/*
	* Store task in BPF internal queue.
	*
	* Task enters BPF scheduler management: track
	* enqueue/dequeue lifecycle and validate state
	* transitions.
	*/
	if (tctx->state == TASK_ENQUEUED)
	scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu",
	p->pid, p->comm, tctx->enqueue_seq);

	if (bpf_map_push_elem(&global_queue, &pid, 0)) {
	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
	__sync_fetch_and_add(&bpf_queue_full, 1);

	tctx->state = TASK_DISPATCHED;
	} else {
	__sync_fetch_and_add(&enqueue_cnt, 1);

	tctx->state = TASK_ENQUEUED;
	tctx->enqueue_seq++;
	}
	break;
	default:
	/* For all other scenarios, dispatch to the global DSQ */
	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
	tctx->state = TASK_DISPATCHED;
	break;
	}

	scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE);
	}

	void BPF_STRUCT_OPS(dequeue_dequeue, struct task_struct *p, u64 deq_flags)
	{
	struct task_ctx *tctx;

	__sync_fetch_and_add(&dequeue_cnt, 1);

	tctx = try_lookup_task_ctx(p);
	if (!tctx)
	return;

	/*
	* For scenarios 0, 1, 3, and 4 (terminal DSQs: local and global),
	* ops.dequeue() should never be called because tasks bypass the
	* BPF scheduler entirely. If we get here, it's a kernel bug.
	*/
	if (test_scenario == 0 \|\| test_scenario == 3) {
	scx_bpf_error("%d (%s): dequeue called for local DSQ scenario",
	p->pid, p->comm);
	return;
	}

	if (test_scenario == 1 \|\| test_scenario == 4) {
	scx_bpf_error("%d (%s): dequeue called for global DSQ scenario",
	p->pid, p->comm);
	return;
	}

	if (deq_flags & SCX_DEQ_SCHED_CHANGE) {
	/*
	* Property change interrupting the workflow. Valid from
	* both ENQUEUED and DISPATCHED states. Transitions task
	* back to NONE state.
	*/
	__sync_fetch_and_add(&change_dequeue_cnt, 1);

	/* Validate state transition */
	if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_DISPATCHED)
	scx_bpf_error("%d (%s): invalid property change dequeue state=%d seq=%llu",
	p->pid, p->comm, tctx->state, tctx->enqueue_seq);

	/*
	* Transition back to NONE: task outside scheduler control.
	*
	* Scenario 6: dispatch() checks tctx->state after popping a
	* PID, if the task is in state NONE, it was dequeued by
	* property change and must not be dispatched (this
	* prevents "target CPU not allowed").
	*/
	tctx->state = TASK_NONE;
	} else {
	/*
	* Regular dispatch dequeue: kernel is moving the task from
	* BPF custody to a terminal DSQ. Normally we come from
	* ENQUEUED state. We can also see TASK_NONE if the task
	* was dequeued by property change (SCX_DEQ_SCHED_CHANGE)
	* while it was already on a DSQ (dispatched but not yet
	* consumed); in that case we just leave state as NONE.
	*/
	__sync_fetch_and_add(&dispatch_dequeue_cnt, 1);

	/*
	* Must be ENQUEUED (normal path) or NONE (already dequeued
	* by property change while on a DSQ).
	*/
	if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_NONE)
	scx_bpf_error("%d (%s): dispatch dequeue from state %d seq=%llu",
	p->pid, p->comm, tctx->state, tctx->enqueue_seq);

	if (tctx->state == TASK_ENQUEUED)
	tctx->state = TASK_DISPATCHED;

	/* NONE: leave as-is, task was already property-change dequeued */
	}
	}

	void BPF_STRUCT_OPS(dequeue_dispatch, s32 cpu, struct task_struct *prev)
	{
	if (test_scenario == 6) {
	struct task_ctx *tctx;
	struct task_struct *p;
	s32 pid;

	if (bpf_map_pop_elem(&global_queue, &pid))
	return;

	p = bpf_task_from_pid(pid);
	if (!p)
	return;

	/*
	* If the task was dequeued by property change
	* (ops.dequeue() set tctx->state = TASK_NONE), skip
	* dispatch.
	*/
	tctx = try_lookup_task_ctx(p);
	if (!tctx \|\| tctx->state == TASK_NONE) {
	bpf_task_release(p);
	return;
	}

	/*
	* Dispatch to this CPU's local DSQ if allowed, otherwise
	* fallback to the global DSQ.
	*/
	if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr))
	scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON \| cpu, SCX_SLICE_DFL, 0);
	else
	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);

	bpf_task_release(p);
	} else {
	scx_bpf_dsq_move_to_local(SHARED_DSQ, 0);
	}
	}

	s32 BPF_STRUCT_OPS(dequeue_init_task, struct task_struct *p,
	struct scx_init_task_args *args)
	{
	struct task_ctx *tctx;

	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0,
	BPF_LOCAL_STORAGE_GET_F_CREATE);
	if (!tctx)
	return -ENOMEM;

	return 0;
	}

	s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_init)
	{
	s32 ret;

	ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
	if (ret)
	return ret;

	return 0;
	}

	void BPF_STRUCT_OPS(dequeue_exit, struct scx_exit_info *ei)
	{
	UEI_RECORD(uei, ei);
	}

	SEC(".struct_ops.link")
	struct sched_ext_ops dequeue_ops = {
	.select_cpu = (void *)dequeue_select_cpu,
	.enqueue = (void *)dequeue_enqueue,
	.dequeue = (void *)dequeue_dequeue,
	.dispatch = (void *)dequeue_dispatch,
	.init_task = (void *)dequeue_init_task,
	.init = (void *)dequeue_init,
	.exit = (void *)dequeue_exit,
	.flags = SCX_OPS_ENQ_LAST,
	.name = "dequeue_test",
	};