blob: 9960011edadc5036f33cb9a7b4cbd6814af7473e [file]
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 KylinSoft Corporation. */
#include <vmlinux.h>
#include <stdbool.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
char _license[] SEC("license") = "GPL";
extern bool CONFIG_PREEMPT_RT __kconfig __weak;
/*
* Bounds for the hardirq busy-wait that holds the scheduling handler
* inside the real window: task_work_add() has published the callback,
* but the handler's tail state transition has not run yet.
* irq_work_single() clears BUSY only after the handler returns, so
* while this wait runs, the context's scheduling irq_work is BUSY,
* even though the first callback may already have completed and
* returned the context to STANDBY.
*
* The wait is bounded both by a bpf_ktime_get_ns() check and by an
* iteration budget. The time bound is checked against a monotonic
* clock: host vCPU preemption can stretch the wall-clock duration of
* the window, which only makes the window longer, never shorter.
*/
#define BUSY_WAIT_NS 4000000ULL
#define BUSY_WAIT_ITERS 1000000ULL
int trigger_tid;
int target_tid;
/*
* High 32 bits of the result slots hold the sequence, low 32 bits the
* signed error. BUSY_WINDOW low bits: 1 open, 2 the in-window attempt
* was seen before the handler returned, 3 the wait bound was reached
* without it. Must match the test.
*/
enum task_work_busy_status {
BUSY_ARM_SEQ,
BUSY_WINDOW,
BUSY_ACK,
BUSY_GEN1_RESULT,
BUSY_GEN2_RESULT,
BUSY_CB1_SEQ,
BUSY_CB2_COUNT,
BUSY_CB2_SEQ,
BUSY_RECOVER_SEQ,
BUSY_LAST_WAIT_NS,
BUSY_STATUS_MAX,
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, BUSY_STATUS_MAX);
__type(key, int);
__type(value, __s64);
} status SEC(".maps");
struct task_work_busy_value {
__u32 seq;
char data[60];
struct bpf_task_work tw;
};
/*
* max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each
* attempt schedules into its own fresh value.
*/
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
__uint(max_entries, 128);
__type(key, int);
__type(value, struct task_work_busy_value);
} hmap SEC(".maps");
static __always_inline void set_status(int key, __s64 value)
{
__s64 *slot;
slot = bpf_map_lookup_elem(&status, &key);
if (slot)
*slot = value;
}
static __always_inline __s64 get_status(int key)
{
__s64 *slot = bpf_map_lookup_elem(&status, &key);
return slot ? *slot : 0;
}
struct busy_wait_ctx {
__u64 seq;
__u64 start;
bool acked;
};
/* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */
static long busy_wait_cb(__u32 i, void *pctx)
{
struct busy_wait_ctx *w = pctx;
if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) {
w->acked = true;
return 1;
}
if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS)
return 1;
return 0;
}
static int busy_cb_gen1(struct bpf_map *map, void *key, void *value)
{
struct task_work_busy_value *work = value;
__u64 we = (__u64)get_status(BUSY_WINDOW) >> 32;
/*
* Backstop for deliveries from abandoned attempts: accept only
* while no window for a different sequence is published. The
* test waits for the old workers to disappear from PID lookup,
* after their exit task_work, before resetting the shared state.
* This callback may run before fexit publishes the window state.
*/
if (we && we != (__u64)work->seq)
return 0;
set_status(BUSY_CB1_SEQ, work->seq);
return 0;
}
static int busy_cb_gen2(struct bpf_map *map, void *key, void *value)
{
struct task_work_busy_value *work = value;
__s64 *count;
int key2 = BUSY_CB2_COUNT;
/*
* Count only deliveries belonging to the sequence whose window is
* (or was) published: a late delivery from an abandoned attempt
* must not satisfy a later exactly-once check.
*/
if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32))
return 0;
set_status(BUSY_CB2_SEQ, work->seq);
count = bpf_map_lookup_elem(&status, &key2);
if (count)
__sync_fetch_and_add(count, 1);
return 0;
}
SEC("fexit/task_work_add")
int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task,
struct callback_head *work, int notify, int retval)
{
struct busy_wait_ctx w = {};
__u64 seq, elapsed;
seq = (__u64)get_status(BUSY_ARM_SEQ);
if (!seq)
return 0;
/*
* One-shot arm. Only this test's own first-round add may consume
* it and hold the window: a successful TWA_SIGNAL add of the
* bpf_task_work callback, issued from the trigger thread's CPU by
* the self-IPI'd scheduling handler and targeting the target
* thread. Everything else leaves the arm untouched.
*/
if (retval != 0 || notify != TWA_SIGNAL ||
BPF_CORE_READ(task, pid) != target_tid)
return 0;
if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid)
return 0;
set_status(BUSY_ARM_SEQ, 0);
set_status(BUSY_WINDOW, (seq << 32) | 1);
w.seq = seq;
w.start = bpf_ktime_get_ns();
bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0);
elapsed = bpf_ktime_get_ns() - w.start;
/*
* Publish elapsed time for logging after worker cleanup. Window
* validity depends on seeing ACK here, not on this diagnostic.
*/
set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed);
if (w.acked)
set_status(BUSY_WINDOW, (seq << 32) | 2);
else
set_status(BUSY_WINDOW, (seq << 32) | 3);
return 0;
}
/*
* Attempt the second scheduling round on the same context, targeting the
* trigger thread. The attempt uses the value keyed by its own sequence,
* so it can never touch another attempt's context. Returns true if a
* terminal result was published into res_key; a transient -EBUSY is left
* unpublished for the caller's retry policy, mirroring the race test's
* convention.
*/
static __always_inline bool busy_try_gen2(__u64 seq, int res_key)
{
struct task_work_busy_value *work;
struct task_struct *task;
int key = (int)seq, err;
task = bpf_task_from_pid(trigger_tid);
if (!task) {
set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH);
return true;
}
work = bpf_map_lookup_elem(&hmap, &key);
if (!work) {
bpf_task_release(task);
set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT);
return true;
}
work->seq = seq;
err = bpf_task_work_schedule_signal(task, &work->tw, &hmap,
busy_cb_gen2);
bpf_task_release(task);
if (err == -EBUSY)
return false;
set_status(res_key, ((__s64)seq << 32) | (__u32)err);
return true;
}
SEC("tracepoint/syscalls/sys_enter_getppid")
int busy_sched_work(void *ctx)
{
__u32 tid = (__u32)bpf_get_current_pid_tgid();
struct task_work_busy_value *work;
struct task_struct *task;
__u64 window, seq;
int key, err;
/* Also keep the kconfig extern in object BTF for the userspace skip. */
if (CONFIG_PREEMPT_RT)
return 0;
if (tid == (__u32)trigger_tid) {
/*
* Trigger thread: schedule the first round once per armed
* sequence, targeting the target thread. The kfunc queues
* the scheduling irq_work and returns; the handler then
* parks inside the fexit window above. A fresh value has no
* expected BUSY source, so publish every scheduling result.
*/
seq = (__u64)get_status(BUSY_ARM_SEQ);
if (!seq ||
((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq)
return 0;
key = (int)seq;
task = bpf_task_from_pid(target_tid);
if (!task) {
set_status(BUSY_GEN1_RESULT,
((__s64)seq << 32) | (__u32)-ESRCH);
set_status(BUSY_ARM_SEQ, 0);
return 0;
}
work = bpf_map_lookup_elem(&hmap, &key);
if (!work) {
bpf_task_release(task);
set_status(BUSY_GEN1_RESULT,
((__s64)seq << 32) | (__u32)-ENOENT);
set_status(BUSY_ARM_SEQ, 0);
return 0;
}
work->seq = seq;
err = bpf_task_work_schedule_signal(task, &work->tw, &hmap,
busy_cb_gen1);
bpf_task_release(task);
set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err);
return 0;
}
if (tid != (__u32)target_tid)
return 0;
window = (__u64)get_status(BUSY_WINDOW);
seq = window >> 32;
if (!seq || (__u32)window == 0)
return 0;
/*
* The first callback must have completed for this sequence. Being
* inside the target thread's own subsequent syscall proves the
* callback wrapper has returned and the context is STANDBY, which
* is stronger than observing the completion marker from another
* thread.
*/
if ((__u64)get_status(BUSY_CB1_SEQ) != seq)
return 0;
if ((__u64)get_status(BUSY_ACK) >> 32 != seq) {
/*
* The in-window attempt, exactly once per sequence. On a
* fixed kernel it must be rejected with -EBUSY: the context
* is STANDBY, the map and program references are alive and
* no other scheduler exists, so only the BUSY gate can
* reject here.
*/
if (!busy_try_gen2(seq, BUSY_ACK))
set_status(BUSY_ACK,
((__s64)seq << 32) | (__u32)-EBUSY);
return 0;
}
/*
* Recovery, gated by the test: the fexit program releasing the
* window does not mean the whole old handler has returned and
* BUSY has cleared yet, so retry until the schedule succeeds.
*/
if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq)
return 0;
if ((__s32)get_status(BUSY_ACK) == 0)
return 0; /* scheduled in-window; nothing to recover */
if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq)
return 0; /* terminal recovery result published */
busy_try_gen2(seq, BUSY_GEN2_RESULT);
return 0;
}