blob: 85739a63e85e6f2784716da2d91c3a2588367865 [file]
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
#include <linux/irq_work_types.h>
#include <linux/types.h>
#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
/*
* rseq_event::has_rseq contains the ABI version number so preserving it
* in AND operations requires a mask.
*/
#define RSEQ_HAS_RSEQ_VERSION_MASK 0xff
/**
* struct rseq_event - Storage for rseq related event management
* @all: Compound to initialize and clear the data efficiently
* @events: Compound to access events with a single load/store
* @sched_switch: True if the task was scheduled and needs update on
* exit to user
* @ids_changed: Indicator that IDs need to be updated
* @user_irq: True on interrupt entry from user mode
* @has_rseq: Greater than 0 if the task has a rseq pointer installed.
* Contains the RSEQ version number
* @error: Compound error code for the slow path to analyze
* @fatal: User space data corrupted or invalid
* @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME
* is required
*
* @sched_switch and @ids_changed must be adjacent and the combo must be
* 16bit aligned to allow a single store, when both are set at the same
* time in the scheduler.
*/
struct rseq_event {
union {
u64 all;
struct {
union {
u32 events;
struct {
u8 sched_switch;
u8 ids_changed;
u8 user_irq;
};
};
u8 has_rseq;
u8 __pad;
union {
u16 error;
struct {
u8 fatal;
u8 slowpath;
};
};
};
};
};
/**
* struct rseq_ids - Cache for ids, which need to be updated
* @cpu_cid: Compound of @cpu_id and @mm_cid to make the
* compiler emit a single compare on 64-bit
* @cpu_id: The CPU ID which was written last to user space
* @mm_cid: The MM CID which was written last to user space
* @node_id: The node ID which was written last to user space
*
* @cpu_id, @mm_cid and @node_id are updated when the data is written to user space.
*/
struct rseq_ids {
union {
u64 cpu_cid;
struct {
u32 cpu_id;
u32 mm_cid;
};
};
u32 node_id;
};
/**
* union rseq_slice_state - Status information for rseq time slice extension
* @state: Compound to access the overall state
* @enabled: Time slice extension is enabled for the task
* @granted: Time slice extension was granted to the task
*/
union rseq_slice_state {
u16 state;
struct {
u8 enabled;
u8 granted;
};
};
/**
* struct rseq_slice - Status information for rseq time slice extension
* @state: Time slice extension state
* @expires: The time when a grant expires
* @yielded: Indicator for rseq_slice_yield()
*/
struct rseq_slice {
union rseq_slice_state state;
u64 expires;
u8 yielded;
};
/**
* struct rseq_data - Storage for all rseq related data
* @usrptr: Pointer to the registered user space RSEQ memory
* @len: Length of the RSEQ region
* @sig: Signature of critical section abort IPs
* @event: Storage for event management
* @ids: Storage for cached CPU ID and MM CID
* @slice: Storage for time slice extension data
*/
struct rseq_data {
struct rseq __user *usrptr;
u32 len;
u32 sig;
struct rseq_event event;
struct rseq_ids ids;
#ifdef CONFIG_RSEQ_SLICE_EXTENSION
struct rseq_slice slice;
#endif
};
#else /* CONFIG_RSEQ */
struct rseq_data { };
#endif /* !CONFIG_RSEQ */
#ifdef CONFIG_SCHED_MM_CID
#define MM_CID_UNSET BIT(31)
#define MM_CID_ONCPU BIT(30)
#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
* @cid: The CID associated to the task either permanently or
* borrowed from the CPU
* @node: Queued in the per MM MMCID list
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
struct hlist_node node;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
* @cid: The CID associated to the CPU either permanently or
* while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
}____cacheline_aligned_in_smp;
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
* @mode: Indicates per CPU and transition mode
* @max_cids: The exclusive maximum CID value for allocation and convergence
* @irq_work: irq_work to handle the affinity mode change case
* @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @user_list: List of the MM CID users of a MM
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
* @pcpu_thrs: Threshold for switching back from per CPU mode
* @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
unsigned int mode;
unsigned int max_cids;
/* Rarely used. Moves @lock and @mutex into the second cacheline */
struct irq_work irq_work;
struct work_struct work;
raw_spinlock_t lock;
struct mutex mutex;
struct hlist_head user_list;
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
unsigned int pcpu_thrs;
unsigned int update_deferred;
} ____cacheline_aligned;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
struct sched_mm_cid { };
#endif /* !CONFIG_SCHED_MM_CID */
#endif