| /* SPDX-License-Identifier: GPL-2.0 */ | |
| #ifndef _LINUX_RSEQ_TYPES_H | |
| #define _LINUX_RSEQ_TYPES_H | |
| #include <linux/irq_work_types.h> | |
| #include <linux/types.h> | |
| #include <linux/workqueue_types.h> | |
| #ifdef CONFIG_RSEQ | |
| struct rseq; | |
| /* | |
| * rseq_event::has_rseq contains the ABI version number so preserving it | |
| * in AND operations requires a mask. | |
| */ | |
| #define RSEQ_HAS_RSEQ_VERSION_MASK 0xff | |
| /** | |
| * struct rseq_event - Storage for rseq related event management | |
| * @all: Compound to initialize and clear the data efficiently | |
| * @events: Compound to access events with a single load/store | |
| * @sched_switch: True if the task was scheduled and needs update on | |
| * exit to user | |
| * @ids_changed: Indicator that IDs need to be updated | |
| * @user_irq: True on interrupt entry from user mode | |
| * @has_rseq: Greater than 0 if the task has a rseq pointer installed. | |
| * Contains the RSEQ version number | |
| * @error: Compound error code for the slow path to analyze | |
| * @fatal: User space data corrupted or invalid | |
| * @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME | |
| * is required | |
| * | |
| * @sched_switch and @ids_changed must be adjacent and the combo must be | |
| * 16bit aligned to allow a single store, when both are set at the same | |
| * time in the scheduler. | |
| */ | |
| struct rseq_event { | |
| union { | |
| u64 all; | |
| struct { | |
| union { | |
| u32 events; | |
| struct { | |
| u8 sched_switch; | |
| u8 ids_changed; | |
| u8 user_irq; | |
| }; | |
| }; | |
| u8 has_rseq; | |
| u8 __pad; | |
| union { | |
| u16 error; | |
| struct { | |
| u8 fatal; | |
| u8 slowpath; | |
| }; | |
| }; | |
| }; | |
| }; | |
| }; | |
| /** | |
| * struct rseq_ids - Cache for ids, which need to be updated | |
| * @cpu_cid: Compound of @cpu_id and @mm_cid to make the | |
| * compiler emit a single compare on 64-bit | |
| * @cpu_id: The CPU ID which was written last to user space | |
| * @mm_cid: The MM CID which was written last to user space | |
| * @node_id: The node ID which was written last to user space | |
| * | |
| * @cpu_id, @mm_cid and @node_id are updated when the data is written to user space. | |
| */ | |
| struct rseq_ids { | |
| union { | |
| u64 cpu_cid; | |
| struct { | |
| u32 cpu_id; | |
| u32 mm_cid; | |
| }; | |
| }; | |
| u32 node_id; | |
| }; | |
| /** | |
| * union rseq_slice_state - Status information for rseq time slice extension | |
| * @state: Compound to access the overall state | |
| * @enabled: Time slice extension is enabled for the task | |
| * @granted: Time slice extension was granted to the task | |
| */ | |
| union rseq_slice_state { | |
| u16 state; | |
| struct { | |
| u8 enabled; | |
| u8 granted; | |
| }; | |
| }; | |
| /** | |
| * struct rseq_slice - Status information for rseq time slice extension | |
| * @state: Time slice extension state | |
| * @expires: The time when a grant expires | |
| * @yielded: Indicator for rseq_slice_yield() | |
| */ | |
| struct rseq_slice { | |
| union rseq_slice_state state; | |
| u64 expires; | |
| u8 yielded; | |
| }; | |
| /** | |
| * struct rseq_data - Storage for all rseq related data | |
| * @usrptr: Pointer to the registered user space RSEQ memory | |
| * @len: Length of the RSEQ region | |
| * @sig: Signature of critical section abort IPs | |
| * @event: Storage for event management | |
| * @ids: Storage for cached CPU ID and MM CID | |
| * @slice: Storage for time slice extension data | |
| */ | |
| struct rseq_data { | |
| struct rseq __user *usrptr; | |
| u32 len; | |
| u32 sig; | |
| struct rseq_event event; | |
| struct rseq_ids ids; | |
| #ifdef CONFIG_RSEQ_SLICE_EXTENSION | |
| struct rseq_slice slice; | |
| #endif | |
| }; | |
| #else /* CONFIG_RSEQ */ | |
| struct rseq_data { }; | |
| #endif /* !CONFIG_RSEQ */ | |
| #ifdef CONFIG_SCHED_MM_CID | |
| #define MM_CID_UNSET BIT(31) | |
| #define MM_CID_ONCPU BIT(30) | |
| #define MM_CID_TRANSIT BIT(29) | |
| /** | |
| * struct sched_mm_cid - Storage for per task MM CID data | |
| * @active: MM CID is active for the task | |
| * @cid: The CID associated to the task either permanently or | |
| * borrowed from the CPU | |
| * @node: Queued in the per MM MMCID list | |
| */ | |
| struct sched_mm_cid { | |
| unsigned int active; | |
| unsigned int cid; | |
| struct hlist_node node; | |
| }; | |
| /** | |
| * struct mm_cid_pcpu - Storage for per CPU MM_CID data | |
| * @cid: The CID associated to the CPU either permanently or | |
| * while a task with a CID is running | |
| */ | |
| struct mm_cid_pcpu { | |
| unsigned int cid; | |
| }____cacheline_aligned_in_smp; | |
| /** | |
| * struct mm_mm_cid - Storage for per MM CID data | |
| * @pcpu: Per CPU storage for CIDs associated to a CPU | |
| * @mode: Indicates per CPU and transition mode | |
| * @max_cids: The exclusive maximum CID value for allocation and convergence | |
| * @irq_work: irq_work to handle the affinity mode change case | |
| * @work: Regular work to handle the affinity mode change case | |
| * @lock: Spinlock to protect against affinity setting which can't take @mutex | |
| * @mutex: Mutex to serialize forks and exits related to this mm | |
| * @user_list: List of the MM CID users of a MM | |
| * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map | |
| * is growth only. | |
| * @users: The number of tasks sharing this MM. Separate from mm::mm_users | |
| * as that is modified by mmget()/mm_put() by other entities which | |
| * do not actually share the MM. | |
| * @pcpu_thrs: Threshold for switching back from per CPU mode | |
| * @update_deferred: A deferred switch back to per task mode is pending. | |
| */ | |
| struct mm_mm_cid { | |
| /* Hotpath read mostly members */ | |
| struct mm_cid_pcpu __percpu *pcpu; | |
| unsigned int mode; | |
| unsigned int max_cids; | |
| /* Rarely used. Moves @lock and @mutex into the second cacheline */ | |
| struct irq_work irq_work; | |
| struct work_struct work; | |
| raw_spinlock_t lock; | |
| struct mutex mutex; | |
| struct hlist_head user_list; | |
| /* Low frequency modified */ | |
| unsigned int nr_cpus_allowed; | |
| unsigned int users; | |
| unsigned int pcpu_thrs; | |
| unsigned int update_deferred; | |
| } ____cacheline_aligned; | |
| #else /* CONFIG_SCHED_MM_CID */ | |
| struct mm_mm_cid { }; | |
| struct sched_mm_cid { }; | |
| #endif /* !CONFIG_SCHED_MM_CID */ | |
| #endif |