include/drm/drm_gpusvm.h

/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
/*
 * Copyright © 2024 Intel Corporation
 */

#ifndef __DRM_GPUSVM_H__
#define __DRM_GPUSVM_H__

#include <linux/dma-mapping.h>
#include <linux/kref.h>
#include <linux/interval_tree.h>
#include <linux/mmu_notifier.h>

struct dev_pagemap_ops;
struct drm_device;
struct drm_gpusvm;
struct drm_gpusvm_notifier;
struct drm_gpusvm_ops;
struct drm_gpusvm_range;
struct drm_pagemap;
struct drm_pagemap_addr;

/**
 * struct drm_gpusvm_ops - Operations structure for GPU SVM
 *
 * This structure defines the operations for GPU Shared Virtual Memory (SVM).
 * These operations are provided by the GPU driver to manage SVM ranges and
 * notifiers.
 */
struct drm_gpusvm_ops {
	/**
	 * @notifier_alloc: Allocate a GPU SVM notifier (optional)
	 *
	 * Allocate a GPU SVM notifier.
	 *
	 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure.
	 */
	struct drm_gpusvm_notifier *(*notifier_alloc)(void);

	/**
	 * @notifier_free: Free a GPU SVM notifier (optional)
	 * @notifier: Pointer to the GPU SVM notifier to be freed
	 *
	 * Free a GPU SVM notifier.
	 */
	void (*notifier_free)(struct drm_gpusvm_notifier *notifier);

	/**
	 * @range_alloc: Allocate a GPU SVM range (optional)
	 * @gpusvm: Pointer to the GPU SVM
	 *
	 * Allocate a GPU SVM range.
	 *
	 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure.
	 */
	struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);

	/**
	 * @range_free: Free a GPU SVM range (optional)
	 * @range: Pointer to the GPU SVM range to be freed
	 *
	 * Free a GPU SVM range.
	 */
	void (*range_free)(struct drm_gpusvm_range *range);

	/**
	 * @invalidate: Invalidate GPU SVM notifier (required)
	 * @gpusvm: Pointer to the GPU SVM
	 * @notifier: Pointer to the GPU SVM notifier
	 * @mmu_range: Pointer to the mmu_notifier_range structure
	 *
	 * Invalidate the GPU page tables. It can safely walk the notifier range
	 * RB tree/list in this function. Called while holding the notifier lock.
	 */
	void (*invalidate)(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_notifier *notifier,
			   const struct mmu_notifier_range *mmu_range);
};

/**
 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier
 *
 * @gpusvm: Pointer to the GPU SVM structure
 * @notifier: MMU interval notifier
 * @itree: Interval tree node for the notifier (inserted in GPU SVM)
 * @entry: List entry to fast interval tree traversal
 * @root: Cached root node of the RB tree containing ranges
 * @range_list: List head containing of ranges in the same order they appear in
 *              interval tree. This is useful to keep iterating ranges while
 *              doing modifications to RB tree.
 * @flags: Flags for notifier
 * @flags.removed: Flag indicating whether the MMU interval notifier has been
 *                 removed
 *
 * This structure represents a GPU SVM notifier.
 */
struct drm_gpusvm_notifier {
	struct drm_gpusvm *gpusvm;
	struct mmu_interval_notifier notifier;
	struct interval_tree_node itree;
	struct list_head entry;
	struct rb_root_cached root;
	struct list_head range_list;
	struct {
		u32 removed : 1;
	} flags;
};

/**
 * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags
 *
 * @unmapped: Flag indicating if the pages has been unmapped
 * @has_devmem_pages: Flag indicating if the pages has devmem pages
 * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
 * @__flags: Flags for pages in u16 form (used for READ_ONCE)
 */
struct drm_gpusvm_pages_flags {
	union {
		struct {
			/* All flags below must be set / cleared under notifier lock */
			u16 unmapped : 1;
			u16 has_devmem_pages : 1;
			u16 has_dma_mapping : 1;
		};
		u16 __flags;
	};
};

/**
 * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
 *
 * @drm: The DRM device that owns the dma mappings
 * @dma_addr: Device address array
 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
 *            Note this is assuming only one drm_pagemap per range is allowed.
 * @state: DMA IOVA state for mapping.
 * @state_offset: DMA IOVA offset for mapping.
 * @notifier_seq: Notifier sequence number of the range's pages
 * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
 */
struct drm_gpusvm_pages {
	struct drm_device *drm;
	struct drm_pagemap_addr *dma_addr;
	struct drm_pagemap *dpagemap;
	struct dma_iova_state state;
	unsigned long state_offset;
	unsigned long notifier_seq;
	struct drm_gpusvm_pages_flags flags;
};

/**
 * struct drm_gpusvm_range_flags - Range-level GPU SVM flags
 *
 * @migrate_devmem: Flag indicating whether the range can be migrated to device memory
 * @unmapped: Flag indicating if the range has been unmapped
 * @partial_unmap: Flag indicating if the range has been partially unmapped
 * @__flags: All flags in u16 form (used for READ_ONCE)
 */
struct drm_gpusvm_range_flags {
	union {
		struct {
			/* All flags below must be set upon creation */
			u16 migrate_devmem	: 1;
			/* All flags below must be set / cleared under notifier lock */
			u16 unmapped		: 1;
			u16 partial_unmap	: 1;
		};
		u16 __flags;
	};
};

/**
 * struct drm_gpusvm_range - Structure representing a GPU SVM range
 *
 * @gpusvm: Pointer to the GPU SVM structure
 * @notifier: Pointer to the GPU SVM notifier
 * @refcount: Reference count for the range
 * @itree: Interval tree node for the range (inserted in GPU SVM notifier)
 * @entry: List entry to fast interval tree traversal
 * @flags: Flags for range see &struct drm_gpusvm_range_flags
 *
 * This structure represents a GPU SVM range used for tracking memory ranges
 * mapped in a DRM device.
 */
struct drm_gpusvm_range {
	struct drm_gpusvm *gpusvm;
	struct drm_gpusvm_notifier *notifier;
	struct kref refcount;
	struct interval_tree_node itree;
	struct list_head entry;
	struct drm_gpusvm_range_flags flags;
};

/**
 * struct drm_gpusvm - GPU SVM structure
 *
 * @name: Name of the GPU SVM
 * @mm: Pointer to the mm_struct for the address space
 * @mm_start: Start address of GPU SVM
 * @mm_range: Range of the GPU SVM
 * @notifier_size: Size of individual notifiers
 * @ops: Pointer to the operations structure for GPU SVM
 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
 *               Entries should be powers of 2 in descending order.
 * @num_chunks: Number of chunks
 * @notifier_lock: Read-write semaphore for protecting notifier operations
 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers
 * @notifier_list: list head containing of notifiers in the same order they
 *                 appear in interval tree. This is useful to keep iterating
 *                 notifiers while doing modifications to RB tree.
 *
 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking
 * memory ranges mapped in a DRM (Direct Rendering Manager) device.
 *
 * No reference counting is provided, as this is expected to be embedded in the
 * driver VM structure along with the struct drm_gpuvm, which handles reference
 * counting.
 */
struct drm_gpusvm {
	const char *name;
	struct mm_struct *mm;
	unsigned long mm_start;
	unsigned long mm_range;
	unsigned long notifier_size;
	const struct drm_gpusvm_ops *ops;
	const unsigned long *chunk_sizes;
	int num_chunks;
	struct rw_semaphore notifier_lock;
	struct rb_root_cached root;
	struct list_head notifier_list;
#ifdef CONFIG_LOCKDEP
	/**
	 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and
	 * drm_gpusvm_range_remove with a driver provided lock.
	 */
	struct lockdep_map *lock_dep_map;
#endif
};

/**
 * struct drm_gpusvm_ctx - DRM GPU SVM context
 *
 * @device_private_page_owner: The device-private page owner to use for
 * this operation
 * @check_pages_threshold: Check CPU pages for present if chunk is less than or
 *                         equal to threshold. If not present, reduce chunk
 *                         size.
 * @timeslice_ms: The timeslice MS which in minimum time a piece of memory
 *		  remains with either exclusive GPU or CPU access.
 * @in_notifier: entering from a MMU notifier
 * @read_only: operating on read-only memory
 * @devmem_possible: possible to use device memory
 * @devmem_only: use only device memory
 * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and
 *               single dpagemap is supported, mixing between multiple dpagemap
 *               is unsupported.
 *
 * Context that is DRM GPUSVM is operating in (i.e. user arguments).
 */
struct drm_gpusvm_ctx {
	void *device_private_page_owner;
	unsigned long check_pages_threshold;
	unsigned long timeslice_ms;
	unsigned int in_notifier :1;
	unsigned int read_only :1;
	unsigned int devmem_possible :1;
	unsigned int devmem_only :1;
	unsigned int allow_mixed :1;
};

int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
		    const char *name,
		    struct mm_struct *mm,
		    unsigned long mm_start, unsigned long mm_range,
		    unsigned long notifier_size,
		    const struct drm_gpusvm_ops *ops,
		    const unsigned long *chunk_sizes, int num_chunks);

void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);

void drm_gpusvm_free(struct drm_gpusvm *gpusvm);

unsigned long
drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
			  unsigned long start,
			  unsigned long end);

struct drm_gpusvm_range *
drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
				unsigned long fault_addr,
				unsigned long gpuva_start,
				unsigned long gpuva_end,
				const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
			     struct drm_gpusvm_range *range);

int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_range *range);

struct drm_gpusvm_range *
drm_gpusvm_range_get(struct drm_gpusvm_range *range);

void drm_gpusvm_range_put(struct drm_gpusvm_range *range);

bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
			    struct drm_gpusvm_pages *svm_pages);

bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
			    unsigned long end);

struct drm_gpusvm_notifier *
drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
			 unsigned long end);

struct drm_gpusvm_range *
drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
		      unsigned long end);

void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
				   struct drm_gpusvm_pages *pages,
				   unsigned int pages_count,
				   const struct mmu_notifier_range *mmu_range);

int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
			 struct drm_gpusvm_pages *svm_pages,
			 struct mm_struct *mm,
			 struct mmu_interval_notifier *notifier,
			 unsigned long pages_start, unsigned long pages_end,
			 const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
			    struct drm_gpusvm_pages *svm_pages,
			    unsigned long npages,
			    const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_pages *svm_pages,
			   unsigned long npages);

/**
 * drm_gpusvm_init_pages() - Initialize a freshly allocated drm_gpusvm_pages
 * @svm_pages: Pointer to the drm_gpusvm_pages to initialize.
 * @drm: The DRM device that will own DMA mappings for this pages object.
 *
 * Drivers that embed one or more drm_gpusvm_pages in their own range
 * structure must call this once on each pages instance after allocation,
 * before the first drm_gpusvm_get_pages() / unmap / free.
 */
static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages,
					 struct drm_device *drm)
{
	memset(svm_pages, 0, sizeof(*svm_pages));
	svm_pages->drm = drm;
	svm_pages->notifier_seq = LONG_MAX;
}

/**
 * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
 * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
 * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as
 * the @pagemap argument to the drm_gpusvm_scan_mm() function.
 * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is
 * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to
 * the same device private owner.
 * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages.
 * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least
 * two different struct dev_pagemaps. All pages belong to the same device private
 * owner.
 * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages
 * and device-private pages. All device-private pages belong to the same device
 * private owner.
 */
enum drm_gpusvm_scan_result {
	DRM_GPUSVM_SCAN_UNPOPULATED,
	DRM_GPUSVM_SCAN_EQUAL,
	DRM_GPUSVM_SCAN_OTHER,
	DRM_GPUSVM_SCAN_SYSTEM,
	DRM_GPUSVM_SCAN_MIXED_DEVICE,
	DRM_GPUSVM_SCAN_MIXED,
};

enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
					       void *dev_private_owner,
					       const struct dev_pagemap *pagemap);

#ifdef CONFIG_LOCKDEP
/**
 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM
 * @gpusvm: Pointer to the GPU SVM structure.
 * @lock: the lock used to protect the gpuva list. The locking primitive
 * must contain a dep_map field.
 *
 * Call this to annotate drm_gpusvm_range_find_or_insert and
 * drm_gpusvm_range_remove.
 */
#define drm_gpusvm_driver_set_lock(gpusvm, lock) \
	do { \
		if (!WARN((gpusvm)->lock_dep_map, \
			  "GPUSVM range lock should be set only once."))\
			(gpusvm)->lock_dep_map = &(lock)->dep_map;	\
	} while (0)
#else
#define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
#endif

/**
 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier
 * @gpusvm__: Pointer to the GPU SVM structure.
 *
 * Abstract client usage GPU SVM notifier lock, take lock
 */
#define drm_gpusvm_notifier_lock(gpusvm__)	\
	down_read(&(gpusvm__)->notifier_lock)

/**
 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier
 * @gpusvm__: Pointer to the GPU SVM structure.
 *
 * Abstract client usage GPU SVM notifier lock, drop lock
 */
#define drm_gpusvm_notifier_unlock(gpusvm__)	\
	up_read(&(gpusvm__)->notifier_lock)

/**
 * drm_gpusvm_range_start() - GPU SVM range start address
 * @range: Pointer to the GPU SVM range
 *
 * Return: GPU SVM range start address
 */
static inline unsigned long
drm_gpusvm_range_start(struct drm_gpusvm_range *range)
{
	return range->itree.start;
}

/**
 * drm_gpusvm_range_end() - GPU SVM range end address
 * @range: Pointer to the GPU SVM range
 *
 * Return: GPU SVM range end address
 */
static inline unsigned long
drm_gpusvm_range_end(struct drm_gpusvm_range *range)
{
	return range->itree.last + 1;
}

/**
 * drm_gpusvm_range_size() - GPU SVM range size
 * @range: Pointer to the GPU SVM range
 *
 * Return: GPU SVM range size
 */
static inline unsigned long
drm_gpusvm_range_size(struct drm_gpusvm_range *range)
{
	return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
}

/**
 * drm_gpusvm_notifier_start() - GPU SVM notifier start address
 * @notifier: Pointer to the GPU SVM notifier
 *
 * Return: GPU SVM notifier start address
 */
static inline unsigned long
drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
{
	return notifier->itree.start;
}

/**
 * drm_gpusvm_notifier_end() - GPU SVM notifier end address
 * @notifier: Pointer to the GPU SVM notifier
 *
 * Return: GPU SVM notifier end address
 */
static inline unsigned long
drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
{
	return notifier->itree.last + 1;
}

/**
 * drm_gpusvm_notifier_size() - GPU SVM notifier size
 * @notifier: Pointer to the GPU SVM notifier
 *
 * Return: GPU SVM notifier size
 */
static inline unsigned long
drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
{
	return drm_gpusvm_notifier_end(notifier) -
		drm_gpusvm_notifier_start(notifier);
}

/**
 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list
 * @range: a pointer to the current GPU SVM range
 *
 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the
 *         current range is the last one or if the input range is NULL.
 */
static inline struct drm_gpusvm_range *
__drm_gpusvm_range_next(struct drm_gpusvm_range *range)
{
	if (range && !list_is_last(&range->entry,
				   &range->notifier->range_list))
		return list_next_entry(range, entry);

	return NULL;
}

/**
 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier
 * @range__: Iterator variable for the ranges. If set, it indicates the start of
 *	     the iterator. If NULL, call drm_gpusvm_range_find() to get the range.
 * @notifier__: Pointer to the GPU SVM notifier
 * @start__: Start address of the range
 * @end__: End address of the range
 *
 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe
 * to use while holding the driver SVM lock or the notifier lock.
 */
#define drm_gpusvm_for_each_range(range__, notifier__, start__, end__)	\
	for ((range__) = (range__) ?:					\
	     drm_gpusvm_range_find((notifier__), (start__), (end__));	\
	     (range__) && (drm_gpusvm_range_start(range__) < (end__));	\
	     (range__) = __drm_gpusvm_range_next(range__))

/**
 * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier
 * @range__: Iterator variable for the ranges
 * @next__: Iterator variable for the ranges temporay storage
 * @notifier__: Pointer to the GPU SVM notifier
 * @start__: Start address of the range
 * @end__: End address of the range
 *
 * This macro is used to iterate over GPU SVM ranges in a notifier while
 * removing ranges from it.
 */
#define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__)	\
	for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)),	\
	     (next__) = __drm_gpusvm_range_next(range__);				\
	     (range__) && (drm_gpusvm_range_start(range__) < (end__));			\
	     (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))

/**
 * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list
 * @notifier: a pointer to the current drm_gpusvm_notifier
 *
 * Return: A pointer to the next drm_gpusvm_notifier if available, or NULL if
 *         the current notifier is the last one or if the input notifier is
 *         NULL.
 */
static inline struct drm_gpusvm_notifier *
__drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
{
	if (notifier && !list_is_last(&notifier->entry,
				      &notifier->gpusvm->notifier_list))
		return list_next_entry(notifier, entry);

	return NULL;
}

/**
 * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm
 * @notifier__: Iterator variable for the notifiers
 * @gpusvm__: Pointer to the GPU SVM notifier
 * @start__: Start address of the notifier
 * @end__: End address of the notifier
 *
 * This macro is used to iterate over GPU SVM notifiers in a gpusvm.
 */
#define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__)		\
	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__));	\
	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
	     (notifier__) = __drm_gpusvm_notifier_next(notifier__))

/**
 * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm
 * @notifier__: Iterator variable for the notifiers
 * @next__: Iterator variable for the notifiers temporay storage
 * @gpusvm__: Pointer to the GPU SVM notifier
 * @start__: Start address of the notifier
 * @end__: End address of the notifier
 *
 * This macro is used to iterate over GPU SVM notifiers in a gpusvm while
 * removing notifiers from it.
 */
#define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__)	\
	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)),	\
	     (next__) = __drm_gpusvm_notifier_next(notifier__);				\
	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
	     (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))

#endif /* __DRM_GPUSVM_H__ */