/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ #ifndef __DRM_GPUVM_H__ #define __DRM_GPUVM_H__ /* * Copyright (c) 2022 Red Hat. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-resv.h> #include <linux/list.h> #include <linux/rbtree.h> #include <linux/types.h> #include <drm/drm_device.h> #include <drm/drm_gem.h> #include <drm/drm_exec.h> struct drm_gpuvm; struct drm_gpuvm_bo; struct drm_gpuvm_ops; /** * enum drm_gpuva_flags - flags for struct drm_gpuva */ enum drm_gpuva_flags { … }; /** * struct drm_gpuva - structure to track a GPU VA mapping * * This structure represents a GPU VA mapping and is associated with a * &drm_gpuvm. * * Typically, this structure is embedded in bigger driver structures. */ struct drm_gpuva { … }; int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); void drm_gpuva_remove(struct drm_gpuva *va); void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo); void drm_gpuva_unlink(struct drm_gpuva *va); struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start); struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end); static inline void drm_gpuva_init(struct drm_gpuva *va, u64 addr, u64 range, struct drm_gem_object *obj, u64 offset) { … } /** * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is * invalidated * @va: the &drm_gpuva to set the invalidate flag for * @invalidate: indicates whether the &drm_gpuva is invalidated */ static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate) { … } /** * drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva * is invalidated * @va: the &drm_gpuva to check * * Returns: %true if the GPU VA is invalidated, %false otherwise */ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) { … } /** * enum drm_gpuvm_flags - flags for struct drm_gpuvm */ enum drm_gpuvm_flags { … }; /** * struct drm_gpuvm - DRM GPU VA Manager * * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using * &maple_tree structures. Typically, this structure is embedded in bigger * driver structures. * * Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or * pages. * * There should be one manager instance per GPU virtual address space. */ struct drm_gpuvm { … }; void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, enum drm_gpuvm_flags flags, struct drm_device *drm, struct drm_gem_object *r_obj, u64 start_offset, u64 range, u64 reserve_offset, u64 reserve_range, const struct drm_gpuvm_ops *ops); /** * drm_gpuvm_get() - acquire a struct drm_gpuvm reference * @gpuvm: the &drm_gpuvm to acquire the reference of * * This function acquires an additional reference to @gpuvm. It is illegal to * call this without already holding a reference. No locks required. * * Returns: the &struct drm_gpuvm pointer */ static inline struct drm_gpuvm * drm_gpuvm_get(struct drm_gpuvm *gpuvm) { … } void drm_gpuvm_put(struct drm_gpuvm *gpuvm); bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gem_object * drm_gpuvm_resv_object_alloc(struct drm_device *drm); /** * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is * set * @gpuvm: the &drm_gpuvm * * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise. */ static inline bool drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm) { … } /** * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv * @gpuvm__: the &drm_gpuvm * * Returns: a pointer to the &drm_gpuvm's shared &dma_resv */ #define drm_gpuvm_resv(gpuvm__) … /** * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's * &dma_resv * @gpuvm__: the &drm_gpuvm * * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared * &dma_resv */ #define drm_gpuvm_resv_obj(gpuvm__) … #define drm_gpuvm_resv_held(gpuvm__) … #define drm_gpuvm_resv_assert_held(gpuvm__) … #define drm_gpuvm_resv_held(gpuvm__) … #define drm_gpuvm_resv_assert_held(gpuvm__) … /** * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an * external object * @gpuvm: the &drm_gpuvm to check * @obj: the &drm_gem_object to check * * Returns: true if the &drm_gem_object &dma_resv differs from the * &drm_gpuvms &dma_resv, false otherwise */ static inline bool drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) { … } static inline struct drm_gpuva * __drm_gpuva_next(struct drm_gpuva *va) { … } /** * drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas * @va__: &drm_gpuva structure to assign to in each iteration step * @gpuvm__: &drm_gpuvm to walk over * @start__: starting offset, the first gpuva will overlap this * @end__: ending offset, the last gpuva will start before this (but may * overlap) * * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie * between @start__ and @end__. It is implemented similarly to list_for_each(), * but is using the &drm_gpuvm's internal interval tree to accelerate * the search for the starting &drm_gpuva, and hence isn't safe against removal * of elements. It assumes that @end__ is within (or is the upper limit of) the * &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's * @kernel_alloc_node. */ #define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) … /** * drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of * &drm_gpuvas * @va__: &drm_gpuva to assign to in each iteration step * @next__: another &drm_gpuva to use as temporary storage * @gpuvm__: &drm_gpuvm to walk over * @start__: starting offset, the first gpuva will overlap this * @end__: ending offset, the last gpuva will start before this (but may * overlap) * * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie * between @start__ and @end__. It is implemented similarly to * list_for_each_safe(), but is using the &drm_gpuvm's internal interval * tree to accelerate the search for the starting &drm_gpuva, and hence is safe * against removal of elements. It assumes that @end__ is within (or is the * upper limit of) the &drm_gpuvm. This iterator does not skip over the * &drm_gpuvm's @kernel_alloc_node. */ #define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) … /** * drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas * @va__: &drm_gpuva to assign to in each iteration step * @gpuvm__: &drm_gpuvm to walk over * * This iterator walks over all &drm_gpuva structures associated with the given * &drm_gpuvm. */ #define drm_gpuvm_for_each_va(va__, gpuvm__) … /** * drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas * @va__: &drm_gpuva to assign to in each iteration step * @next__: another &drm_gpuva to use as temporary storage * @gpuvm__: &drm_gpuvm to walk over * * This iterator walks over all &drm_gpuva structures associated with the given * &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and * hence safe against the removal of elements. */ #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) … /** * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec * * This structure should be created on the stack as &drm_exec should be. * * Optionally, @extra can be set in order to lock additional &drm_gem_objects. */ struct drm_gpuvm_exec { … }; int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences); int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences); int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, u64 addr, u64 range, unsigned int num_fences); int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec); int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, struct drm_gem_object **objs, unsigned int num_objs); int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, u64 addr, u64 range); /** * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs * @vm_exec: the &drm_gpuvm_exec wrapper * * Releases all dma-resv locks of all &drm_gem_objects previously acquired * through drm_gpuvm_exec_lock() or its variants. * * Returns: 0 on success, negative error code on failure. */ static inline void drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) { … } int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec); void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, struct drm_exec *exec, struct dma_fence *fence, enum dma_resv_usage private_usage, enum dma_resv_usage extobj_usage); /** * drm_gpuvm_exec_resv_add_fence() - add fence to private and all extobj * @vm_exec: the &drm_gpuvm_exec wrapper * @fence: fence to add * @private_usage: private dma-resv usage * @extobj_usage: extobj dma-resv usage * * See drm_gpuvm_resv_add_fence(). */ static inline void drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec, struct dma_fence *fence, enum dma_resv_usage private_usage, enum dma_resv_usage extobj_usage) { … } /** * drm_gpuvm_exec_validate() - validate all BOs marked as evicted * @vm_exec: the &drm_gpuvm_exec wrapper * * See drm_gpuvm_validate(). * * Returns: 0 on success, negative error code on failure. */ static inline int drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec) { … } /** * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and * &drm_gem_object combination * * This structure is an abstraction representing a &drm_gpuvm and * &drm_gem_object combination. It serves as an indirection to accelerate * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same * &drm_gem_object. * * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to * accelerate validation. * * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once * a GEM object is mapped first in a GPU-VM and release the instance once the * last mapping of the GEM object in this GPU-VM is unmapped. */ struct drm_gpuvm_bo { … }; struct drm_gpuvm_bo * drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj); struct drm_gpuvm_bo * drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj); struct drm_gpuvm_bo * drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo); /** * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference * @vm_bo: the &drm_gpuvm_bo to acquire the reference of * * This function acquires an additional reference to @vm_bo. It is illegal to * call this without already holding a reference. No locks required. * * Returns: the &struct vm_bo pointer */ static inline struct drm_gpuvm_bo * drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) { … } bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); struct drm_gpuvm_bo * drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj); void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict); /** * drm_gpuvm_bo_gem_evict() - add/remove all &drm_gpuvm_bo's in the list * to/from the &drm_gpuvms evicted list * @obj: the &drm_gem_object * @evict: indicates whether @obj is evicted * * See drm_gpuvm_bo_evict(). */ static inline void drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict) { … } void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo); /** * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva * @va__: &drm_gpuva structure to assign to in each iteration step * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with * * This iterator walks over all &drm_gpuva structures associated with the * &drm_gpuvm_bo. * * The caller must hold the GEM's gpuva lock. */ #define drm_gpuvm_bo_for_each_va(va__, vm_bo__) … /** * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of * &drm_gpuva * @va__: &drm_gpuva structure to assign to in each iteration step * @next__: &next &drm_gpuva to store the next step * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with * * This iterator walks over all &drm_gpuva structures associated with the * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence * it is save against removal of elements. * * The caller must hold the GEM's gpuva lock. */ #define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) … /** * enum drm_gpuva_op_type - GPU VA operation type * * Operations to alter the GPU VA mappings tracked by the &drm_gpuvm. */ enum drm_gpuva_op_type { … }; /** * struct drm_gpuva_op_map - GPU VA map operation * * This structure represents a single map operation generated by the * DRM GPU VA manager. */ struct drm_gpuva_op_map { … }; /** * struct drm_gpuva_op_unmap - GPU VA unmap operation * * This structure represents a single unmap operation generated by the * DRM GPU VA manager. */ struct drm_gpuva_op_unmap { … }; /** * struct drm_gpuva_op_remap - GPU VA remap operation * * This represents a single remap operation generated by the DRM GPU VA manager. * * A remap operation is generated when an existing GPU VA mmapping is split up * by inserting a new GPU VA mapping or by partially unmapping existent * mapping(s), hence it consists of a maximum of two map and one unmap * operation. * * The @unmap operation takes care of removing the original existing mapping. * @prev is used to remap the preceding part, @next the subsequent part. * * If either a new mapping's start address is aligned with the start address * of the old mapping or the new mapping's end address is aligned with the * end address of the old mapping, either @prev or @next is NULL. * * Note, the reason for a dedicated remap operation, rather than arbitrary * unmap and map operations, is to give drivers the chance of extracting driver * specific data for creating the new mappings from the unmap operations's * &drm_gpuva structure which typically is embedded in larger driver specific * structures. */ struct drm_gpuva_op_remap { … }; /** * struct drm_gpuva_op_prefetch - GPU VA prefetch operation * * This structure represents a single prefetch operation generated by the * DRM GPU VA manager. */ struct drm_gpuva_op_prefetch { … }; /** * struct drm_gpuva_op - GPU VA operation * * This structure represents a single generic operation. * * The particular type of the operation is defined by @op. */ struct drm_gpuva_op { … }; /** * struct drm_gpuva_ops - wraps a list of &drm_gpuva_op */ struct drm_gpuva_ops { … }; /** * drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops * @op: &drm_gpuva_op to assign in each iteration step * @ops: &drm_gpuva_ops to walk * * This iterator walks over all ops within a given list of operations. */ #define drm_gpuva_for_each_op(op, ops) … /** * drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops * @op: &drm_gpuva_op to assign in each iteration step * @next: &next &drm_gpuva_op to store the next step * @ops: &drm_gpuva_ops to walk * * This iterator walks over all ops within a given list of operations. It is * implemented with list_for_each_safe(), so save against removal of elements. */ #define drm_gpuva_for_each_op_safe(op, next, ops) … /** * drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point * @op: &drm_gpuva_op to assign in each iteration step * @ops: &drm_gpuva_ops to walk * * This iterator walks over all ops within a given list of operations beginning * from the given operation in reverse order. */ #define drm_gpuva_for_each_op_from_reverse(op, ops) … /** * drm_gpuva_for_each_op_reverse - iterator to walk over &drm_gpuva_ops in reverse * @op: &drm_gpuva_op to assign in each iteration step * @ops: &drm_gpuva_ops to walk * * This iterator walks over all ops within a given list of operations in reverse */ #define drm_gpuva_for_each_op_reverse(op, ops) … /** * drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops * @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from */ #define drm_gpuva_first_op(ops) … /** * drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops * @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from */ #define drm_gpuva_last_op(ops) … /** * drm_gpuva_prev_op() - previous &drm_gpuva_op in the list * @op: the current &drm_gpuva_op */ #define drm_gpuva_prev_op(op) … /** * drm_gpuva_next_op() - next &drm_gpuva_op in the list * @op: the current &drm_gpuva_op */ #define drm_gpuva_next_op(op) … struct drm_gpuva_ops * drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, u64 addr, u64 range, struct drm_gem_object *obj, u64 offset); struct drm_gpuva_ops * drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gpuva_ops * drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gpuva_ops * drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo); void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, struct drm_gpuva_ops *ops); static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) { … } /** * struct drm_gpuvm_ops - callbacks for split/merge steps * * This structure defines the callbacks used by &drm_gpuvm_sm_map and * &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap * operations to drivers. */ struct drm_gpuvm_ops { … }; int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, u64 addr, u64 range, struct drm_gem_object *obj, u64 offset); int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 addr, u64 range); void drm_gpuva_map(struct drm_gpuvm *gpuvm, struct drm_gpuva *va, struct drm_gpuva_op_map *op); void drm_gpuva_remap(struct drm_gpuva *prev, struct drm_gpuva *next, struct drm_gpuva_op_remap *op); void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op); /** * drm_gpuva_op_remap_to_unmap_range() - Helper to get the start and range of * the unmap stage of a remap op. * @op: Remap op. * @start_addr: Output pointer for the start of the required unmap. * @range: Output pointer for the length of the required unmap. * * The given start address and range will be set such that they represent the * range of the address space that was previously covered by the mapping being * re-mapped, but is now empty. */ static inline void drm_gpuva_op_remap_to_unmap_range(const struct drm_gpuva_op_remap *op, u64 *start_addr, u64 *range) { … } #endif /* __DRM_GPUVM_H__ */
Codebase Browser
linux