// SPDX-License-Identifier: GPL-2.0-only OR MIT /* * Copyright (c) 2022 Red Hat. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: * Danilo Krummrich <[email protected]> * */ #include <drm/drm_gpuvm.h> #include <linux/interval_tree_generic.h> #include <linux/mm.h> /** * DOC: Overview * * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a * GPU's virtual address (VA) space and manages the corresponding virtual * mappings represented by &drm_gpuva objects. It also keeps track of the * mapping's backing &drm_gem_object buffers. * * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing * all existent GPU VA mappings using this &drm_gem_object as backing buffer. * * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'. * * The GPU VA manager internally uses a rb-tree to manage the * &drm_gpuva mappings within a GPU's virtual address space. * * The &drm_gpuvm structure contains a special &drm_gpuva representing the * portion of VA space reserved by the kernel. This node is initialized together * with the GPU VA manager instance and removed when the GPU VA manager is * destroyed. * * In a typical application drivers would embed struct drm_gpuvm and * struct drm_gpuva within their own driver specific structures, there won't be * any memory allocations of its own nor memory allocations of &drm_gpuva * entries. * * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva * entries from within dma-fence signalling critical sections it is enough to * pre-allocate the &drm_gpuva structures. * * &drm_gem_objects which are private to a single VM can share a common * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec). * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in * the following called 'resv object', which serves as the container of the * GPUVM's shared &dma_resv. This resv object can be a driver specific * &drm_gem_object, such as the &drm_gem_object containing the root page table, * but it can also be a 'dummy' object, which can be allocated with * drm_gpuvm_resv_object_alloc(). * * In order to connect a struct drm_gpuva its backing &drm_gem_object each * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each * &drm_gpuvm_bo contains a list of &drm_gpuva structures. * * A &drm_gpuvm_bo is an abstraction that represents a combination of a * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique. * This is ensured by the API through drm_gpuvm_bo_obtain() and * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this * particular combination. If not existent a new instance is created and linked * to the &drm_gem_object. * * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used * as entry for the &drm_gpuvm's lists of external and evicted objects. Those * lists are maintained in order to accelerate locking of dma-resv locks and * validation of evicted objects bound in a &drm_gpuvm. For instance, all * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in * order to validate all evicted &drm_gem_objects. It is also possible to lock * additional &drm_gem_objects by providing the corresponding parameters to * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making * use of helper functions such as drm_gpuvm_prepare_range() or * drm_gpuvm_prepare_objects(). * * Every bound &drm_gem_object is treated as external object when its &dma_resv * structure is different than the &drm_gpuvm's common &dma_resv structure. */ /** * DOC: Split and Merge * * Besides its capability to manage and represent a GPU VA space, the * GPU VA manager also provides functions to let the &drm_gpuvm calculate a * sequence of operations to satisfy a given map or unmap request. * * Therefore the DRM GPU VA manager provides an algorithm implementing splitting * and merging of existent GPU VA mappings with the ones that are requested to * be mapped or unmapped. This feature is required by the Vulkan API to * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this * as VM BIND. * * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks * containing map, unmap and remap operations for a given newly requested * mapping. The sequence of callbacks represents the set of operations to * execute in order to integrate the new mapping cleanly into the current state * of the GPU VA space. * * Depending on how the new GPU VA mapping intersects with the existent mappings * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount * of unmap operations, a maximum of two remap operations and a single map * operation. The caller might receive no callback at all if no operation is * required, e.g. if the requested mapping already exists in the exact same way. * * The single map operation represents the original map operation requested by * the caller. * * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the * &drm_gpuva to unmap is physically contiguous with the original mapping * request. Optionally, if 'keep' is set, drivers may keep the actual page table * entries for this &drm_gpuva, adding the missing page table entries only and * update the &drm_gpuvm's view of things accordingly. * * Drivers may do the same optimization, namely delta page table updates, also * for remap operations. This is possible since &drm_gpuva_op_remap consists of * one unmap operation and one or two map operations, such that drivers can * derive the page table update delta accordingly. * * Note that there can't be more than two existent mappings to split up, one at * the beginning and one at the end of the new mapping, hence there is a * maximum of two remap operations. * * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to * call back into the driver in order to unmap a range of GPU VA space. The * logic behind this function is way simpler though: For all existent mappings * enclosed by the given range unmap operations are created. For mappings which * are only partically located within the given range, remap operations are * created such that those mappings are split up and re-mapped partically. * * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(), * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used * to directly obtain an instance of struct drm_gpuva_ops containing a list of * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list * contains the &drm_gpuva_ops analogous to the callbacks one would receive when * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory * allocations are possible (e.g. to allocate GPU page tables) and once in the * dma-fence signalling critical path. * * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and * drm_gpuva_remove() may be used. These functions can safely be used from * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and * drm_gpuva_unmap() instead. * * The following diagram depicts the basic relationships of existent GPU VA * mappings, a newly requested mapping and the resulting mappings as implemented * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these. * * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs * could be kept. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----------| (bo_offset=n) * * 0 a 1 * new: |-----------| (bo_offset=n) * * * 2) Requested mapping is identical, except for the BO offset, hence replace * the mapping. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----------| (bo_offset=m) * * 0 a 1 * new: |-----------| (bo_offset=m) * * * 3) Requested mapping is identical, except for the backing BO, hence replace * the mapping. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 b 1 * req: |-----------| (bo_offset=n) * * 0 b 1 * new: |-----------| (bo_offset=n) * * * 4) Existent mapping is a left aligned subset of the requested one, hence * replace the existent one. * * :: * * 0 a 1 * old: |-----| (bo_offset=n) * * 0 a 2 * req: |-----------| (bo_offset=n) * * 0 a 2 * new: |-----------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different BO * and/or non-contiguous BO offset. * * * 5) Requested mapping's range is a left aligned subset of the existent one, * but backed by a different BO. Hence, map the requested mapping and split * the existent one adjusting its BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 0 b 1 * req: |-----| (bo_offset=n) * * 0 b 1 a' 2 * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1) * * .. note:: * We expect to see the same result for a request with a different BO * and/or non-contiguous BO offset. * * * 6) Existent mapping is a superset of the requested mapping. Split it up, but * indicate that the backing PTEs could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----| (bo_offset=n) * * 0 a 1 a' 2 * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1) * * * 7) Requested mapping's range is a right aligned subset of the existent one, * but backed by a different BO. Hence, map the requested mapping and split * the existent one, without adjusting the BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 b 2 * req: |-----| (bo_offset=m) * * 0 a 1 b 2 * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m) * * * 8) Existent mapping is a superset of the requested mapping. Split it up, but * indicate that the backing PTEs could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 a 2 * req: |-----| (bo_offset=n+1) * * 0 a' 1 a 2 * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1) * * * 9) Existent mapping is overlapped at the end by the requested mapping backed * by a different BO. Hence, map the requested mapping and split up the * existent one, without adjusting the BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 b 3 * req: |-----------| (bo_offset=m) * * 0 a 1 b 3 * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m) * * * 10) Existent mapping is overlapped by the requested mapping, both having the * same backing BO with a contiguous offset. Indicate the backing PTEs of * the old mapping could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 a 3 * req: |-----------| (bo_offset=n+1) * * 0 a' 1 a 3 * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1) * * * 11) Requested mapping's range is a centered subset of the existent one * having a different backing BO. Hence, map the requested mapping and split * up the existent one in two mappings, adjusting the BO offset of the right * one accordingly. * * :: * * 0 a 3 * old: |-----------------| (bo_offset=n) * * 1 b 2 * req: |-----| (bo_offset=m) * * 0 a 1 b 2 a' 3 * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2) * * * 12) Requested mapping is a contiguous subset of the existent one. Split it * up, but indicate that the backing PTEs could be kept. * * :: * * 0 a 3 * old: |-----------------| (bo_offset=n) * * 1 a 2 * req: |-----| (bo_offset=n+1) * * 0 a' 1 a 2 a'' 3 * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2) * * * 13) Existent mapping is a right aligned subset of the requested one, hence * replace the existent one. * * :: * * 1 a 2 * old: |-----| (bo_offset=n+1) * * 0 a 2 * req: |-----------| (bo_offset=n) * * 0 a 2 * new: |-----------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different bo * and/or non-contiguous bo_offset. * * * 14) Existent mapping is a centered subset of the requested one, hence * replace the existent one. * * :: * * 1 a 2 * old: |-----| (bo_offset=n+1) * * 0 a 3 * req: |----------------| (bo_offset=n) * * 0 a 3 * new: |----------------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different bo * and/or non-contiguous bo_offset. * * * 15) Existent mappings is overlapped at the beginning by the requested mapping * backed by a different BO. Hence, map the requested mapping and split up * the existent one, adjusting its BO offset accordingly. * * :: * * 1 a 3 * old: |-----------| (bo_offset=n) * * 0 b 2 * req: |-----------| (bo_offset=m) * * 0 b 2 a' 3 * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2) */ /** * DOC: Locking * * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of * locking itself, it is the drivers responsibility to take care about locking. * Drivers might want to protect the following operations: inserting, removing * and iterating &drm_gpuva objects as well as generating all kinds of * operations, such as split / merge or prefetch. * * DRM GPUVM also does not take care of the locking of the backing * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by * itself; drivers are responsible to enforce mutual exclusion using either the * GEMs dma_resv lock or alternatively a driver specific external lock. For the * latter see also drm_gem_gpuva_set_lock(). * * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put(). * * The latter is required since on creation and destruction of a &drm_gpuvm_bo * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list. * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and * &drm_gem_object must be able to observe previous creations and destructions * of &drm_gpuvm_bos in order to keep instances unique. * * The &drm_gpuvm's lists for keeping track of external and evicted objects are * protected against concurrent insertion / removal and iteration internally. * * However, drivers still need ensure to protect concurrent calls to functions * iterating those lists, namely drm_gpuvm_prepare_objects() and * drm_gpuvm_validate(). * * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate * that the corresponding &dma_resv locks are held in order to protect the * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and * the corresponding lockdep checks are enabled. This is an optimization for * drivers which are capable of taking the corresponding &dma_resv locks and * hence do not require internal locking. */ /** * DOC: Examples * * This section gives two examples on how to let the DRM GPUVA Manager generate * &drm_gpuva_op in order to satisfy a given map or unmap request and how to * make use of them. * * The below code is strictly limited to illustrate the generic usage pattern. * To maintain simplicitly, it doesn't make use of any abstractions for common * code, different (asyncronous) stages with fence signalling critical paths, * any other helpers or error handling in terms of freeing memory and dropping * previously taken locks. * * 1) Obtain a list of &drm_gpuva_op to create a new mapping:: * * // Allocates a new &drm_gpuva. * struct drm_gpuva * driver_gpuva_alloc(void); * * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva * // structure in individual driver structures and lock the dma-resv with * // drm_exec or similar helpers. * int driver_mapping_create(struct drm_gpuvm *gpuvm, * u64 addr, u64 range, * struct drm_gem_object *obj, u64 offset) * { * struct drm_gpuva_ops *ops; * struct drm_gpuva_op *op * struct drm_gpuvm_bo *vm_bo; * * driver_lock_va_space(); * ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range, * obj, offset); * if (IS_ERR(ops)) * return PTR_ERR(ops); * * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj); * if (IS_ERR(vm_bo)) * return PTR_ERR(vm_bo); * * drm_gpuva_for_each_op(op, ops) { * struct drm_gpuva *va; * * switch (op->op) { * case DRM_GPUVA_OP_MAP: * va = driver_gpuva_alloc(); * if (!va) * ; // unwind previous VA space updates, * // free memory and unlock * * driver_vm_map(); * drm_gpuva_map(gpuvm, va, &op->map); * drm_gpuva_link(va, vm_bo); * * break; * case DRM_GPUVA_OP_REMAP: { * struct drm_gpuva *prev = NULL, *next = NULL; * * va = op->remap.unmap->va; * * if (op->remap.prev) { * prev = driver_gpuva_alloc(); * if (!prev) * ; // unwind previous VA space * // updates, free memory and * // unlock * } * * if (op->remap.next) { * next = driver_gpuva_alloc(); * if (!next) * ; // unwind previous VA space * // updates, free memory and * // unlock * } * * driver_vm_remap(); * drm_gpuva_remap(prev, next, &op->remap); * * if (prev) * drm_gpuva_link(prev, va->vm_bo); * if (next) * drm_gpuva_link(next, va->vm_bo); * drm_gpuva_unlink(va); * * break; * } * case DRM_GPUVA_OP_UNMAP: * va = op->unmap->va; * * driver_vm_unmap(); * drm_gpuva_unlink(va); * drm_gpuva_unmap(&op->unmap); * * break; * default: * break; * } * } * drm_gpuvm_bo_put(vm_bo); * driver_unlock_va_space(); * * return 0; * } * * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: * * struct driver_context { * struct drm_gpuvm *gpuvm; * struct drm_gpuvm_bo *vm_bo; * struct drm_gpuva *new_va; * struct drm_gpuva *prev_va; * struct drm_gpuva *next_va; * }; * * // ops to pass to drm_gpuvm_init() * static const struct drm_gpuvm_ops driver_gpuvm_ops = { * .sm_step_map = driver_gpuva_map, * .sm_step_remap = driver_gpuva_remap, * .sm_step_unmap = driver_gpuva_unmap, * }; * * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva * // structure in individual driver structures and lock the dma-resv with * // drm_exec or similar helpers. * int driver_mapping_create(struct drm_gpuvm *gpuvm, * u64 addr, u64 range, * struct drm_gem_object *obj, u64 offset) * { * struct driver_context ctx; * struct drm_gpuvm_bo *vm_bo; * struct drm_gpuva_ops *ops; * struct drm_gpuva_op *op; * int ret = 0; * * ctx.gpuvm = gpuvm; * * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj); * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) { * ret = -ENOMEM; * goto out; * } * * // Typically protected with a driver specific GEM gpuva lock * // used in the fence signaling path for drm_gpuva_link() and * // drm_gpuva_unlink(), hence pre-allocate. * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo); * * driver_lock_va_space(); * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); * driver_unlock_va_space(); * * out: * drm_gpuvm_bo_put(ctx.vm_bo); * kfree(ctx.new_va); * kfree(ctx.prev_va); * kfree(ctx.next_va); * return ret; * } * * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx) * { * struct driver_context *ctx = __ctx; * * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); * * drm_gpuva_link(ctx->new_va, ctx->vm_bo); * * // prevent the new GPUVA from being freed in * // driver_mapping_create() * ctx->new_va = NULL; * * return 0; * } * * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) * { * struct driver_context *ctx = __ctx; * struct drm_gpuva *va = op->remap.unmap->va; * * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); * * if (op->remap.prev) { * drm_gpuva_link(ctx->prev_va, va->vm_bo); * ctx->prev_va = NULL; * } * * if (op->remap.next) { * drm_gpuva_link(ctx->next_va, va->vm_bo); * ctx->next_va = NULL; * } * * drm_gpuva_unlink(va); * kfree(va); * * return 0; * } * * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx) * { * drm_gpuva_unlink(op->unmap.va); * drm_gpuva_unmap(&op->unmap); * kfree(op->unmap.va); * * return 0; * } */ /** * get_next_vm_bo_from_list() - get the next vm_bo element * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * @__local_list: a pointer to the local list used to store already iterated items * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list() * * This helper is here to provide lockless list iteration. Lockless as in, the * iterator releases the lock immediately after picking the first element from * the list, so list insertion deletion can happen concurrently. * * Elements popped from the original list are kept in a local list, so removal * and is_empty checks can still happen while we're iterating the list. */ #define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) … /** * for_each_vm_bo_in_list() - internal vm_bo list iterator * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * @__local_list: a pointer to the local list used to store already iterated items * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step * * This helper is here to provide lockless list iteration. Lockless as in, the * iterator releases the lock immediately after picking the first element from the * list, hence list insertion and deletion can happen concurrently. * * It is not allowed to re-assign the vm_bo pointer from inside this loop. * * Typical use: * * struct drm_gpuvm_bo *vm_bo; * LIST_HEAD(my_local_list); * * ret = 0; * for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) { * ret = do_something_with_vm_bo(..., vm_bo); * if (ret) * break; * } * // Drop ref in case we break out of the loop. * drm_gpuvm_bo_put(vm_bo); * restore_vm_bo_list(gpuvm, <list_name>, &my_local_list); * * * Only used for internal list iterations, not meant to be exposed to the outside * world. */ #define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) … static void __restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *list, struct list_head **local_list) { … } /** * restore_vm_bo_list() - move vm_bo elements back to their original list * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * * When we're done iterating a vm_bo list, we should call restore_vm_bo_list() * to restore the original state and let new iterations take place. */ #define restore_vm_bo_list(__gpuvm, __list_name) … static void cond_spin_lock(spinlock_t *lock, bool cond) { … } static void cond_spin_unlock(spinlock_t *lock, bool cond) { … } static void __drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *entry, struct list_head *list) { … } /** * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Inserts the given @__vm_bo into the list specified by @__list_name. */ #define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) … static void __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *entry, bool init) { … } /** * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Removes the given @__vm_bo from the list specified by @__list_name. */ #define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) … /** * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Removes the given @__vm_bo from the list specified by @__list_name. */ #define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) … #define to_drm_gpuva(__node) … #define GPUVA_START(node) … #define GPUVA_LAST(node) … /* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain * about this. */ INTERVAL_TREE_DEFINE(…) static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); static void __drm_gpuva_remove(struct drm_gpuva *va); static bool drm_gpuvm_check_overflow(u64 addr, u64 range) { … } static bool drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } static bool drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } static bool drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } /** * drm_gpuvm_range_valid() - checks whether the given range is valid for the * given &drm_gpuvm * @gpuvm: the GPUVM to check the range for * @addr: the base address * @range: the range starting from the base address * * Checks whether the range is within the GPUVM's managed boundaries. * * Returns: true for a valid range, false otherwise */ bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); static void drm_gpuvm_gem_object_free(struct drm_gem_object *obj) { … } static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = …; /** * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object * @drm: the drivers &drm_device * * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in * order to serve as root GEM object providing the &drm_resv shared across * &drm_gem_objects local to a single GPUVM. * * Returns: the &drm_gem_object on success, NULL on failure */ struct drm_gem_object * drm_gpuvm_resv_object_alloc(struct drm_device *drm) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_init() - initialize a &drm_gpuvm * @gpuvm: pointer to the &drm_gpuvm to initialize * @name: the name of the GPU VA space * @flags: the &drm_gpuvm_flags for this GPUVM * @drm: the &drm_device this VM resides in * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv * @start_offset: the start offset of the GPU VA space * @range: the size of the GPU VA space * @reserve_offset: the start of the kernel reserved GPU VA area * @reserve_range: the size of the kernel reserved GPU VA area * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap * * The &drm_gpuvm must be initialized with this function before use. * * Note that @gpuvm must be cleared to 0 before calling this function. The given * &name is expected to be managed by the surrounding driver structures. */ void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, enum drm_gpuvm_flags flags, struct drm_device *drm, struct drm_gem_object *r_obj, u64 start_offset, u64 range, u64 reserve_offset, u64 reserve_range, const struct drm_gpuvm_ops *ops) { … } EXPORT_SYMBOL_GPL(…); static void drm_gpuvm_fini(struct drm_gpuvm *gpuvm) { … } static void drm_gpuvm_free(struct kref *kref) { … } /** * drm_gpuvm_put() - drop a struct drm_gpuvm reference * @gpuvm: the &drm_gpuvm to release the reference of * * This releases a reference to @gpuvm. * * This function may be called from atomic context. */ void drm_gpuvm_put(struct drm_gpuvm *gpuvm) { … } EXPORT_SYMBOL_GPL(…); static int exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, unsigned int num_fences) { … } /** * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec context * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if * @num_fences is zero drm_exec_lock_obj() is called instead. * * Using this function directly, it is the drivers responsibility to call * drm_exec_init() and drm_exec_fini() accordingly. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences) { … } EXPORT_SYMBOL_GPL(…); static int __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences) { … } static int drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences) { … } /** * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec locking context * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj() * is called instead. * * Using this function directly, it is the drivers responsibility to call * drm_exec_init() and drm_exec_fini() accordingly. * * Note: This function is safe against concurrent insertion and removal of * external objects, however it is not safe against concurrent usage itself. * * Drivers need to make sure to protect this case with either an outer VM lock * or by calling drm_gpuvm_prepare_vm() before this function within the * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures * mutual exclusion. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec locking context * @addr: the start address within the VA space * @range: the range to iterate within the VA space * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called * instead. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, u64 addr, u64 range, unsigned int num_fences) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs * @vm_exec: the &drm_gpuvm_exec wrapper * * Acquires all dma-resv locks of all &drm_gem_objects the given * &drm_gpuvm contains mappings of. * * Addionally, when calling this function with struct drm_gpuvm_exec::extra * being set the driver receives the given @fn callback to lock additional * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers * would call drm_exec_prepare_obj() from within this callback. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec) { … } EXPORT_SYMBOL_GPL(…); static int fn_lock_array(struct drm_gpuvm_exec *vm_exec) { … } /** * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs * @vm_exec: the &drm_gpuvm_exec wrapper * @objs: additional &drm_gem_objects to lock * @num_objs: the number of additional &drm_gem_objects to lock * * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm * contains mappings of, plus the ones given through @objs. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, struct drm_gem_object **objs, unsigned int num_objs) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range * @vm_exec: the &drm_gpuvm_exec wrapper * @addr: the start address within the VA space * @range: the range to iterate within the VA space * * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and * @addr + @range. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); static int __drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) { … } static int drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec) { … } /** * drm_gpuvm_validate() - validate all BOs marked as evicted * @gpuvm: the &drm_gpuvm to validate evicted BOs * @exec: the &drm_exec instance used for locking the GPUVM * * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer * objects being mapped in the given &drm_gpuvm. * * Returns: 0 on success, negative error code on failure. */ int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_resv_add_fence - add fence to private and all extobj * dma-resv * @gpuvm: the &drm_gpuvm to add a fence to * @exec: the &drm_exec locking context * @fence: fence to add * @private_usage: private dma-resv usage * @extobj_usage: extobj dma-resv usage */ void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, struct drm_exec *exec, struct dma_fence *fence, enum dma_resv_usage private_usage, enum dma_resv_usage extobj_usage) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * If provided by the driver, this function uses the &drm_gpuvm_ops * vm_bo_alloc() callback to allocate. * * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure */ struct drm_gpuvm_bo * drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) { … } EXPORT_SYMBOL_GPL(…); static void drm_gpuvm_bo_destroy(struct kref *kref) { … } /** * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference * @vm_bo: the &drm_gpuvm_bo to release the reference of * * This releases a reference to @vm_bo. * * If the reference count drops to zero, the &gpuvm_bo is destroyed, which * includes removing it from the GEMs gpuva list. Hence, if a call to this * function can potentially let the reference count drop to zero the caller must * hold the dma-resv or driver specific GEM gpuva lock. * * This function may only be called from non-atomic context. * * Returns: true if vm_bo was destroyed, false otherwise. */ bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo) { … } EXPORT_SYMBOL_GPL(…); static struct drm_gpuvm_bo * __drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) { … } /** * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given * &drm_gpuvm and &drm_gem_object * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the &drm_gpuvm_bo accordingly. * * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure */ struct drm_gpuvm_bo * drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the * given &drm_gpuvm and &drm_gem_object * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new * &drm_gpuvm_bo. * * A new &drm_gpuvm_bo is added to the GEMs gpuva list. * * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure */ struct drm_gpuvm_bo * drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo * for the given &drm_gpuvm and &drm_gem_object * @__vm_bo: A pre-allocated struct drm_gpuvm_bo. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference * count is decreased. If not found @__vm_bo is returned without further * increase of the reference count. * * A new &drm_gpuvm_bo is added to the GEMs gpuva list. * * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing * &drm_gpuvm_bo was found */ struct drm_gpuvm_bo * drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's * extobj list * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list. * * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list * already and if the corresponding &drm_gem_object is an external object, * actually. */ void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms * evicted list * @vm_bo: the &drm_gpuvm_bo to add or remove * @evict: indicates whether the object is evicted * * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list. */ void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict) { … } EXPORT_SYMBOL_GPL(…); static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va) { … } /** * drm_gpuva_insert() - insert a &drm_gpuva * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in * @va: the &drm_gpuva to insert * * Insert a &drm_gpuva with a given address and range into a * &drm_gpuvm. * * It is safe to use this function using the safe versions of iterating the GPU * VA space, such as drm_gpuvm_for_each_va_safe() and * drm_gpuvm_for_each_va_range_safe(). * * Returns: 0 on success, negative error code on failure. */ int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va) { … } EXPORT_SYMBOL_GPL(…); static void __drm_gpuva_remove(struct drm_gpuva *va) { … } /** * drm_gpuva_remove() - remove a &drm_gpuva * @va: the &drm_gpuva to remove * * This removes the given &va from the underlaying tree. * * It is safe to use this function using the safe versions of iterating the GPU * VA space, such as drm_gpuvm_for_each_va_safe() and * drm_gpuvm_for_each_va_range_safe(). */ void drm_gpuva_remove(struct drm_gpuva *va) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_link() - link a &drm_gpuva * @va: the &drm_gpuva to link * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to * * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the * &drm_gpuvm_bo to the &drm_gem_object it is associated with. * * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional * reference of the latter is taken. * * This function expects the caller to protect the GEM's GPUVA list against * concurrent access using either the GEMs dma_resv lock or a driver specific * lock set through drm_gem_gpuva_set_lock(). */ void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_unlink() - unlink a &drm_gpuva * @va: the &drm_gpuva to unlink * * This removes the given &va from the GPU VA list of the &drm_gem_object it is * associated with. * * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo. * * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of * the latter is dropped. * * This function expects the caller to protect the GEM's GPUVA list against * concurrent access using either the GEMs dma_resv lock or a driver specific * lock set through drm_gem_gpuva_set_lock(). */ void drm_gpuva_unlink(struct drm_gpuva *va) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_find_first() - find the first &drm_gpuva in the given range * @gpuvm: the &drm_gpuvm to search in * @addr: the &drm_gpuvas address * @range: the &drm_gpuvas range * * Returns: the first &drm_gpuva within the given range */ struct drm_gpuva * drm_gpuva_find_first(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_find() - find a &drm_gpuva * @gpuvm: the &drm_gpuvm to search in * @addr: the &drm_gpuvas address * @range: the &drm_gpuvas range * * Returns: the &drm_gpuva at a given &addr and with a given &range */ struct drm_gpuva * drm_gpuva_find(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_find_prev() - find the &drm_gpuva before the given address * @gpuvm: the &drm_gpuvm to search in * @start: the given GPU VA's start address * * Find the adjacent &drm_gpuva before the GPU VA with given &start address. * * Note that if there is any free space between the GPU VA mappings no mapping * is returned. * * Returns: a pointer to the found &drm_gpuva or NULL if none was found */ struct drm_gpuva * drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_find_next() - find the &drm_gpuva after the given address * @gpuvm: the &drm_gpuvm to search in * @end: the given GPU VA's end address * * Find the adjacent &drm_gpuva after the GPU VA with given &end address. * * Note that if there is any free space between the GPU VA mappings no mapping * is returned. * * Returns: a pointer to the found &drm_gpuva or NULL if none was found */ struct drm_gpuva * drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space * is empty * @gpuvm: the &drm_gpuvm to check the range for * @addr: the start address of the range * @range: the range of the interval * * Returns: true if the interval is empty, false otherwise */ bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_map() - helper to insert a &drm_gpuva according to a * &drm_gpuva_op_map * @gpuvm: the &drm_gpuvm * @va: the &drm_gpuva to insert * @op: the &drm_gpuva_op_map to initialize @va with * * Initializes the @va from the @op and inserts it into the given @gpuvm. */ void drm_gpuva_map(struct drm_gpuvm *gpuvm, struct drm_gpuva *va, struct drm_gpuva_op_map *op) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a * &drm_gpuva_op_remap * @prev: the &drm_gpuva to remap when keeping the start of a mapping * @next: the &drm_gpuva to remap when keeping the end of a mapping * @op: the &drm_gpuva_op_remap to initialize @prev and @next with * * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or * @next. */ void drm_gpuva_remap(struct drm_gpuva *prev, struct drm_gpuva *next, struct drm_gpuva_op_remap *op) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a * &drm_gpuva_op_unmap * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove * * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap. */ void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op) { … } EXPORT_SYMBOL_GPL(…); static int op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, u64 addr, u64 range, struct drm_gem_object *obj, u64 offset) { … } static int op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv, struct drm_gpuva_op_map *prev, struct drm_gpuva_op_map *next, struct drm_gpuva_op_unmap *unmap) { … } static int op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, struct drm_gpuva *va, bool merge) { … } static int __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, u64 req_addr, u64 req_range, struct drm_gem_object *req_obj, u64 req_offset) { … } static int __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, u64 req_addr, u64 req_range) { … } /** * drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req_addr: the start address of the new mapping * @req_range: the range of the new mapping * @req_obj: the &drm_gem_object to map * @req_offset: the offset within the &drm_gem_object * @priv: pointer to a driver private data structure * * This function iterates the given range of the GPU VA space. It utilizes the * &drm_gpuvm_ops to call back into the driver providing the split and merge * steps. * * Drivers may use these callbacks to update the GPU VA space right away within * the callback. In case the driver decides to copy and store the operations for * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to * be called before the &drm_gpuvm's view of the GPU VA space was * updated with the previous set of operations. To update the * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * A sequence of callbacks can contain map, unmap and remap operations, but * the sequence of callbacks might also be empty if no operation is required, * e.g. if the requested mapping already exists in the exact same way. * * There can be an arbitrary amount of unmap operations, a maximum of two remap * operations and a single map operation. The latter one represents the original * map operation requested by the caller. * * Returns: 0 on success or a negative error code */ int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, u64 req_addr, u64 req_range, struct drm_gem_object *req_obj, u64 req_offset) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap * @gpuvm: the &drm_gpuvm representing the GPU VA space * @priv: pointer to a driver private data structure * @req_addr: the start address of the range to unmap * @req_range: the range of the mappings to unmap * * This function iterates the given range of the GPU VA space. It utilizes the * &drm_gpuvm_ops to call back into the driver providing the operations to * unmap and, if required, split existent mappings. * * Drivers may use these callbacks to update the GPU VA space right away within * the callback. In case the driver decides to copy and store the operations for * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be * called before the &drm_gpuvm's view of the GPU VA space was updated * with the previous set of operations. To update the &drm_gpuvm's view * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or * drm_gpuva_destroy_unlocked() should be used. * * A sequence of callbacks can contain unmap and remap operations, depending on * whether there are actual overlapping mappings to split. * * There can be an arbitrary amount of unmap operations and a maximum of two * remap operations. * * Returns: 0 on success or a negative error code */ int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 req_addr, u64 req_range) { … } EXPORT_SYMBOL_GPL(…); static struct drm_gpuva_op * gpuva_op_alloc(struct drm_gpuvm *gpuvm) { … } static void gpuva_op_free(struct drm_gpuvm *gpuvm, struct drm_gpuva_op *op) { … } static int drm_gpuva_sm_step(struct drm_gpuva_op *__op, void *priv) { … } static const struct drm_gpuvm_ops gpuvm_list_ops = …; /** * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req_addr: the start address of the new mapping * @req_range: the range of the new mapping * @req_obj: the &drm_gem_object to map * @req_offset: the offset within the &drm_gem_object * * This function creates a list of operations to perform splitting and merging * of existent mapping(s) with the newly requested one. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain map, unmap and remap operations, but it * also can be empty if no operation is required, e.g. if the requested mapping * already exists is the exact same way. * * There can be an arbitrary amount of unmap operations, a maximum of two remap * operations and a single map operation. The latter one represents the original * map operation requested by the caller. * * Note that before calling this function again with another mapping request it * is necessary to update the &drm_gpuvm's view of the GPU VA space. The * previously obtained operations must be either processed or abandoned. To * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */ struct drm_gpuva_ops * drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, u64 req_addr, u64 req_range, struct drm_gem_object *req_obj, u64 req_offset) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on * unmap * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req_addr: the start address of the range to unmap * @req_range: the range of the mappings to unmap * * This function creates a list of operations to perform unmapping and, if * required, splitting of the mappings overlapping the unmap range. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain unmap and remap operations, depending on * whether there are actual overlapping mappings to split. * * There can be an arbitrary amount of unmap operations and a maximum of two * remap operations. * * Note that before calling this function again with another range to unmap it * is necessary to update the &drm_gpuvm's view of the GPU VA space. The * previously obtained operations must be processed or abandoned. To update the * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */ struct drm_gpuva_ops * drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, u64 req_addr, u64 req_range) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch * @gpuvm: the &drm_gpuvm representing the GPU VA space * @addr: the start address of the range to prefetch * @range: the range of the mappings to prefetch * * This function creates a list of operations to perform prefetching. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain prefetch operations. * * There can be an arbitrary amount of prefetch operations. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */ struct drm_gpuva_ops * drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, u64 addr, u64 range) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM * @vm_bo: the &drm_gpuvm_bo abstraction * * This function creates a list of operations to perform unmapping for every * GPUVA attached to a GEM. * * The list can be iterated with &drm_gpuva_for_each_op and consists out of an * arbitrary amount of unmap operations. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * It is the callers responsibility to protect the GEMs GPUVA list against * concurrent access using the GEMs dma_resv lock. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */ struct drm_gpuva_ops * drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo) { … } EXPORT_SYMBOL_GPL(…); /** * drm_gpuva_ops_free() - free the given &drm_gpuva_ops * @gpuvm: the &drm_gpuvm the ops were created for * @ops: the &drm_gpuva_ops to free * * Frees the given &drm_gpuva_ops structure including all the ops associated * with it. */ void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, struct drm_gpuva_ops *ops) { … } EXPORT_SYMBOL_GPL(…); MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux