/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ #define pr_fmt(fmt) … #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> #include <linux/dma-resv.h> #include "ttm_module.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) { … } /** * ttm_bo_move_to_lru_tail * * @bo: The buffer object. * * Move this BO to the tail of all lru lists used to lookup and reserve an * object. This function must be called with struct ttm_global::lru_lock * held, and is used to make a BO less likely to be considered for eviction. */ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { … } EXPORT_SYMBOL(…); /** * ttm_bo_set_bulk_move - update BOs bulk move object * * @bo: The buffer object. * @bulk: bulk move structure * * Update the BOs bulk move object, making sure that resources are added/removed * as well. A bulk move allows to move many resource on the LRU at once, * resulting in much less overhead of maintaining the LRU. * The only requirement is that the resources stay together on the LRU and are * never separated. This is enforces by setting the bulk_move structure on a BO. * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of * their LRU list. */ void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, struct ttm_lru_bulk_move *bulk) { … } EXPORT_SYMBOL(…); static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_resource *mem, bool evict, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { … } /* * Call bo::reserved. * Will release GPU memory type usage on destruction. * This is the place to put in driver specific hooks to release * driver private resources. * Will release the bo::reserved lock. */ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { … } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) { … } static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) { … } /** * ttm_bo_cleanup_refs * If bo idle, remove from lru lists, and unref. * If not idle, block if possible. * * Must be called with lru_lock and reservation held, this function * will drop the lru lock and optionally the reservation lock before returning. * * @bo: The buffer object to clean-up * @interruptible: Any sleeps should occur interruptibly. * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. * @unlock_resv: Unlock the reservation lock as well. */ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_gpu, bool unlock_resv) { … } /* * Block for the dma_resv object to become idle, lock the buffer and clean up * the resource and tt object. */ static void ttm_bo_delayed_delete(struct work_struct *work) { … } static void ttm_bo_release(struct kref *kref) { … } /** * ttm_bo_put * * @bo: The buffer object. * * Unreference a buffer object. */ void ttm_bo_put(struct ttm_buffer_object *bo) { … } EXPORT_SYMBOL(…); static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { … } static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { … } /** * ttm_bo_eviction_valuable * * @bo: The buffer object to evict * @place: the placement we need to make room for * * Check if it is valuable to evict the BO to make room for the given placement. */ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { … } EXPORT_SYMBOL(…); /* * Check the target bo is allowable to be evicted or swapout, including cases: * * a. if share same reservation object with ctx->resv, have assumption * reservation objects should already be locked, so not lock again and * return true directly when either the opreation allow_reserved_eviction * or the target bo already is in delayed free list; * * b. Otherwise, trylock it. */ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, const struct ttm_place *place, bool *locked, bool *busy) { … } /** * ttm_mem_evict_wait_busy - wait for a busy BO to become available * * @busy_bo: BO which couldn't be locked with trylock * @ctx: operation context * @ticket: acquire ticket * * Try to lock a busy buffer object to avoid failing eviction. */ static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, struct ttm_operation_ctx *ctx, struct ww_acquire_ctx *ticket) { … } int ttm_mem_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_operation_ctx *ctx, struct ww_acquire_ctx *ticket) { … } /** * ttm_bo_pin - Pin the buffer object. * @bo: The buffer object to pin * * Make sure the buffer is not evicted any more during memory pressure. * @bo must be unpinned again by calling ttm_bo_unpin(). */ void ttm_bo_pin(struct ttm_buffer_object *bo) { … } EXPORT_SYMBOL(…); /** * ttm_bo_unpin - Unpin the buffer object. * @bo: The buffer object to unpin * * Allows the buffer object to be evicted again during memory pressure. */ void ttm_bo_unpin(struct ttm_buffer_object *bo) { … } EXPORT_SYMBOL(…); /* * Add the last move fence to the BO as kernel dependency and reserve a new * fence slot. */ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, struct ttm_resource_manager *man, bool no_wait_gpu) { … } /** * ttm_bo_alloc_resource - Allocate backing store for a BO * * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for * @placement: Proposed new placement for the buffer object * @ctx: if and how to sleep, lock buffers and alloc memory * @force_space: If we should evict buffers to force space * @res: The resulting struct ttm_resource. * * Allocates a resource for the buffer object pointed to by @bo, using the * placement flags in @placement, potentially evicting other buffer objects when * @force_space is true. * This function may sleep while waiting for resources to become available. * Returns: * -EBUSY: No space available (only if no_wait == true). * -ENOSPC: Could not allocate space for the buffer object, either due to * fragmentation or concurrent allocators. * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. */ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx, bool force_space, struct ttm_resource **res) { … } /* * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource * * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for * @placement: Proposed new placement for the buffer object * @res: The resulting struct ttm_resource. * @ctx: if and how to sleep, lock buffers and alloc memory * * Tries both idle allocation and forcefully eviction of buffers. See * ttm_bo_alloc_resource for details. */ int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_resource **res, struct ttm_operation_ctx *ctx) { … } EXPORT_SYMBOL(…); /** * ttm_bo_validate * * @bo: The buffer object. * @placement: Proposed placement for the buffer object. * @ctx: validation parameters. * * Changes placement and caching policy of the buffer object * according proposed placement. * Returns * -EINVAL on invalid proposed placement. * -ENOMEM on out-of-memory condition. * -EBUSY if no_wait is true and buffer busy. * -ERESTARTSYS if interrupted by a signal. */ int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) { … } EXPORT_SYMBOL(…); /** * ttm_bo_init_reserved * * @bdev: Pointer to a ttm_device struct. * @bo: Pointer to a ttm_buffer_object to be initialized. * @type: Requested type of buffer object. * @placement: Initial placement for buffer object. * @alignment: Data alignment in pages. * @ctx: TTM operation context for memory allocation. * @sg: Scatter-gather table. * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. * @destroy: Destroy function. Use NULL for kfree(). * * This function initializes a pre-allocated struct ttm_buffer_object. * As this object may be part of a larger structure, this function, * together with the @destroy function, enables driver-specific objects * derived from a ttm_buffer_object. * * On successful return, the caller owns an object kref to @bo. The kref and * list_kref are usually set to 1, but note that in some situations, other * tasks may already be holding references to @bo as well. * Furthermore, if resv == NULL, the buffer's reservation lock will be held, * and it is the caller's responsibility to call ttm_bo_unreserve. * * If a failure occurs, the function will call the @destroy function. Thus, * after a failure, dereferencing @bo is illegal and will likely cause memory * corruption. * * Returns * -ENOMEM: Out of memory. * -EINVAL: Invalid placement flags. * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. */ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t alignment, struct ttm_operation_ctx *ctx, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { … } EXPORT_SYMBOL(…); /** * ttm_bo_init_validate * * @bdev: Pointer to a ttm_device struct. * @bo: Pointer to a ttm_buffer_object to be initialized. * @type: Requested type of buffer object. * @placement: Initial placement for buffer object. * @alignment: Data alignment in pages. * @interruptible: If needing to sleep to wait for GPU resources, * sleep interruptible. * pinned in physical memory. If this behaviour is not desired, this member * holds a pointer to a persistent shmem object. Typically, this would * point to the shmem object backing a GEM object if TTM is used to back a * GEM user interface. * @sg: Scatter-gather table. * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. * @destroy: Destroy function. Use NULL for kfree(). * * This function initializes a pre-allocated struct ttm_buffer_object. * As this object may be part of a larger structure, this function, * together with the @destroy function, * enables driver-specific objects derived from a ttm_buffer_object. * * On successful return, the caller owns an object kref to @bo. The kref and * list_kref are usually set to 1, but note that in some situations, other * tasks may already be holding references to @bo as well. * * If a failure occurs, the function will call the @destroy function, Thus, * after a failure, dereferencing @bo is illegal and will likely cause memory * corruption. * * Returns * -ENOMEM: Out of memory. * -EINVAL: Invalid placement flags. * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. */ int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t alignment, bool interruptible, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { … } EXPORT_SYMBOL(…); /* * buffer object vm functions. */ /** * ttm_bo_unmap_virtual * * @bo: tear down the virtual mappings for this BO */ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { … } EXPORT_SYMBOL(…); /** * ttm_bo_wait_ctx - wait for buffer idle. * * @bo: The buffer object. * @ctx: defines how to wait * * Waits for the buffer to be idle. Used timeout depends on the context. * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or * zero on success. */ int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { … } EXPORT_SYMBOL(…); int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, gfp_t gfp_flags) { … } void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) { … }
Codebase Browser
linux