i915_gem_ttm_move.c | Explore in Territory

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include <drm/ttm/ttm_tt.h>

#include "i915_deps.h"
#include "i915_drv.h"
#include "intel_memory_region.h"
#include "intel_region_ttm.h"

#include "gem/i915_gem_object.h"
#include "gem/i915_gem_region.h"
#include "gem/i915_gem_ttm.h"
#include "gem/i915_gem_ttm_move.h"

#include "gt/intel_engine_pm.h"
#include "gt/intel_gt.h"
#include "gt/intel_migrate.h"

/**
 * DOC: Selftest failure modes for failsafe migration:
 *
 * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
 * rather than a copy blit, and then we force the failure paths as if
 * the blit fence returned an error.
 *
 * For fail_work_allocation we fail the kmalloc of the async worker, we
 * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
 * true, then a memcpy operation is performed sync.
 */
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
static bool fail_gpu_migration;
static bool fail_work_allocation;
static bool ban_memcpy;

void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
					bool work_allocation)
{ … }

void i915_ttm_migrate_set_ban_memcpy(bool ban)
{ … }
#endif

static enum i915_cache_level
i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
		     struct ttm_tt *ttm)
{ … }

static struct intel_memory_region *
i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
{ … }

/**
 * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
 * TTM move
 * @obj: The gem object
 */
void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
{ … }

/**
 * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
 * @obj: The gem object
 *
 * Adjusts the GEM object's region, mem_flags and cache coherency after a
 * TTM move.
 */
void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
{ … }

/**
 * i915_ttm_move_notify - Prepare an object for move
 * @bo: The ttm buffer object.
 *
 * This function prepares an object for move by removing all GPU bindings,
 * removing all CPU mappings and finally releasing the pages sg-table.
 *
 * Return: 0 if successful, negative error code on error.
 */
int i915_ttm_move_notify(struct ttm_buffer_object *bo)
{ … }

static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
					     bool clear,
					     struct ttm_resource *dst_mem,
					     struct ttm_tt *dst_ttm,
					     struct sg_table *dst_st,
					     const struct i915_deps *deps)
{ … }

/**
 * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
 * @_dst_iter: Storage space for the destination kmap iterator.
 * @_src_iter: Storage space for the source kmap iterator.
 * @dst_iter: Pointer to the destination kmap iterator.
 * @src_iter: Pointer to the source kmap iterator.
 * @num_pages: Number of pages
 * @clear: Whether to clear instead of copy.
 * @src_rsgt: Refcounted scatter-gather list of source memory.
 * @dst_rsgt: Refcounted scatter-gather list of destination memory.
 */
struct i915_ttm_memcpy_arg { … };

/**
 * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
 * @fence: The dma-fence.
 * @work: The work struct use for the memcpy work.
 * @lock: The fence lock. Not used to protect anything else ATM.
 * @irq_work: Low latency worker to signal the fence since it can't be done
 * from the callback for lockdep reasons.
 * @cb: Callback for the accelerated migration fence.
 * @arg: The argument for the memcpy functionality.
 * @i915: The i915 pointer.
 * @obj: The GEM object.
 * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy
 * or memset, we wedge the device and set the @obj unknown_state, to prevent
 * further access to the object with the CPU or GPU.  On some devices we might
 * only be permitted to use the blitter engine for such operations.
 */
struct i915_ttm_memcpy_work { … };

static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
{ … }

static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
				 struct ttm_buffer_object *bo, bool clear,
				 struct ttm_resource *dst_mem,
				 struct ttm_tt *dst_ttm,
				 struct i915_refct_sgt *dst_rsgt)
{ … }

static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
{ … }

static void __memcpy_work(struct work_struct *work)
{ … }

static void __memcpy_irq_work(struct irq_work *irq_work)
{ … }

static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
{ … }

static const char *get_driver_name(struct dma_fence *fence)
{ … }

static const char *get_timeline_name(struct dma_fence *fence)
{ … }

static const struct dma_fence_ops dma_fence_memcpy_ops = …;

static struct dma_fence *
i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
			 struct dma_fence *dep)
{ … }

static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo,
				    struct ttm_resource *dst_mem)
{ … }

static struct dma_fence *
__i915_ttm_move(struct ttm_buffer_object *bo,
		const struct ttm_operation_ctx *ctx, bool clear,
		struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
		struct i915_refct_sgt *dst_rsgt, bool allow_accel,
		const struct i915_deps *move_deps)
{ … }

/**
 * i915_ttm_move - The TTM move callback used by i915.
 * @bo: The buffer object.
 * @evict: Whether this is an eviction.
 * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits should be
 *       performed if waiting
 * @dst_mem: The destination ttm resource.
 * @hop: If we need multihop, what temporary memory type to move to.
 *
 * Return: 0 if successful, negative error code otherwise.
 */
int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
		  struct ttm_operation_ctx *ctx,
		  struct ttm_resource *dst_mem,
		  struct ttm_place *hop)
{ … }

/**
 * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
 * another
 * @dst: The destination object
 * @src: The source object
 * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
 * @intr: Whether to perform waits interruptible:
 *
 * Note: The caller is responsible for assuring that the underlying
 * TTM objects are populated if needed and locked.
 *
 * Return: Zero on success. Negative error code on error. If @intr == true,
 * then it may return -ERESTARTSYS or -EINTR.
 */
int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
			  struct drm_i915_gem_object *src,
			  bool allow_accel, bool intr)
{ … }
linux/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
