/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * */ /* * Authors: * Jerome Glisse <[email protected]> * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> * Dave Airlie */ #include <linux/dma-mapping.h> #include <linux/iommu.h> #include <linux/pagemap.h> #include <linux/sched/task.h> #include <linux/sched/mm.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/swap.h> #include <linux/dma-buf.h> #include <linux/sizes.h> #include <linux/module.h> #include <drm/drm_drv.h> #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_range_manager.h> #include <drm/ttm/ttm_tt.h> #include <drm/amdgpu_drm.h> #include "amdgpu.h" #include "amdgpu_object.h" #include "amdgpu_trace.h" #include "amdgpu_amdkfd.h" #include "amdgpu_sdma.h" #include "amdgpu_ras.h" #include "amdgpu_hmm.h" #include "amdgpu_atomfirmware.h" #include "amdgpu_res_cursor.h" #include "bif/bif_4_1_d.h" MODULE_IMPORT_NS(…); #define AMDGPU_TTM_VRAM_MAX_DW_READ … static int amdgpu_ttm_backend_bind(struct ttm_device *bdev, struct ttm_tt *ttm, struct ttm_resource *bo_mem); static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev, unsigned int type, uint64_t size_in_page) { … } /** * amdgpu_evict_flags - Compute placement flags * * @bo: The buffer object to evict * @placement: Possible destination(s) for evicted BO * * Fill in placement data when ttm_bo_evict() is called */ static void amdgpu_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement) { … } /** * amdgpu_ttm_map_buffer - Map memory into the GART windows * @bo: buffer object to map * @mem: memory object to map * @mm_cur: range to map * @window: which GART window to use * @ring: DMA ring to use for the copy * @tmz: if we should setup a TMZ enabled mapping * @size: in number of bytes to map, out number of bytes mapped * @addr: resulting address inside the MC address space * * Setup one of the GART windows to access a specific piece of memory or return * the physical address for local memory. */ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo, struct ttm_resource *mem, struct amdgpu_res_cursor *mm_cur, unsigned int window, struct amdgpu_ring *ring, bool tmz, uint64_t *size, uint64_t *addr) { … } /** * amdgpu_ttm_copy_mem_to_mem - Helper function for copy * @adev: amdgpu device * @src: buffer/address where to read from * @dst: buffer/address where to write to * @size: number of bytes to copy * @tmz: if a secure copy should be used * @resv: resv object to sync to * @f: Returns the last fence if multiple jobs are submitted. * * The function copies @size bytes from {src->mem + src->offset} to * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a * move and different for a BO to BO copy. * */ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, const struct amdgpu_copy_mem *src, const struct amdgpu_copy_mem *dst, uint64_t size, bool tmz, struct dma_resv *resv, struct dma_fence **f) { … } /* * amdgpu_move_blit - Copy an entire buffer to another buffer * * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to * help move buffers to and from VRAM. */ static int amdgpu_move_blit(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *new_mem, struct ttm_resource *old_mem) { … } /** * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU * @adev: amdgpu device * @res: the resource to check * * Returns: true if the full resource is CPU visible, false otherwise. */ bool amdgpu_res_cpu_visible(struct amdgpu_device *adev, struct ttm_resource *res) { … } /* * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy * * Called by amdgpu_bo_move() */ static bool amdgpu_res_copyable(struct amdgpu_device *adev, struct ttm_resource *mem) { … } /* * amdgpu_bo_move - Move a buffer object to a new memory location * * Called by ttm_bo_handle_move_mem() */ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, struct ttm_operation_ctx *ctx, struct ttm_resource *new_mem, struct ttm_place *hop) { … } /* * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault * * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault() */ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) { … } static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo, unsigned long page_offset) { … } /** * amdgpu_ttm_domain_start - Returns GPU start address * @adev: amdgpu device object * @type: type of the memory * * Returns: * GPU start address of a memory domain */ uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type) { … } /* * TTM backend functions. */ struct amdgpu_ttm_tt { … }; #define ttm_to_amdgpu_ttm_tt(ptr) … #ifdef CONFIG_DRM_AMDGPU_USERPTR /* * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user * memory and start HMM tracking CPU page table update * * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only * once afterwards to stop HMM tracking */ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages, struct hmm_range **range) { … } /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations */ void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm, struct hmm_range *range) { … } /* * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change * Check if the pages backing this ttm range have been invalidated * * Returns: true if pages are still valid */ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm, struct hmm_range *range) { … } #endif /* * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary. * * Called by amdgpu_cs_list_validate(). This creates the page list * that backs user memory and will ultimately be mapped into the device * address space. */ void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages) { … } /* * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages * * Called by amdgpu_ttm_backend_bind() **/ static int amdgpu_ttm_tt_pin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) { … } /* * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages */ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) { … } /* * total_pages is constructed as MQD0+CtrlStack0 + MQD1+CtrlStack1 + ... * MQDn+CtrlStackn where n is the number of XCCs per partition. * pages_per_xcc is the size of one MQD+CtrlStack. The first page is MQD * and uses memory type default, UC. The rest of pages_per_xcc are * Ctrl stack and modify their memory type to NC. */ static void amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device *adev, struct ttm_tt *ttm, uint64_t flags) { … } static void amdgpu_ttm_gart_bind(struct amdgpu_device *adev, struct ttm_buffer_object *tbo, uint64_t flags) { … } /* * amdgpu_ttm_backend_bind - Bind GTT memory * * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem(). * This handles binding GTT memory to the device address space. */ static int amdgpu_ttm_backend_bind(struct ttm_device *bdev, struct ttm_tt *ttm, struct ttm_resource *bo_mem) { … } /* * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either * through AGP or GART aperture. * * If bo is accessible through AGP aperture, then use AGP aperture * to access bo; otherwise allocate logical space in GART aperture * and map bo to GART aperture. */ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo) { … } /* * amdgpu_ttm_recover_gart - Rebind GTT pages * * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to * rebind GTT pages during a GPU reset. */ void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo) { … } /* * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages * * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and * ttm_tt_destroy(). */ static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) { … } static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) { … } /** * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO * * @bo: The buffer object to create a GTT ttm_tt object around * @page_flags: Page flags to be added to the ttm_tt object * * Called by ttm_tt_create(). */ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) { … } /* * amdgpu_ttm_tt_populate - Map GTT pages visible to the device * * Map the pages of a ttm_tt object to an address space visible * to the underlying device. */ static int amdgpu_ttm_tt_populate(struct ttm_device *bdev, struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { … } /* * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays * * Unmaps pages of a ttm_tt object from the device address space and * unpopulates the page array backing it. */ static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) { … } /** * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current * task * * @tbo: The ttm_buffer_object that contains the userptr * @user_addr: The returned value */ int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo, uint64_t *user_addr) { … } /** * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current * task * * @bo: The ttm_buffer_object to bind this userptr to * @addr: The address in the current tasks VM space to use * @flags: Requirements of userptr object. * * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to * initialize GPU VM for a KFD process. */ int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo, uint64_t addr, uint32_t flags) { … } /* * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object */ struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) { … } /* * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an * address range for the current task. * */ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, unsigned long end, unsigned long *userptr) { … } /* * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr? */ bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) { … } /* * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only? */ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) { … } /** * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object * * @ttm: The ttm_tt object to compute the flags for * @mem: The memory registry backing this ttm_tt object * * Figure out the flags to use for a VM PDE (Page Directory Entry). */ uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) { … } /** * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object * * @adev: amdgpu_device pointer * @ttm: The ttm_tt object to compute the flags for * @mem: The memory registry backing this ttm_tt object * * Figure out the flags to use for a VM PTE (Page Table Entry). */ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, struct ttm_resource *mem) { … } /* * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer * object. * * Return true if eviction is sensible. Called by ttm_mem_evict_first() on * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until * it can find space for a new object and by ttm_bo_force_list_clean() which is * used to clean out a memory space. */ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { … } static void amdgpu_ttm_vram_mm_access(struct amdgpu_device *adev, loff_t pos, void *buf, size_t size, bool write) { … } static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo, unsigned long offset, void *buf, int len, int write) { … } /** * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object. * * @bo: The buffer object to read/write * @offset: Offset into buffer object * @buf: Secondary buffer to write/read from * @len: Length in bytes of access * @write: true if writing * * This is used to access VRAM that backs a buffer object via MMIO * access for debugging purposes. */ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo, unsigned long offset, void *buf, int len, int write) { … } static void amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo) { … } static struct ttm_device_funcs amdgpu_bo_driver = …; /* * Firmware Reservation functions */ /** * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram * * @adev: amdgpu_device pointer * * free fw reserved vram if it has been reserved. */ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) { … } /* * Driver Reservation functions */ /** * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram * * @adev: amdgpu_device pointer * * free drv reserved vram if it has been reserved. */ static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev) { … } /** * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw * * @adev: amdgpu_device pointer * * create bo vram reservation from fw. */ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) { … } /** * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver * * @adev: amdgpu_device pointer * * create bo vram reservation from drv. */ static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev) { … } /* * Memoy training reservation functions */ /** * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram * * @adev: amdgpu_device pointer * * free memory training reserved vram if it has been reserved. */ static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev) { … } static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev, uint32_t reserve_size) { … } /* * reserve TMR memory at the top of VRAM which holds * IP Discovery data and is protected by PSP. */ static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) { … } static int amdgpu_ttm_pools_init(struct amdgpu_device *adev) { … } static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev) { … } /* * amdgpu_ttm_init - Init the memory management (ttm) as well as various * gtt/vram related fields. * * This initializes all of the memory space pools that the TTM layer * will need such as the GTT space (system memory mapped to the device), * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which * can be mapped per VMID. */ int amdgpu_ttm_init(struct amdgpu_device *adev) { … } /* * amdgpu_ttm_fini - De-initialize the TTM memory pools */ void amdgpu_ttm_fini(struct amdgpu_device *adev) { … } /** * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions * * @adev: amdgpu_device pointer * @enable: true when we can use buffer functions. * * Enable/disable use of buffer functions during suspend/resume. This should * only be called at bootup or when userspace isn't running. */ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) { … } static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev, bool direct_submit, unsigned int num_dw, struct dma_resv *resv, bool vm_needs_flush, struct amdgpu_job **job, bool delayed) { … } int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count, struct dma_resv *resv, struct dma_fence **fence, bool direct_submit, bool vm_needs_flush, uint32_t copy_flags) { … } static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data, uint64_t dst_addr, uint32_t byte_count, struct dma_resv *resv, struct dma_fence **fence, bool vm_needs_flush, bool delayed) { … } /** * amdgpu_ttm_clear_buffer - clear memory buffers * @bo: amdgpu buffer object * @resv: reservation object * @fence: dma_fence associated with the operation * * Clear the memory buffer resource. * * Returns: * 0 for success or a negative error code on failure. */ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo, struct dma_resv *resv, struct dma_fence **fence) { … } int amdgpu_fill_buffer(struct amdgpu_bo *bo, uint32_t src_data, struct dma_resv *resv, struct dma_fence **f, bool delayed) { … } /** * amdgpu_ttm_evict_resources - evict memory buffers * @adev: amdgpu device object * @mem_type: evicted BO's memory type * * Evicts all @mem_type buffers on the lru list of the memory type. * * Returns: * 0 for success or a negative error code on failure. */ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type) { … } #if defined(CONFIG_DEBUG_FS) static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused) { … } DEFINE_SHOW_ATTRIBUTE(…); /* * amdgpu_ttm_vram_read - Linear read access to VRAM * * Accesses VRAM via MMIO for debugging purposes. */ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { … } /* * amdgpu_ttm_vram_write - Linear write access to VRAM * * Accesses VRAM via MMIO for debugging purposes. */ static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) { … } static const struct file_operations amdgpu_ttm_vram_fops = …; /* * amdgpu_iomem_read - Virtual read access to GPU mapped memory * * This function is used to read memory that has been mapped to the * GPU and the known addresses are not physical addresses but instead * bus addresses (e.g., what you'd put in an IB or ring buffer). */ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { … } /* * amdgpu_iomem_write - Virtual write access to GPU mapped memory * * This function is used to write memory that has been mapped to the * GPU and the known addresses are not physical addresses but instead * bus addresses (e.g., what you'd put in an IB or ring buffer). */ static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) { … } static const struct file_operations amdgpu_ttm_iomem_fops = …; #endif void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) { … }
Codebase Browser
linux