/* * Copyright 2018 Advanced Micro Devices, Inc. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * */ #include <linux/io-64-nonatomic-lo-hi.h> #ifdef CONFIG_X86 #include <asm/hypervisor.h> #endif #include "amdgpu.h" #include "amdgpu_gmc.h" #include "amdgpu_ras.h" #include "amdgpu_reset.h" #include "amdgpu_xgmi.h" #include <drm/drm_drv.h> #include <drm/ttm/ttm_tt.h> /** * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 * * @adev: amdgpu_device pointer * * Allocate video memory for pdb0 and map it for CPU access * Returns 0 for success, error for failure. */ int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev) { … } /** * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO * * @bo: the BO to get the PDE for * @level: the level in the PD hirarchy * @addr: resulting addr * @flags: resulting flags * * Get the address and flags to be used for a PDE (Page Directory Entry). */ void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, uint64_t *addr, uint64_t *flags) { … } /* * amdgpu_gmc_pd_addr - return the address of the root directory */ uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) { … } /** * amdgpu_gmc_set_pte_pde - update the page tables using CPU * * @adev: amdgpu_device pointer * @cpu_pt_addr: cpu address of the page table * @gpu_page_idx: entry in the page table to update * @addr: dst addr to write into pte/pde * @flags: access flags * * Update the page tables using CPU. */ int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, uint32_t gpu_page_idx, uint64_t addr, uint64_t flags) { … } /** * amdgpu_gmc_agp_addr - return the address in the AGP address space * * @bo: TTM BO which needs the address, must be in GTT domain * * Tries to figure out how to access the BO through the AGP aperture. Returns * AMDGPU_BO_INVALID_OFFSET if that is not possible. */ uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) { … } /** * amdgpu_gmc_vram_location - try to find VRAM location * * @adev: amdgpu device structure holding all necessary information * @mc: memory controller structure holding memory information * @base: base address at which to put VRAM * * Function will try to place VRAM at base address provided * as parameter. */ void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, u64 base) { … } /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture * * @adev: amdgpu device structure holding all necessary information * @mc: memory controller structure holding memory information * * This function is only used if use GART for FB translation. In such * case, we use sysvm aperture (vmid0 page tables) for both vram * and gart (aka system memory) access. * * GPUVM (and our organization of vmid0 page tables) require sysvm * aperture to be placed at a location aligned with 8 times of native * page size. For example, if vm_context0_cntl.page_table_block_size * is 12, then native page size is 8G (2M*2^12), sysvm should start * with a 64G aligned address. For simplicity, we just put sysvm at * address 0. So vram start at address 0 and gart is right after vram. */ void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) { … } /** * amdgpu_gmc_gart_location - try to find GART location * * @adev: amdgpu device structure holding all necessary information * @mc: memory controller structure holding memory information * @gart_placement: GART placement policy with respect to VRAM * * Function will place try to place GART before or after VRAM. * If GART size is bigger than space left then we ajust GART size. * Thus function will never fails. */ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, enum amdgpu_gart_placement gart_placement) { … } /** * amdgpu_gmc_agp_location - try to find AGP location * @adev: amdgpu device structure holding all necessary information * @mc: memory controller structure holding memory information * * Function will place try to find a place for the AGP BAR in the MC address * space. * * AGP BAR will be assigned the largest available hole in the address space. * Should be called after VRAM and GART locations are setup. */ void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) { … } /** * amdgpu_gmc_set_agp_default - Set the default AGP aperture value. * @adev: amdgpu device structure holding all necessary information * @mc: memory controller structure holding memory information * * To disable the AGP aperture, you need to set the start to a larger * value than the end. This function sets the default value which * can then be overridden using amdgpu_gmc_agp_location() if you want * to enable the AGP aperture on a specific chip. * */ void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev, struct amdgpu_gmc *mc) { … } /** * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid * * @addr: 48 bit physical address, page aligned (36 significant bits) * @pasid: 16 bit process address space identifier */ static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid) { … } /** * amdgpu_gmc_filter_faults - filter VM faults * * @adev: amdgpu device structure * @ih: interrupt ring that the fault received from * @addr: address of the VM fault * @pasid: PASID of the process causing the fault * @timestamp: timestamp of the fault * * Returns: * True if the fault was filtered and should not be processed further. * False if the fault is a new one and needs to be handled. */ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, uint64_t addr, uint16_t pasid, uint64_t timestamp) { … } /** * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter * * @adev: amdgpu device structure * @addr: address of the VM fault * @pasid: PASID of the process causing the fault * * Remove the address from fault filter, then future vm fault on this address * will pass to retry fault handler to recover. */ void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, uint16_t pasid) { … } int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev) { … } int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) { … } void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) { … } /* * The latest engine allocation on gfx9/10 is: * Engine 2, 3: firmware * Engine 0, 1, 4~16: amdgpu ring, * subject to change when ring number changes * Engine 17: Gart flushes */ #define AMDGPU_VMHUB_INV_ENG_BITMAP … int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) { … } void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, uint32_t vmhub, uint32_t flush_type) { … } int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, uint32_t flush_type, bool all_hub, uint32_t inst) { … } void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask, uint32_t xcc_inst) { … } /** * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ * @adev: amdgpu_device pointer * * Check and set if an the device @adev supports Trusted Memory * Zones (TMZ). */ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) { … } /** * amdgpu_gmc_noretry_set -- set per asic noretry defaults * @adev: amdgpu_device pointer * * Set a per asic default for the no-retry parameter. * */ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) { … } void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, bool enable) { … } void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev) { … } /** * amdgpu_gmc_init_pdb0 - initialize PDB0 * * @adev: amdgpu_device pointer * * This function is only used when GART page table is used * for FB address translatioin. In such a case, we construct * a 2-level system VM page table: PDB0->PTB, to cover both * VRAM of the hive and system memory. * * PDB0 is static, initialized once on driver initialization. * The first n entries of PDB0 are used as PTE by setting * P bit to 1, pointing to VRAM. The n+1'th entry points * to a big PTB covering system memory. * */ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev) { … } /** * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC * address * * @adev: amdgpu_device pointer * @mc_addr: MC address of buffer */ uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr) { … } /** * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from * GPU's view * * @adev: amdgpu_device pointer * @bo: amdgpu buffer object */ uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) { … } /** * amdgpu_gmc_vram_cpu_pa - calculate vram buffer object's physical address * from CPU's view * * @adev: amdgpu_device pointer * @bo: amdgpu buffer object */ uint64_t amdgpu_gmc_vram_cpu_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) { … } int amdgpu_gmc_vram_checking(struct amdgpu_device *adev) { … } static ssize_t current_memory_partition_show( struct device *dev, struct device_attribute *addr, char *buf) { … } static DEVICE_ATTR_RO(current_memory_partition); int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev) { … } void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev) { … } int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, struct amdgpu_mem_partition_info *mem_ranges, int exp_ranges) { … }
Codebase Browser
linux