/* * Copyright 2012 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Alex Deucher */ #include <linux/firmware.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/slab.h> #include <drm/drm_vblank.h> #include "atom.h" #include "evergreen.h" #include "cik_blit_shaders.h" #include "cik.h" #include "cikd.h" #include "clearstate_ci.h" #include "r600.h" #include "radeon.h" #include "radeon_asic.h" #include "radeon_audio.h" #include "radeon_ucode.h" #include "si.h" #include "vce.h" #define SH_MEM_CONFIG_GFX_DEFAULT … MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); static void cik_rlc_stop(struct radeon_device *rdev); static void cik_pcie_gen3_enable(struct radeon_device *rdev); static void cik_program_aspm(struct radeon_device *rdev); static void cik_init_pg(struct radeon_device *rdev); static void cik_init_cg(struct radeon_device *rdev); static void cik_fini_pg(struct radeon_device *rdev); static void cik_fini_cg(struct radeon_device *rdev); static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, bool enable); /** * cik_get_allowed_info_register - fetch the register for the info ioctl * * @rdev: radeon_device pointer * @reg: register offset in bytes * @val: register value * * Returns 0 for success or -EINVAL for an invalid register * */ int cik_get_allowed_info_register(struct radeon_device *rdev, u32 reg, u32 *val) { … } /* * Indirect registers accessor */ u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg) { … } void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v) { … } /* get temperature in millidegrees */ int ci_get_temp(struct radeon_device *rdev) { … } /* get temperature in millidegrees */ int kv_get_temp(struct radeon_device *rdev) { … } /* * Indirect registers accessor */ u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) { … } void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) { … } static const u32 spectre_rlc_save_restore_register_list[] = …; static const u32 kalindi_rlc_save_restore_register_list[] = …; static const u32 bonaire_golden_spm_registers[] = …; static const u32 bonaire_golden_common_registers[] = …; static const u32 bonaire_golden_registers[] = …; static const u32 bonaire_mgcg_cgcg_init[] = …; static const u32 spectre_golden_spm_registers[] = …; static const u32 spectre_golden_common_registers[] = …; static const u32 spectre_golden_registers[] = …; static const u32 spectre_mgcg_cgcg_init[] = …; static const u32 kalindi_golden_spm_registers[] = …; static const u32 kalindi_golden_common_registers[] = …; static const u32 kalindi_golden_registers[] = …; static const u32 kalindi_mgcg_cgcg_init[] = …; static const u32 hawaii_golden_spm_registers[] = …; static const u32 hawaii_golden_common_registers[] = …; static const u32 hawaii_golden_registers[] = …; static const u32 hawaii_mgcg_cgcg_init[] = …; static const u32 godavari_golden_registers[] = …; static void cik_init_golden_registers(struct radeon_device *rdev) { … } /** * cik_get_xclk - get the xclk * * @rdev: radeon_device pointer * * Returns the reference clock used by the gfx engine * (CIK). */ u32 cik_get_xclk(struct radeon_device *rdev) { … } /** * cik_mm_rdoorbell - read a doorbell dword * * @rdev: radeon_device pointer * @index: doorbell index * * Returns the value in the doorbell aperture at the * requested doorbell index (CIK). */ u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) { … } /** * cik_mm_wdoorbell - write a doorbell dword * * @rdev: radeon_device pointer * @index: doorbell index * @v: value to write * * Writes @v to the doorbell aperture at the * requested doorbell index (CIK). */ void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) { … } #define BONAIRE_IO_MC_REGS_SIZE … static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = …; #define HAWAII_IO_MC_REGS_SIZE … static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = …; /** * cik_srbm_select - select specific register instances * * @rdev: radeon_device pointer * @me: selected ME (micro engine) * @pipe: pipe * @queue: queue * @vmid: VMID * * Switches the currently active registers instances. Some * registers are instanced per VMID, others are instanced per * me/pipe/queue combination. */ static void cik_srbm_select(struct radeon_device *rdev, u32 me, u32 pipe, u32 queue, u32 vmid) { … } /* ucode loading */ /** * ci_mc_load_microcode - load MC ucode into the hw * * @rdev: radeon_device pointer * * Load the GDDR MC ucode into the hw (CIK). * Returns 0 on success, error on failure. */ int ci_mc_load_microcode(struct radeon_device *rdev) { … } /** * cik_init_microcode - load ucode images from disk * * @rdev: radeon_device pointer * * Use the firmware interface to load the ucode images into * the driver (not loaded into hw). * Returns 0 on success, error on failure. */ static int cik_init_microcode(struct radeon_device *rdev) { … } /* * Core functions */ /** * cik_tiling_mode_table_init - init the hw tiling table * * @rdev: radeon_device pointer * * Starting with SI, the tiling setup is done globally in a * set of 32 tiling modes. Rather than selecting each set of * parameters per surface as on older asics, we just select * which index in the tiling table we want to use, and the * surface uses those parameters (CIK). */ static void cik_tiling_mode_table_init(struct radeon_device *rdev) { … } /** * cik_select_se_sh - select which SE, SH to address * * @rdev: radeon_device pointer * @se_num: shader engine to address * @sh_num: sh block to address * * Select which SE, SH combinations to address. Certain * registers are instanced per SE or SH. 0xffffffff means * broadcast to all SEs or SHs (CIK). */ static void cik_select_se_sh(struct radeon_device *rdev, u32 se_num, u32 sh_num) { … } /** * cik_create_bitmask - create a bitmask * * @bit_width: length of the mask * * create a variable length bit mask (CIK). * Returns the bitmask. */ static u32 cik_create_bitmask(u32 bit_width) { … } /** * cik_get_rb_disabled - computes the mask of disabled RBs * * @rdev: radeon_device pointer * @max_rb_num_per_se: max RBs (render backends) per SE (shader engine) for the asic * @sh_per_se: number of SH blocks per SE for the asic * * Calculates the bitmask of disabled RBs (CIK). * Returns the disabled RB bitmask. */ static u32 cik_get_rb_disabled(struct radeon_device *rdev, u32 max_rb_num_per_se, u32 sh_per_se) { … } /** * cik_setup_rb - setup the RBs on the asic * * @rdev: radeon_device pointer * @se_num: number of SEs (shader engines) for the asic * @sh_per_se: number of SH blocks per SE for the asic * @max_rb_num_per_se: max RBs (render backends) per SE for the asic * * Configures per-SE/SH RB registers (CIK). */ static void cik_setup_rb(struct radeon_device *rdev, u32 se_num, u32 sh_per_se, u32 max_rb_num_per_se) { … } /** * cik_gpu_init - setup the 3D engine * * @rdev: radeon_device pointer * * Configures the 3D engine and tiling configuration * registers so that the 3D engine is usable. */ static void cik_gpu_init(struct radeon_device *rdev) { … } /* * GPU scratch registers helpers function. */ /** * cik_scratch_init - setup driver info for CP scratch regs * * @rdev: radeon_device pointer * * Set up the number and offset of the CP scratch registers. * NOTE: use of CP scratch registers is a legacy inferface and * is not used by default on newer asics (r6xx+). On newer asics, * memory buffers are used for fences rather than scratch regs. */ static void cik_scratch_init(struct radeon_device *rdev) { … } /** * cik_ring_test - basic gfx ring test * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information * * Allocate a scratch register and write to it using the gfx ring (CIK). * Provides a basic gfx ring test to verify that the ring is working. * Used by cik_cp_gfx_resume(); * Returns 0 on success, error on failure. */ int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) { … } /** * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp * * @rdev: radeon_device pointer * @ridx: radeon ring index * * Emits an hdp flush on the cp. */ static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, int ridx) { … } /** * cik_fence_gfx_ring_emit - emit a fence on the gfx ring * * @rdev: radeon_device pointer * @fence: radeon fence object * * Emits a fence sequnce number on the gfx ring and flushes * GPU caches. */ void cik_fence_gfx_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence) { … } /** * cik_fence_compute_ring_emit - emit a fence on the compute ring * * @rdev: radeon_device pointer * @fence: radeon fence object * * Emits a fence sequnce number on the compute ring and flushes * GPU caches. */ void cik_fence_compute_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence) { … } /** * cik_semaphore_ring_emit - emit a semaphore on the CP ring * * @rdev: radeon_device pointer * @ring: radeon ring buffer object * @semaphore: radeon semaphore object * @emit_wait: Is this a semaphore wait? * * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP * from running ahead of semaphore waits. */ bool cik_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *ring, struct radeon_semaphore *semaphore, bool emit_wait) { … } /** * cik_copy_cpdma - copy pages using the CP DMA engine * * @rdev: radeon_device pointer * @src_offset: src GPU address * @dst_offset: dst GPU address * @num_gpu_pages: number of GPU pages to xfer * @resv: reservation object to sync to * * Copy GPU paging using the CP DMA engine (CIK+). * Used by the radeon ttm implementation to move pages if * registered as the asic copy callback. */ struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, uint64_t src_offset, uint64_t dst_offset, unsigned num_gpu_pages, struct dma_resv *resv) { … } /* * IB stuff */ /** * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring * * @rdev: radeon_device pointer * @ib: radeon indirect buffer object * * Emits a DE (drawing engine) or CE (constant engine) IB * on the gfx ring. IBs are usually generated by userspace * acceleration drivers and submitted to the kernel for * scheduling on the ring. This function schedules the IB * on the gfx ring for execution by the GPU. */ void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { … } /** * cik_ib_test - basic gfx ring IB test * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information * * Allocate an IB and execute it on the gfx ring (CIK). * Provides a basic gfx ring test to verify that IBs are working. * Returns 0 on success, error on failure. */ int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) { … } /* * CP. * On CIK, gfx and compute now have independant command processors. * * GFX * Gfx consists of a single ring and can process both gfx jobs and * compute jobs. The gfx CP consists of three microengines (ME): * PFP - Pre-Fetch Parser * ME - Micro Engine * CE - Constant Engine * The PFP and ME make up what is considered the Drawing Engine (DE). * The CE is an asynchronous engine used for updating buffer desciptors * used by the DE so that they can be loaded into cache in parallel * while the DE is processing state update packets. * * Compute * The compute CP consists of two microengines (ME): * MEC1 - Compute MicroEngine 1 * MEC2 - Compute MicroEngine 2 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. * The queues are exposed to userspace and are programmed directly * by the compute runtime. */ /** * cik_cp_gfx_enable - enable/disable the gfx CP MEs * * @rdev: radeon_device pointer * @enable: enable or disable the MEs * * Halts or unhalts the gfx MEs. */ static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) { … } /** * cik_cp_gfx_load_microcode - load the gfx CP ME ucode * * @rdev: radeon_device pointer * * Loads the gfx PFP, ME, and CE ucode. * Returns 0 for success, -EINVAL if the ucode is not available. */ static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) { … } /** * cik_cp_gfx_start - start the gfx ring * * @rdev: radeon_device pointer * * Enables the ring and loads the clear state context and other * packets required to init the ring. * Returns 0 for success, error for failure. */ static int cik_cp_gfx_start(struct radeon_device *rdev) { … } /** * cik_cp_gfx_fini - stop the gfx ring * * @rdev: radeon_device pointer * * Stop the gfx ring and tear down the driver ring * info. */ static void cik_cp_gfx_fini(struct radeon_device *rdev) { … } /** * cik_cp_gfx_resume - setup the gfx ring buffer registers * * @rdev: radeon_device pointer * * Program the location and size of the gfx ring buffer * and test it to make sure it's working. * Returns 0 for success, error for failure. */ static int cik_cp_gfx_resume(struct radeon_device *rdev) { … } u32 cik_gfx_get_rptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } u32 cik_gfx_get_wptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } void cik_gfx_set_wptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } u32 cik_compute_get_rptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } u32 cik_compute_get_wptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } void cik_compute_set_wptr(struct radeon_device *rdev, struct radeon_ring *ring) { … } static void cik_compute_stop(struct radeon_device *rdev, struct radeon_ring *ring) { … } /** * cik_cp_compute_enable - enable/disable the compute CP MEs * * @rdev: radeon_device pointer * @enable: enable or disable the MEs * * Halts or unhalts the compute MEs. */ static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) { … } /** * cik_cp_compute_load_microcode - load the compute CP ME ucode * * @rdev: radeon_device pointer * * Loads the compute MEC1&2 ucode. * Returns 0 for success, -EINVAL if the ucode is not available. */ static int cik_cp_compute_load_microcode(struct radeon_device *rdev) { … } /** * cik_cp_compute_start - start the compute queues * * @rdev: radeon_device pointer * * Enable the compute queues. * Returns 0 for success, error for failure. */ static int cik_cp_compute_start(struct radeon_device *rdev) { … } /** * cik_cp_compute_fini - stop the compute queues * * @rdev: radeon_device pointer * * Stop the compute queues and tear down the driver queue * info. */ static void cik_cp_compute_fini(struct radeon_device *rdev) { … } static void cik_mec_fini(struct radeon_device *rdev) { … } #define MEC_HPD_SIZE … static int cik_mec_init(struct radeon_device *rdev) { … } struct hqd_registers { … }; struct bonaire_mqd { … }; /** * cik_cp_compute_resume - setup the compute queue registers * * @rdev: radeon_device pointer * * Program the compute queues and test them to make sure they * are working. * Returns 0 for success, error for failure. */ static int cik_cp_compute_resume(struct radeon_device *rdev) { … } static void cik_cp_enable(struct radeon_device *rdev, bool enable) { … } static int cik_cp_load_microcode(struct radeon_device *rdev) { … } static void cik_cp_fini(struct radeon_device *rdev) { … } static int cik_cp_resume(struct radeon_device *rdev) { … } static void cik_print_gpu_status_regs(struct radeon_device *rdev) { … } /** * cik_gpu_check_soft_reset - check which blocks are busy * * @rdev: radeon_device pointer * * Check which blocks are busy and return the relevant reset * mask to be used by cik_gpu_soft_reset(). * Returns a mask of the blocks to be reset. */ u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) { … } /** * cik_gpu_soft_reset - soft reset GPU * * @rdev: radeon_device pointer * @reset_mask: mask of which blocks to reset * * Soft reset the blocks specified in @reset_mask. */ static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) { … } struct kv_reset_save_regs { … }; static void kv_save_regs_for_reset(struct radeon_device *rdev, struct kv_reset_save_regs *save) { … } static void kv_restore_regs_for_reset(struct radeon_device *rdev, struct kv_reset_save_regs *save) { … } static void cik_gpu_pci_config_reset(struct radeon_device *rdev) { … } /** * cik_asic_reset - soft reset GPU * * @rdev: radeon_device pointer * @hard: force hard reset * * Look up which blocks are hung and attempt * to reset them. * Returns 0 for success. */ int cik_asic_reset(struct radeon_device *rdev, bool hard) { … } /** * cik_gfx_is_lockup - check if the 3D engine is locked up * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information * * Check if the 3D engine is locked up (CIK). * Returns true if the engine is locked, false if not. */ bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) { … } /* MC */ /** * cik_mc_program - program the GPU memory controller * * @rdev: radeon_device pointer * * Set the location of vram, gart, and AGP in the GPU's * physical address space (CIK). */ static void cik_mc_program(struct radeon_device *rdev) { … } /** * cik_mc_init - initialize the memory controller driver params * * @rdev: radeon_device pointer * * Look up the amount of vram, vram width, and decide how to place * vram and gart within the GPU's physical address space (CIK). * Returns 0 for success. */ static int cik_mc_init(struct radeon_device *rdev) { … } /* * GART * VMID 0 is the physical GPU addresses as used by the kernel. * VMIDs 1-15 are used for userspace clients and are handled * by the radeon vm/hsa code. */ /** * cik_pcie_gart_tlb_flush - gart tlb flush callback * * @rdev: radeon_device pointer * * Flush the TLB for the VMID 0 page table (CIK). */ void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) { … } /** * cik_pcie_gart_enable - gart enable * * @rdev: radeon_device pointer * * This sets up the TLBs, programs the page tables for VMID0, * sets up the hw for VMIDs 1-15 which are allocated on * demand, and sets up the global locations for the LDS, GDS, * and GPUVM for FSA64 clients (CIK). * Returns 0 for success, errors for failure. */ static int cik_pcie_gart_enable(struct radeon_device *rdev) { … } /** * cik_pcie_gart_disable - gart disable * * @rdev: radeon_device pointer * * This disables all VM page table (CIK). */ static void cik_pcie_gart_disable(struct radeon_device *rdev) { … } /** * cik_pcie_gart_fini - vm fini callback * * @rdev: radeon_device pointer * * Tears down the driver GART/VM setup (CIK). */ static void cik_pcie_gart_fini(struct radeon_device *rdev) { … } /* vm parser */ /** * cik_ib_parse - vm ib_parse callback * * @rdev: radeon_device pointer * @ib: indirect buffer pointer * * CIK uses hw IB checking so this is a nop (CIK). */ int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) { … } /* * vm * VMID 0 is the physical GPU addresses as used by the kernel. * VMIDs 1-15 are used for userspace clients and are handled * by the radeon vm/hsa code. */ /** * cik_vm_init - cik vm init callback * * @rdev: radeon_device pointer * * Inits cik specific vm parameters (number of VMs, base of vram for * VMIDs 1-15) (CIK). * Returns 0 for success. */ int cik_vm_init(struct radeon_device *rdev) { … } /** * cik_vm_fini - cik vm fini callback * * @rdev: radeon_device pointer * * Tear down any asic specific VM setup (CIK). */ void cik_vm_fini(struct radeon_device *rdev) { … } /** * cik_vm_decode_fault - print human readable fault info * * @rdev: radeon_device pointer * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value * * Print human readable fault information (CIK). */ static void cik_vm_decode_fault(struct radeon_device *rdev, u32 status, u32 addr, u32 mc_client) { … } /* * cik_vm_flush - cik vm flush using the CP * * Update the page table base and flush the VM TLB * using the CP (CIK). */ void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, unsigned vm_id, uint64_t pd_addr) { … } /* * RLC * The RLC is a multi-purpose microengine that handles a * variety of functions, the most important of which is * the interrupt controller. */ static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, bool enable) { … } static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) { … } static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) { … } static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) { … } static u32 cik_halt_rlc(struct radeon_device *rdev) { … } void cik_enter_rlc_safe_mode(struct radeon_device *rdev) { … } void cik_exit_rlc_safe_mode(struct radeon_device *rdev) { … } /** * cik_rlc_stop - stop the RLC ME * * @rdev: radeon_device pointer * * Halt the RLC ME (MicroEngine) (CIK). */ static void cik_rlc_stop(struct radeon_device *rdev) { … } /** * cik_rlc_start - start the RLC ME * * @rdev: radeon_device pointer * * Unhalt the RLC ME (MicroEngine) (CIK). */ static void cik_rlc_start(struct radeon_device *rdev) { … } /** * cik_rlc_resume - setup the RLC hw * * @rdev: radeon_device pointer * * Initialize the RLC registers, load the ucode, * and start the RLC (CIK). * Returns 0 for success, -EINVAL if the ucode is not available. */ static int cik_rlc_resume(struct radeon_device *rdev) { … } static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) { … } static const u32 mc_cg_registers[] = …; static void cik_enable_mc_ls(struct radeon_device *rdev, bool enable) { … } static void cik_enable_mc_mgcg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_sdma_mgcg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_sdma_mgls(struct radeon_device *rdev, bool enable) { … } static void cik_enable_uvd_mgcg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_bif_mgls(struct radeon_device *rdev, bool enable) { … } static void cik_enable_hdp_mgcg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_hdp_ls(struct radeon_device *rdev, bool enable) { … } void cik_update_cg(struct radeon_device *rdev, u32 block, bool enable) { … } static void cik_init_cg(struct radeon_device *rdev) { … } static void cik_fini_cg(struct radeon_device *rdev) { … } static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, bool enable) { … } static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, bool enable) { … } static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) { … } #define CP_ME_TABLE_SIZE … #define CP_ME_TABLE_OFFSET … #define CP_MEC_TABLE_OFFSET … void cik_init_cp_pg_table(struct radeon_device *rdev) { … } static void cik_enable_gfx_cgpg(struct radeon_device *rdev, bool enable) { … } static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) { … } static void cik_init_ao_cu_mask(struct radeon_device *rdev) { … } static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, bool enable) { … } static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, bool enable) { … } #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET … #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET … static void cik_init_gfx_cgpg(struct radeon_device *rdev) { … } static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) { … } u32 cik_get_csb_size(struct radeon_device *rdev) { … } void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) { … } static void cik_init_pg(struct radeon_device *rdev) { … } static void cik_fini_pg(struct radeon_device *rdev) { … } /* * Interrupts * Starting with r6xx, interrupts are handled via a ring buffer. * Ring buffers are areas of GPU accessible memory that the GPU * writes interrupt vectors into and the host reads vectors out of. * There is a rptr (read pointer) that determines where the * host is currently reading, and a wptr (write pointer) * which determines where the GPU has written. When the * pointers are equal, the ring is idle. When the GPU * writes vectors to the ring buffer, it increments the * wptr. When there is an interrupt, the host then starts * fetching commands and processing them until the pointers are * equal again at which point it updates the rptr. */ /** * cik_enable_interrupts - Enable the interrupt ring buffer * * @rdev: radeon_device pointer * * Enable the interrupt ring buffer (CIK). */ static void cik_enable_interrupts(struct radeon_device *rdev) { … } /** * cik_disable_interrupts - Disable the interrupt ring buffer * * @rdev: radeon_device pointer * * Disable the interrupt ring buffer (CIK). */ static void cik_disable_interrupts(struct radeon_device *rdev) { … } /** * cik_disable_interrupt_state - Disable all interrupt sources * * @rdev: radeon_device pointer * * Clear all interrupt enable bits used by the driver (CIK). */ static void cik_disable_interrupt_state(struct radeon_device *rdev) { … } /** * cik_irq_init - init and enable the interrupt ring * * @rdev: radeon_device pointer * * Allocate a ring buffer for the interrupt controller, * enable the RLC, disable interrupts, enable the IH * ring buffer and enable it (CIK). * Called at device load and reume. * Returns 0 for success, errors for failure. */ static int cik_irq_init(struct radeon_device *rdev) { … } /** * cik_irq_set - enable/disable interrupt sources * * @rdev: radeon_device pointer * * Enable interrupt sources on the GPU (vblanks, hpd, * etc.) (CIK). * Returns 0 for success, errors for failure. */ int cik_irq_set(struct radeon_device *rdev) { … } /** * cik_irq_ack - ack interrupt sources * * @rdev: radeon_device pointer * * Ack interrupt sources on the GPU (vblanks, hpd, * etc.) (CIK). Certain interrupts sources are sw * generated and do not require an explicit ack. */ static inline void cik_irq_ack(struct radeon_device *rdev) { … } /** * cik_irq_disable - disable interrupts * * @rdev: radeon_device pointer * * Disable interrupts on the hw (CIK). */ static void cik_irq_disable(struct radeon_device *rdev) { … } /** * cik_irq_suspend - disable interrupts for suspend * * @rdev: radeon_device pointer * * Disable interrupts and stop the RLC (CIK). * Used for suspend. */ static void cik_irq_suspend(struct radeon_device *rdev) { … } /** * cik_irq_fini - tear down interrupt support * * @rdev: radeon_device pointer * * Disable interrupts on the hw and free the IH ring * buffer (CIK). * Used for driver unload. */ static void cik_irq_fini(struct radeon_device *rdev) { … } /** * cik_get_ih_wptr - get the IH ring buffer wptr * * @rdev: radeon_device pointer * * Get the IH ring buffer wptr from either the register * or the writeback memory buffer (CIK). Also check for * ring buffer overflow and deal with it. * Used by cik_irq_process(). * Returns the value of the wptr. */ static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) { … } /* CIK IV Ring * Each IV ring entry is 128 bits: * [7:0] - interrupt source id * [31:8] - reserved * [59:32] - interrupt source data * [63:60] - reserved * [71:64] - RINGID * CP: * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher * - for gfx, hw shader state (0=PS...5=LS, 6=CS) * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes * PIPE_ID - ME0 0=3D * - ME1&2 compute dispatcher (4 pipes each) * SDMA: * INSTANCE_ID [1:0], QUEUE_ID[1:0] * INSTANCE_ID - 0 = sdma0, 1 = sdma1 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 * [79:72] - VMID * [95:80] - PASID * [127:96] - reserved */ /** * cik_irq_process - interrupt handler * * @rdev: radeon_device pointer * * Interrupt hander (CIK). Walk the IH ring, * ack interrupts and schedule work to handle * interrupt events. * Returns irq process return code. */ int cik_irq_process(struct radeon_device *rdev) { … } /* * startup/shutdown callbacks */ static void cik_uvd_init(struct radeon_device *rdev) { … } static void cik_uvd_start(struct radeon_device *rdev) { … } static void cik_uvd_resume(struct radeon_device *rdev) { … } static void cik_vce_init(struct radeon_device *rdev) { … } static void cik_vce_start(struct radeon_device *rdev) { … } static void cik_vce_resume(struct radeon_device *rdev) { … } /** * cik_startup - program the asic to a functional state * * @rdev: radeon_device pointer * * Programs the asic to a functional state (CIK). * Called by cik_init() and cik_resume(). * Returns 0 for success, error for failure. */ static int cik_startup(struct radeon_device *rdev) { … } /** * cik_resume - resume the asic to a functional state * * @rdev: radeon_device pointer * * Programs the asic to a functional state (CIK). * Called at resume. * Returns 0 for success, error for failure. */ int cik_resume(struct radeon_device *rdev) { … } /** * cik_suspend - suspend the asic * * @rdev: radeon_device pointer * * Bring the chip into a state suitable for suspend (CIK). * Called at suspend. * Returns 0 for success. */ int cik_suspend(struct radeon_device *rdev) { … } /* Plan is to move initialization in that function and use * helper function so that radeon_device_init pretty much * do nothing more than calling asic specific function. This * should also allow to remove a bunch of callback function * like vram_info. */ /** * cik_init - asic specific driver and hw init * * @rdev: radeon_device pointer * * Setup asic specific driver variables and program the hw * to a functional state (CIK). * Called at driver startup. * Returns 0 for success, errors for failure. */ int cik_init(struct radeon_device *rdev) { … } /** * cik_fini - asic specific driver and hw fini * * @rdev: radeon_device pointer * * Tear down the asic specific driver variables and program the hw * to an idle state (CIK). * Called at driver unload. */ void cik_fini(struct radeon_device *rdev) { … } void dce8_program_fmt(struct drm_encoder *encoder) { … } /* display watermark setup */ /** * dce8_line_buffer_adjust - Set up the line buffer * * @rdev: radeon_device pointer * @radeon_crtc: the selected display controller * @mode: the current display mode on the selected display * controller * * Setup up the line buffer allocation for * the selected display controller (CIK). * Returns the line buffer size in pixels. */ static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, struct radeon_crtc *radeon_crtc, struct drm_display_mode *mode) { … } /** * cik_get_number_of_dram_channels - get the number of dram channels * * @rdev: radeon_device pointer * * Look up the number of video ram channels (CIK). * Used for display watermark bandwidth calculations * Returns the number of dram channels */ static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) { … } struct dce8_wm_params { … }; /** * dce8_dram_bandwidth - get the dram bandwidth * * @wm: watermark calculation data * * Calculate the raw dram bandwidth (CIK). * Used for display watermark bandwidth calculations * Returns the dram bandwidth in MBytes/s */ static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_dram_bandwidth_for_display - get the dram bandwidth for display * * @wm: watermark calculation data * * Calculate the dram bandwidth used for display (CIK). * Used for display watermark bandwidth calculations * Returns the dram bandwidth for display in MBytes/s */ static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) { … } /** * dce8_data_return_bandwidth - get the data return bandwidth * * @wm: watermark calculation data * * Calculate the data return bandwidth used for display (CIK). * Used for display watermark bandwidth calculations * Returns the data return bandwidth in MBytes/s */ static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_dmif_request_bandwidth - get the dmif bandwidth * * @wm: watermark calculation data * * Calculate the dmif bandwidth used for display (CIK). * Used for display watermark bandwidth calculations * Returns the dmif bandwidth in MBytes/s */ static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_available_bandwidth - get the min available bandwidth * * @wm: watermark calculation data * * Calculate the min available bandwidth used for display (CIK). * Used for display watermark bandwidth calculations * Returns the min available bandwidth in MBytes/s */ static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_average_bandwidth - get the average available bandwidth * * @wm: watermark calculation data * * Calculate the average available bandwidth used for display (CIK). * Used for display watermark bandwidth calculations * Returns the average available bandwidth in MBytes/s */ static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_latency_watermark - get the latency watermark * * @wm: watermark calculation data * * Calculate the latency watermark (CIK). * Used for display watermark bandwidth calculations * Returns the latency watermark in ns */ static u32 dce8_latency_watermark(struct dce8_wm_params *wm) { … } /** * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check * average and available dram bandwidth * * @wm: watermark calculation data * * Check if the display average bandwidth fits in the display * dram bandwidth (CIK). * Used for display watermark bandwidth calculations * Returns true if the display fits, false if not. */ static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) { … } /** * dce8_average_bandwidth_vs_available_bandwidth - check * average and available bandwidth * * @wm: watermark calculation data * * Check if the display average bandwidth fits in the display * available bandwidth (CIK). * Used for display watermark bandwidth calculations * Returns true if the display fits, false if not. */ static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) { … } /** * dce8_check_latency_hiding - check latency hiding * * @wm: watermark calculation data * * Check latency hiding (CIK). * Used for display watermark bandwidth calculations * Returns true if the display fits, false if not. */ static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) { … } /** * dce8_program_watermarks - program display watermarks * * @rdev: radeon_device pointer * @radeon_crtc: the selected display controller * @lb_size: line buffer size * @num_heads: number of display controllers in use * * Calculate and program the display watermarks for the * selected display controller (CIK). */ static void dce8_program_watermarks(struct radeon_device *rdev, struct radeon_crtc *radeon_crtc, u32 lb_size, u32 num_heads) { … } /** * dce8_bandwidth_update - program display watermarks * * @rdev: radeon_device pointer * * Calculate and program the display watermarks and line * buffer allocation (CIK). */ void dce8_bandwidth_update(struct radeon_device *rdev) { … } /** * cik_get_gpu_clock_counter - return GPU clock counter snapshot * * @rdev: radeon_device pointer * * Fetches a GPU clock counter snapshot (SI). * Returns the 64 bit clock counter snapshot. */ uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) { … } static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, u32 cntl_reg, u32 status_reg) { … } int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) { … } int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) { … } static void cik_pcie_gen3_enable(struct radeon_device *rdev) { … } static void cik_program_aspm(struct radeon_device *rdev) { … }
Codebase Browser
linux