/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/delay.h> #include <linux/kernel.h> #include <linux/firmware.h> #include <linux/module.h> #include <linux/pci.h> #include "amdgpu.h" #include "amdgpu_gfx.h" #include "amdgpu_psp.h" #include "amdgpu_smu.h" #include "amdgpu_atomfirmware.h" #include "imu_v12_0.h" #include "soc24.h" #include "nvd.h" #include "gc/gc_12_0_0_offset.h" #include "gc/gc_12_0_0_sh_mask.h" #include "soc24_enum.h" #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" #include "soc15.h" #include "soc15d.h" #include "clearstate_gfx12.h" #include "v12_structs.h" #include "gfx_v12_0.h" #include "nbif_v6_3_1.h" #include "mes_v12_0.h" #define GFX12_NUM_GFX_RINGS … #define GFX12_MEC_HPD_SIZE … #define RLCG_UCODE_LOADING_START_ADDRESS … MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; MODULE_FIRMWARE(…) …; static const struct amdgpu_hwip_reg_entry gc_reg_list_12_0[] = …; static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_12[] = …; static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_12[] = …; #define DEFAULT_SH_MEM_CONFIG … static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev); static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev); static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev); static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev); static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev); static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev); static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev, struct amdgpu_cu_info *cu_info); static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev); static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance, int xcc_id); static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val); static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, uint16_t pasid, uint32_t flush_type, bool all_hub, uint8_t dst_sel); static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev, bool enable); static void gfx_v12_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) { … } static void gfx_v12_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, struct amdgpu_ring *ring) { … } static void gfx_v12_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, struct amdgpu_ring *ring, enum amdgpu_unmap_queues_action action, u64 gpu_addr, u64 seq) { … } static void gfx_v12_0_kiq_query_status(struct amdgpu_ring *kiq_ring, struct amdgpu_ring *ring, u64 addr, u64 seq) { … } static void gfx_v12_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, uint16_t pasid, uint32_t flush_type, bool all_hub) { … } static const struct kiq_pm4_funcs gfx_v12_0_kiq_pm4_funcs = …; static void gfx_v12_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) { … } static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, int mem_space, int opt, uint32_t addr0, uint32_t addr1, uint32_t ref, uint32_t mask, uint32_t inv) { … } static int gfx_v12_0_ring_test_ring(struct amdgpu_ring *ring) { … } static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) { … } static void gfx_v12_0_free_microcode(struct amdgpu_device *adev) { … } static int gfx_v12_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) { … } static int gfx_v12_0_init_microcode(struct amdgpu_device *adev) { … } static u32 gfx_v12_0_get_csb_size(struct amdgpu_device *adev) { … } static void gfx_v12_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer) { … } static void gfx_v12_0_rlc_fini(struct amdgpu_device *adev) { … } static void gfx_v12_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) { … } static int gfx_v12_0_rlc_init(struct amdgpu_device *adev) { … } static void gfx_v12_0_mec_fini(struct amdgpu_device *adev) { … } static void gfx_v12_0_me_init(struct amdgpu_device *adev) { … } static int gfx_v12_0_mec_init(struct amdgpu_device *adev) { … } static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) { … } static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, uint32_t thread, uint32_t regno, uint32_t num, uint32_t *out) { … } static void gfx_v12_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) { … } static void gfx_v12_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t start, uint32_t size, uint32_t *dst) { … } static void gfx_v12_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t thread, uint32_t start, uint32_t size, uint32_t *dst) { … } static void gfx_v12_0_select_me_pipe_q(struct amdgpu_device *adev, u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) { … } static const struct amdgpu_gfx_funcs gfx_v12_0_gfx_funcs = …; static int gfx_v12_0_gpu_early_init(struct amdgpu_device *adev) { … } static int gfx_v12_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, int me, int pipe, int queue) { … } static int gfx_v12_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, int mec, int pipe, int queue) { … } static struct { … } rlc_autoload_info[SOC24_FIRMWARE_ID_MAX]; #define RLC_TOC_OFFSET_DWUNIT … #define RLC_SIZE_MULTIPLE … #define RLC_TOC_UMF_SIZE_inM … #define RLC_TOC_FORMAT_API … static void gfx_v12_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) { … } static uint32_t gfx_v12_0_calc_toc_total_size(struct amdgpu_device *adev) { … } static int gfx_v12_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, SOC24_FIRMWARE_ID id, const void *fw_data, uint32_t fw_size) { … } static void gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev) { … } static int gfx_v12_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) { … } static void gfx_v12_0_alloc_ip_dump(struct amdgpu_device *adev) { … } static int gfx_v12_0_sw_init(void *handle) { … } static void gfx_v12_0_pfp_fini(struct amdgpu_device *adev) { … } static void gfx_v12_0_me_fini(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) { … } static int gfx_v12_0_sw_fini(void *handle) { … } static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance, int xcc_id) { … } static u32 gfx_v12_0_get_sa_active_bitmap(struct amdgpu_device *adev) { … } static u32 gfx_v12_0_get_rb_active_bitmap(struct amdgpu_device *adev) { … } static void gfx_v12_0_setup_rb(struct amdgpu_device *adev) { … } #define LDS_APP_BASE … #define SCRATCH_APP_BASE … static void gfx_v12_0_init_compute_vmid(struct amdgpu_device *adev) { … } static void gfx_v12_0_tcp_harvest(struct amdgpu_device *adev) { … } static void gfx_v12_0_get_tcc_info(struct amdgpu_device *adev) { … } static void gfx_v12_0_constants_init(struct amdgpu_device *adev) { … } static void gfx_v12_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_init_csb(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_stop(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_reset(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, bool enable) { … } static void gfx_v12_0_rlc_start(struct amdgpu_device *adev) { … } static void gfx_v12_0_rlc_enable_srm(struct amdgpu_device *adev) { … } static void gfx_v12_0_load_rlcg_microcode(struct amdgpu_device *adev) { … } static void gfx_v12_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) { … } static int gfx_v12_0_rlc_load_microcode(struct amdgpu_device *adev) { … } static int gfx_v12_0_rlc_resume(struct amdgpu_device *adev) { … } static void gfx_v12_0_config_gfx_rs64(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_pfp_ucode_start_addr(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_me_ucode_start_addr(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_mec_ucode_start_addr(struct amdgpu_device *adev) { … } static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) { … } static int gfx_v12_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) { … } static int gfx_v12_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) { … } static int gfx_v12_0_cp_gfx_load_microcode(struct amdgpu_device *adev) { … } static int gfx_v12_0_cp_gfx_start(struct amdgpu_device *adev) { … } static void gfx_v12_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, CP_PIPE_ID pipe) { … } static void gfx_v12_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, struct amdgpu_ring *ring) { … } static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev) { … } static void gfx_v12_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) { … } static void gfx_v12_0_kiq_setting(struct amdgpu_ring *ring) { … } static void gfx_v12_0_cp_set_doorbell_range(struct amdgpu_device *adev) { … } static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, struct amdgpu_mqd_prop *prop) { … } static int gfx_v12_0_gfx_init_queue(struct amdgpu_ring *ring) { … } static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) { … } static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m, struct amdgpu_mqd_prop *prop) { … } static int gfx_v12_0_kiq_init_register(struct amdgpu_ring *ring) { … } static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring) { … } static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring) { … } static int gfx_v12_0_kiq_resume(struct amdgpu_device *adev) { … } static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev) { … } static int gfx_v12_0_cp_resume(struct amdgpu_device *adev) { … } static void gfx_v12_0_cp_enable(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_gfxhub_enable(struct amdgpu_device *adev) { … } static int get_gb_addr_config(struct amdgpu_device *adev) { … } static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev) { … } static int gfx_v12_0_hw_init(void *handle) { … } static int gfx_v12_0_kiq_disable_kgq(struct amdgpu_device *adev) { … } static int gfx_v12_0_hw_fini(void *handle) { … } static int gfx_v12_0_suspend(void *handle) { … } static int gfx_v12_0_resume(void *handle) { … } static bool gfx_v12_0_is_idle(void *handle) { … } static int gfx_v12_0_wait_for_idle(void *handle) { … } static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev) { … } static int gfx_v12_0_early_init(void *handle) { … } static int gfx_v12_0_late_init(void *handle) { … } static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) { … } static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) { … } static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev, bool enable) { … } static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid) { … } static const struct amdgpu_rlc_funcs gfx_v12_0_rlc_funcs = …; #if 0 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable) { /* TODO */ } static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable) { /* TODO */ } #endif static int gfx_v12_0_set_powergating_state(void *handle, enum amd_powergating_state state) { … } static void gfx_v12_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, bool enable) { … } static void gfx_v12_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, bool enable) { … } static void gfx_v12_0_update_repeater_fgcg(struct amdgpu_device *adev, bool enable) { … } static void gfx_v12_0_update_sram_fgcg(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_update_gfx_clock_gating(struct amdgpu_device *adev, bool enable) { … } static int gfx_v12_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { … } static void gfx_v12_0_get_clockgating_state(void *handle, u64 *flags) { … } static u64 gfx_v12_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) { … } static u64 gfx_v12_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) { … } static u64 gfx_v12_0_ring_get_rptr_compute(struct amdgpu_ring *ring) { … } static u64 gfx_v12_0_ring_get_wptr_compute(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_set_wptr_compute(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, uint32_t flags) { … } static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, uint32_t flags) { … } static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned flags) { … } static void gfx_v12_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, uint16_t pasid, uint32_t flush_type, bool all_hub, uint8_t dst_sel) { … } static void gfx_v12_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { … } static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned int flags) { … } static void gfx_v12_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) { … } static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, uint64_t addr) { … } static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring) { … } static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure) { … } static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t reg_val_offs) { … } static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) { … } static void gfx_v12_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, uint32_t val, uint32_t mask) { … } static void gfx_v12_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask) { … } static void gfx_v12_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) { … } static void gfx_v12_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, uint32_t me, uint32_t pipe, enum amdgpu_interrupt_state state) { … } static void gfx_v12_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, int me, int pipe, enum amdgpu_interrupt_state state) { … } static int gfx_v12_0_set_eop_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type, enum amdgpu_interrupt_state state) { … } static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { … } static int gfx_v12_0_set_priv_reg_fault_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned type, enum amdgpu_interrupt_state state) { … } static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned type, enum amdgpu_interrupt_state state) { … } static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { … } static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { … } static int gfx_v12_0_priv_inst_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { … } static void gfx_v12_0_emit_mem_sync(struct amdgpu_ring *ring) { … } static void gfx_v12_ip_print(void *handle, struct drm_printer *p) { … } static void gfx_v12_ip_dump(void *handle) { … } static const struct amd_ip_funcs gfx_v12_0_ip_funcs = …; static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = …; static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_compute = …; static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_kiq = …; static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev) { … } static const struct amdgpu_irq_src_funcs gfx_v12_0_eop_irq_funcs = …; static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_reg_irq_funcs = …; static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_inst_irq_funcs = …; static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev) { … } static void gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, u32 bitmap) { … } static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) { … } static u32 gfx_v12_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) { … } static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev, struct amdgpu_cu_info *cu_info) { … } const struct amdgpu_ip_block_version gfx_v12_0_ip_block = …;
Codebase Browser
linux