#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/pci.h>
#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include "soc15.h"
#include "soc15d.h"
#include "amdgpu_atomfirmware.h"
#include "amdgpu_pm.h"
#include "gc/gc_9_0_offset.h"
#include "gc/gc_9_0_sh_mask.h"
#include "vega10_enum.h"
#include "soc15_common.h"
#include "clearstate_gfx9.h"
#include "v9_structs.h"
#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
#include "amdgpu_ras.h"
#include "amdgpu_ring_mux.h"
#include "gfx_v9_4.h"
#include "gfx_v9_0.h"
#include "gfx_v9_4_2.h"
#include "asic_reg/pwr/pwr_10_0_offset.h"
#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
#include "asic_reg/gc/gc_9_0_default.h"
#define GFX9_NUM_GFX_RINGS …
#define GFX9_NUM_SW_GFX_RINGS …
#define GFX9_MEC_HPD_SIZE …
#define RLCG_UCODE_LOADING_START_ADDRESS …
#define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET …
#define mmGCEA_PROBE_MAP …
#define mmGCEA_PROBE_MAP_BASE_IDX …
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
#define mmTCP_CHAN_STEER_0_ARCT …
#define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX …
#define mmTCP_CHAN_STEER_1_ARCT …
#define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX …
#define mmTCP_CHAN_STEER_2_ARCT …
#define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX …
#define mmTCP_CHAN_STEER_3_ARCT …
#define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX …
#define mmTCP_CHAN_STEER_4_ARCT …
#define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX …
#define mmTCP_CHAN_STEER_5_ARCT …
#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX …
#define mmGOLDEN_TSC_COUNT_UPPER_Renoir …
#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX …
#define mmGOLDEN_TSC_COUNT_LOWER_Renoir …
#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX …
static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = …;
static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = …;
enum ta_ras_gfx_subblock { … };
struct ras_gfx_subblock { … };
#define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) …
static const struct ras_gfx_subblock ras_gfx_subblocks[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_0[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_1[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = …;
static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = …;
static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = …;
static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = …;
static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = …;
#define VEGA10_GB_ADDR_CONFIG_GOLDEN …
#define VEGA12_GB_ADDR_CONFIG_GOLDEN …
#define RAVEN_GB_ADDR_CONFIG_GOLDEN …
#define RAVEN2_GB_ADDR_CONFIG_GOLDEN …
static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
struct amdgpu_cu_info *cu_info);
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
void *ras_error_status);
static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
void *inject_if, uint32_t instance_mask);
static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
unsigned int vmid);
static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
uint64_t queue_mask)
{ … }
static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
struct amdgpu_ring *ring,
enum amdgpu_unmap_queues_action action,
u64 gpu_addr, u64 seq)
{ … }
static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
struct amdgpu_ring *ring,
u64 addr,
u64 seq)
{ … }
static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
uint16_t pasid, uint32_t flush_type,
bool all_hub)
{ … }
static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = …;
static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
bool wc, uint32_t reg, uint32_t val)
{ … }
static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
int mem_space, int opt, uint32_t addr0,
uint32_t addr1, uint32_t ref, uint32_t mask,
uint32_t inv)
{ … }
static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
{ … }
static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
{ … }
struct amdgpu_gfxoff_quirk { … };
static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = …;
static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
{ … }
static bool is_raven_kicker(struct amdgpu_device *adev)
{ … }
static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
char *chip_name)
{ … }
static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
char *chip_name)
{ … }
static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
char *chip_name)
{ … }
static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
{ … }
static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
volatile u32 *buffer)
{ … }
static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
{ … }
static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
{ … }
static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
{ … }
static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
{ … }
static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
uint32_t wave, uint32_t thread,
uint32_t regno, uint32_t num, uint32_t *out)
{ … }
static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
{ … }
static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
uint32_t wave, uint32_t start,
uint32_t size, uint32_t *dst)
{ … }
static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
uint32_t wave, uint32_t thread,
uint32_t start, uint32_t size,
uint32_t *dst)
{ … }
static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
{ … }
static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = …;
const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = …;
static struct amdgpu_gfx_ras gfx_v9_0_ras = …;
static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
int mec, int pipe, int queue)
{ … }
static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_sw_init(void *handle)
{ … }
static int gfx_v9_0_sw_fini(void *handle)
{ … }
static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
{ … }
void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
u32 instance, int xcc_id)
{ … }
static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
uint32_t first_vmid,
uint32_t last_vmid)
{ … }
#define DEFAULT_SH_MEM_BASES …
static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
{ … }
static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
int indirect_offset,
int list_size,
int *unique_indirect_regs,
int unique_indirect_reg_count,
int *indirect_start_offsets,
int *indirect_start_offsets_count,
int max_start_offsets_count)
{ … }
static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
{ … }
static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
{ … }
static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
{ … }
static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
{ … }
static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
{ … }
static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
{ … }
static int gfx_v9_0_hw_init(void *handle)
{ … }
static int gfx_v9_0_hw_fini(void *handle)
{ … }
static int gfx_v9_0_suspend(void *handle)
{ … }
static int gfx_v9_0_resume(void *handle)
{ … }
static bool gfx_v9_0_is_idle(void *handle)
{ … }
static int gfx_v9_0_wait_for_idle(void *handle)
{ … }
static int gfx_v9_0_soft_reset(void *handle)
{ … }
static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
{ … }
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
uint32_t vmid,
uint32_t gds_base, uint32_t gds_size,
uint32_t gws_base, uint32_t gws_size,
uint32_t oa_base, uint32_t oa_size)
{ … }
static const u32 vgpr_init_compute_shader[] = …;
static const u32 sgpr_init_compute_shader[] = …;
static const u32 vgpr_init_compute_shader_arcturus[] = …;
static const struct soc15_reg_entry vgpr_init_regs[] = …;
static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = …;
static const struct soc15_reg_entry sgpr1_init_regs[] = …;
static const struct soc15_reg_entry sgpr2_init_regs[] = …;
static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = …;
static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_early_init(void *handle)
{ … }
static int gfx_v9_0_ecc_late_init(void *handle)
{ … }
static int gfx_v9_0_late_init(void *handle)
{ … }
static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
{ … }
static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
{ … }
static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
bool enable)
{ … }
static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
unsigned int vmid)
{ … }
static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
{ … }
static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
uint32_t offset,
struct soc15_reg_rlcg *entries, int arr_size)
{ … }
static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
{ … }
static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = …;
static int gfx_v9_0_set_powergating_state(void *handle,
enum amd_powergating_state state)
{ … }
static int gfx_v9_0_set_clockgating_state(void *handle,
enum amd_clockgating_state state)
{ … }
static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
{ … }
static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
{ … }
static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
struct amdgpu_job *job,
struct amdgpu_ib *ib,
uint32_t flags)
{ … }
static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
unsigned offset)
{ … }
static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
unsigned offset)
{ … }
static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
unsigned offset)
{ … }
static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
struct amdgpu_job *job,
struct amdgpu_ib *ib,
uint32_t flags)
{ … }
static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
u64 seq, unsigned flags)
{ … }
static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
unsigned vmid, uint64_t pd_addr)
{ … }
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
{ … }
static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
u64 seq, unsigned int flags)
{ … }
static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
{ … }
static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
{ … }
static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
bool secure)
{ … }
static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
{ … }
static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
uint64_t addr)
{ … }
static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
uint32_t reg_val_offs)
{ … }
static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
uint32_t val)
{ … }
static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
uint32_t val, uint32_t mask)
{ … }
static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
uint32_t reg0, uint32_t reg1,
uint32_t ref, uint32_t mask)
{ … }
static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
{ … }
static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
enum amdgpu_interrupt_state state)
{ … }
static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
int me, int pipe,
enum amdgpu_interrupt_state state)
{ … }
static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
unsigned type,
enum amdgpu_interrupt_state state)
{ … }
static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
unsigned type,
enum amdgpu_interrupt_state state)
{ … }
#define ENABLE_ECC_ON_ME_PIPE(me, pipe) …
#define DISABLE_ECC_ON_ME_PIPE(me, pipe) …
static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
unsigned type,
enum amdgpu_interrupt_state state)
{ … }
static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
struct amdgpu_irq_src *src,
unsigned type,
enum amdgpu_interrupt_state state)
{ … }
static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{ … }
static void gfx_v9_0_fault(struct amdgpu_device *adev,
struct amdgpu_iv_entry *entry)
{ … }
static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{ … }
static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{ … }
static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = …;
static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
void *inject_if, uint32_t instance_mask)
{ … }
static const char * const vml2_mems[] = …;
static const char * const vml2_walker_mems[] = …;
static const char * const atc_l2_cache_2m_mems[] = …;
static const char *atc_l2_cache_4k_mems[] = …;
static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
struct ras_err_data *err_data)
{ … }
static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
const struct soc15_reg_entry *reg,
uint32_t se_id, uint32_t inst_id, uint32_t value,
uint32_t *sec_count, uint32_t *ded_count)
{ … }
static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
void *ras_error_status)
{ … }
static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
{ … }
static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
uint32_t pipe, bool enable)
{ … }
static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
{ … }
static void gfx_v9_ip_print(void *handle, struct drm_printer *p)
{ … }
static void gfx_v9_ip_dump(void *handle)
{ … }
static const struct amd_ip_funcs gfx_v9_0_ip_funcs = …;
static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = …;
static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = …;
static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = …;
static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = …;
static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
{ … }
static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = …;
static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = …;
static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = …;
static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = …;
static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
{ … }
static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
u32 bitmap)
{ … }
static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
{ … }
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
struct amdgpu_cu_info *cu_info)
{ … }
const struct amdgpu_ip_block_version gfx_v9_0_ip_block = …;