linux/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/pci.h>

#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include "amdgpu_ring.h"
#include "vi.h"
#include "vi_structs.h"
#include "vid.h"
#include "amdgpu_ucode.h"
#include "amdgpu_atombios.h"
#include "atombios_i2c.h"
#include "clearstate_vi.h"

#include "gmc/gmc_8_2_d.h"
#include "gmc/gmc_8_2_sh_mask.h"

#include "oss/oss_3_0_d.h"
#include "oss/oss_3_0_sh_mask.h"

#include "bif/bif_5_0_d.h"
#include "bif/bif_5_0_sh_mask.h"
#include "gca/gfx_8_0_d.h"
#include "gca/gfx_8_0_enum.h"
#include "gca/gfx_8_0_sh_mask.h"

#include "dce/dce_10_0_d.h"
#include "dce/dce_10_0_sh_mask.h"

#include "smu/smu_7_1_3_d.h"

#include "ivsrcid/ivsrcid_vislands30.h"

#define GFX8_NUM_GFX_RINGS …
#define GFX8_MEC_HPD_SIZE …

#define TOPAZ_GB_ADDR_CONFIG_GOLDEN …
#define CARRIZO_GB_ADDR_CONFIG_GOLDEN …
#define POLARIS11_GB_ADDR_CONFIG_GOLDEN …
#define TONGA_GB_ADDR_CONFIG_GOLDEN …

#define ARRAY_MODE(x) …
#define PIPE_CONFIG(x) …
#define TILE_SPLIT(x) …
#define MICRO_TILE_MODE_NEW(x) …
#define SAMPLE_SPLIT(x) …
#define BANK_WIDTH(x) …
#define BANK_HEIGHT(x) …
#define MACRO_TILE_ASPECT(x) …
#define NUM_BANKS(x) …

#define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK …
#define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK …
#define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK …
#define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK …
#define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK …
#define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK …

/* BPM SERDES CMD */
#define SET_BPM_SERDES_CMD …
#define CLE_BPM_SERDES_CMD …

/* BPM Register Address*/
enum { … };

#define RLC_FormatDirectRegListLength …

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = …;

static const u32 golden_settings_tonga_a11[] = …;

static const u32 tonga_golden_common_all[] = …;

static const u32 tonga_mgcg_cgcg_init[] = …;

static const u32 golden_settings_vegam_a11[] = …;

static const u32 vegam_golden_common_all[] = …;

static const u32 golden_settings_polaris11_a11[] = …;

static const u32 polaris11_golden_common_all[] = …;

static const u32 golden_settings_polaris10_a11[] = …;

static const u32 polaris10_golden_common_all[] = …;

static const u32 fiji_golden_common_all[] = …;

static const u32 golden_settings_fiji_a10[] = …;

static const u32 fiji_mgcg_cgcg_init[] = …;

static const u32 golden_settings_iceland_a11[] = …;

static const u32 iceland_golden_common_all[] = …;

static const u32 iceland_mgcg_cgcg_init[] = …;

static const u32 cz_golden_settings_a11[] = …;

static const u32 cz_golden_common_all[] = …;

static const u32 cz_mgcg_cgcg_init[] = …;

static const u32 stoney_golden_settings_a11[] = …;

static const u32 stoney_golden_common_all[] = …;

static const u32 stoney_mgcg_cgcg_init[] = …;


static const char * const sq_edc_source_names[] = …;

static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);

#define CG_ACLK_CNTL__ACLK_DIVIDER_MASK …
#define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT …

static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
{ … }

static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
{ … }


static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
				    volatile u32 *buffer)
{ … }

static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
{ … }

static const u32 vgpr_init_compute_shader[] = …;

static const u32 sgpr_init_compute_shader[] = …;

static const u32 vgpr_init_regs[] = …;

static const u32 sgpr1_init_regs[] = …;

static const u32 sgpr2_init_regs[] = …;

static const u32 sec_ded_counter_registers[] = …;

static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
					int mec, int pipe, int queue)
{ … }

static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);

static int gfx_v8_0_sw_init(void *handle)
{ … }

static int gfx_v8_0_sw_fini(void *handle)
{ … }

static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
				  u32 se_num, u32 sh_num, u32 instance,
				  int xcc_id)
{ … }

static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
{ … }

static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
{ … }

static void
gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
{ … }

static void
gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
					u32 raster_config, u32 raster_config_1,
					unsigned rb_mask, unsigned num_rb)
{ … }

static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
{ … }

#define DEFAULT_SH_MEM_BASES …
/**
 * gfx_v8_0_init_compute_vmid - gart enable
 *
 * @adev: amdgpu_device pointer
 *
 * Initialize compute vmid sh_mem registers
 *
 */
static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_config_init(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
					       bool enable)
{ … }

static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
				int ind_offset,
				int list_size,
				int *unique_indices,
				int *indices_count,
				int max_indices,
				int *ind_start_offsets,
				int *offset_count,
				int max_offset)
{ … }

static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
{ … }

static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
						bool enable)
{ … }

static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
						  bool enable)
{ … }

static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
{ … }

static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
{ … }

static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
{ … }
static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
{ … }

static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
{ … }

/* KIQ functions */
static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
{ … }

static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
{ … }

static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd)
{ … }

static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
{ … }

static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
			struct vi_mqd *mqd)
{ … }

static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
{ … }

static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
{ … }

static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
{ … }

static int gfx_v8_0_hw_init(void *handle)
{ … }

static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
{ … }

static bool gfx_v8_0_is_idle(void *handle)
{ … }

static bool gfx_v8_0_rlc_is_idle(void *handle)
{ … }

static int gfx_v8_0_wait_for_rlc_idle(void *handle)
{ … }

static int gfx_v8_0_wait_for_idle(void *handle)
{ … }

static int gfx_v8_0_hw_fini(void *handle)
{ … }

static int gfx_v8_0_suspend(void *handle)
{ … }

static int gfx_v8_0_resume(void *handle)
{ … }

static bool gfx_v8_0_check_soft_reset(void *handle)
{ … }

static int gfx_v8_0_pre_soft_reset(void *handle)
{ … }

static int gfx_v8_0_soft_reset(void *handle)
{ … }

static int gfx_v8_0_post_soft_reset(void *handle)
{ … }

/**
 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
 *
 * @adev: amdgpu_device pointer
 *
 * Fetches a GPU clock counter snapshot.
 * Returns the 64 bit clock counter snapshot.
 */
static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
					  uint32_t vmid,
					  uint32_t gds_base, uint32_t gds_size,
					  uint32_t gws_base, uint32_t gws_size,
					  uint32_t oa_base, uint32_t oa_size)
{ … }

static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
{ … }

static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
			   uint32_t wave, uint32_t thread,
			   uint32_t regno, uint32_t num, uint32_t *out)
{ … }

static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
{ … }

static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
				     uint32_t wave, uint32_t start,
				     uint32_t size, uint32_t *dst)
{ … }


static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = …;

static int gfx_v8_0_early_init(void *handle)
{ … }

static int gfx_v8_0_late_init(void *handle)
{ … }

static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
						       bool enable)
{ … }

static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
							bool enable)
{ … }

static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
		bool enable)
{ … }

static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
					  bool enable)
{ … }

static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
						bool enable)
{ … }

static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
					  bool enable)
{ … }

static int gfx_v8_0_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{ … }

static void gfx_v8_0_get_clockgating_state(void *handle, u64 *flags)
{ … }

static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
				     uint32_t reg_addr, uint32_t cmd)
{ … }

#define MSG_ENTER_RLC_SAFE_MODE …
#define MSG_EXIT_RLC_SAFE_MODE …
#define RLC_GPR_REG2__REQ_MASK …
#define RLC_GPR_REG2__REQ__SHIFT …
#define RLC_GPR_REG2__MESSAGE__SHIFT …
#define RLC_GPR_REG2__MESSAGE_MASK …

static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
{ … }

static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
{ … }

static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
{ … }

static const struct amdgpu_rlc_funcs iceland_rlc_funcs = …;

static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
						      bool enable)
{ … }

static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
						      bool enable)
{ … }
static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
					    bool enable)
{ … }

static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
					  enum amd_clockgating_state state)
{ … }

static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
					  enum amd_clockgating_state state)
{ … }

static int gfx_v8_0_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{ … }

static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
{ … }

static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
					struct amdgpu_job *job,
					struct amdgpu_ib *ib,
					uint32_t flags)
{ … }

static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
					  struct amdgpu_job *job,
					  struct amdgpu_ib *ib,
					  uint32_t flags)
{ … }

static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
					 u64 seq, unsigned flags)
{ … }

static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
					unsigned vmid, uint64_t pd_addr)
{ … }

static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
					     u64 addr, u64 seq,
					     unsigned flags)
{ … }

static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
					 u64 seq, unsigned int flags)
{ … }

static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
{ … }

static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
						  uint64_t addr)
{ … }

static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
				    uint32_t reg_val_offs)
{ … }

static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
				  uint32_t val)
{ … }

static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
{ … }

static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
						 enum amdgpu_interrupt_state state)
{ … }

static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
						     int me, int pipe,
						     enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
					     struct amdgpu_irq_src *source,
					     unsigned type,
					     enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
					      struct amdgpu_irq_src *source,
					      unsigned type,
					      enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
					    struct amdgpu_irq_src *src,
					    unsigned type,
					    enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
					 struct amdgpu_irq_src *source,
					 unsigned int type,
					 enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
				     struct amdgpu_irq_src *source,
				     unsigned int type,
				     enum amdgpu_interrupt_state state)
{ … }

static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
			    struct amdgpu_irq_src *source,
			    struct amdgpu_iv_entry *entry)
{ … }

static void gfx_v8_0_fault(struct amdgpu_device *adev,
			   struct amdgpu_iv_entry *entry)
{ … }

static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
				 struct amdgpu_irq_src *source,
				 struct amdgpu_iv_entry *entry)
{ … }

static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
				  struct amdgpu_irq_src *source,
				  struct amdgpu_iv_entry *entry)
{ … }

static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
				     struct amdgpu_irq_src *source,
				     struct amdgpu_iv_entry *entry)
{ … }

static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data,
				  bool from_wq)
{ … }

static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
{ … }

static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
			   struct amdgpu_irq_src *source,
			   struct amdgpu_iv_entry *entry)
{ … }

static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring)
{ … }


/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
#define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT …
static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
					uint32_t pipe, bool enable)
{ … }

#define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT …
static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
{ … }

static const struct amd_ip_funcs gfx_v8_0_ip_funcs = …;

static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = …;

static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = …;

static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = …;

static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
{ … }

static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = …;

static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = …;

static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = …;

static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = …;

static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = …;

static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
						 u32 bitmap)
{ … }

static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
{ … }

static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
{ … }

const struct amdgpu_ip_block_version gfx_v8_0_ip_block = …;

const struct amdgpu_ip_block_version gfx_v8_1_ip_block = …;

static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
{ … }

static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
{ … }