linux/drivers/gpu/drm/radeon/si.c

/*
 * Copyright 2011 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Alex Deucher
 */

#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>

#include <drm/drm_vblank.h>
#include <drm/radeon_drm.h>

#include "atom.h"
#include "clearstate_si.h"
#include "evergreen.h"
#include "r600.h"
#include "radeon.h"
#include "radeon_asic.h"
#include "radeon_audio.h"
#include "radeon_ucode.h"
#include "si_blit_shaders.h"
#include "si.h"
#include "sid.h"


MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

MODULE_FIRMWARE(…) …;

static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
static void si_pcie_gen3_enable(struct radeon_device *rdev);
static void si_program_aspm(struct radeon_device *rdev);
extern void sumo_rlc_fini(struct radeon_device *rdev);
extern int sumo_rlc_init(struct radeon_device *rdev);
static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
					 bool enable);
static void si_init_pg(struct radeon_device *rdev);
static void si_init_cg(struct radeon_device *rdev);
static void si_fini_pg(struct radeon_device *rdev);
static void si_fini_cg(struct radeon_device *rdev);
static void si_rlc_stop(struct radeon_device *rdev);

static const u32 crtc_offsets[] = …;

static const u32 si_disp_int_status[] = …;

#define DC_HPDx_CONTROL(x) …
#define DC_HPDx_INT_CONTROL(x) …
#define DC_HPDx_INT_STATUS_REG(x) …

static const u32 verde_rlc_save_restore_register_list[] = …;

static const u32 tahiti_golden_rlc_registers[] = …;

static const u32 tahiti_golden_registers[] = …;

static const u32 tahiti_golden_registers2[] = …;

static const u32 pitcairn_golden_rlc_registers[] = …;

static const u32 pitcairn_golden_registers[] = …;

static const u32 verde_golden_rlc_registers[] = …;

static const u32 verde_golden_registers[] = …;

static const u32 oland_golden_rlc_registers[] = …;

static const u32 oland_golden_registers[] = …;

static const u32 hainan_golden_registers[] = …;

static const u32 hainan_golden_registers2[] = …;

static const u32 tahiti_mgcg_cgcg_init[] = …;

static const u32 pitcairn_mgcg_cgcg_init[] = …;

static const u32 verde_mgcg_cgcg_init[] = …;

static const u32 oland_mgcg_cgcg_init[] = …;

static const u32 hainan_mgcg_cgcg_init[] = …;

static u32 verde_pg_init[] = …;

static void si_init_golden_registers(struct radeon_device *rdev)
{ … }

/**
 * si_get_allowed_info_register - fetch the register for the info ioctl
 *
 * @rdev: radeon_device pointer
 * @reg: register offset in bytes
 * @val: register value
 *
 * Returns 0 for success or -EINVAL for an invalid register
 *
 */
int si_get_allowed_info_register(struct radeon_device *rdev,
				 u32 reg, u32 *val)
{ … }

#define PCIE_BUS_CLK …
#define TCLK …

/**
 * si_get_xclk - get the xclk
 *
 * @rdev: radeon_device pointer
 *
 * Returns the reference clock used by the gfx engine
 * (SI).
 */
u32 si_get_xclk(struct radeon_device *rdev)
{ … }

/* get temperature in millidegrees */
int si_get_temp(struct radeon_device *rdev)
{ … }

#define TAHITI_IO_MC_REGS_SIZE …

static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = …;

static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = …;

static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = …;

static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = …;

static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = …;

/* ucode loading */
int si_mc_load_microcode(struct radeon_device *rdev)
{ … }

static int si_init_microcode(struct radeon_device *rdev)
{ … }

/* watermark setup */
static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
				   struct radeon_crtc *radeon_crtc,
				   struct drm_display_mode *mode,
				   struct drm_display_mode *other_mode)
{ … }

static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
{ … }

struct dce6_wm_params { … };

static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
{ … }

static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
{ … }

static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
{ … }

static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
{ … }

static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
{ … }

static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
{ … }

static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
{ … }

static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
{ … }

static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
{
	if (dce6_average_bandwidth(wm) <=
	    (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
		return true;
	else
		return false;
};

static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
{
	if (dce6_average_bandwidth(wm) <=
	    (dce6_available_bandwidth(wm) / wm->num_heads))
		return true;
	else
		return false;
};

static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
{ … }

static void dce6_program_watermarks(struct radeon_device *rdev,
					 struct radeon_crtc *radeon_crtc,
					 u32 lb_size, u32 num_heads)
{ … }

void dce6_bandwidth_update(struct radeon_device *rdev)
{ … }

/*
 * Core functions
 */
static void si_tiling_mode_table_init(struct radeon_device *rdev)
{ … }

static void si_select_se_sh(struct radeon_device *rdev,
			    u32 se_num, u32 sh_num)
{ … }

static u32 si_create_bitmask(u32 bit_width)
{ … }

static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
{ … }

static void si_setup_spi(struct radeon_device *rdev,
			 u32 se_num, u32 sh_per_se,
			 u32 cu_per_sh)
{ … }

static u32 si_get_rb_disabled(struct radeon_device *rdev,
			      u32 max_rb_num_per_se,
			      u32 sh_per_se)
{ … }

static void si_setup_rb(struct radeon_device *rdev,
			u32 se_num, u32 sh_per_se,
			u32 max_rb_num_per_se)
{ … }

static void si_gpu_init(struct radeon_device *rdev)
{ … }

/*
 * GPU scratch registers helpers function.
 */
static void si_scratch_init(struct radeon_device *rdev)
{ … }

void si_fence_ring_emit(struct radeon_device *rdev,
			struct radeon_fence *fence)
{ … }

/*
 * IB stuff
 */
void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
{ … }

/*
 * CP.
 */
static void si_cp_enable(struct radeon_device *rdev, bool enable)
{ … }

static int si_cp_load_microcode(struct radeon_device *rdev)
{ … }

static int si_cp_start(struct radeon_device *rdev)
{ … }

static void si_cp_fini(struct radeon_device *rdev)
{ … }

static int si_cp_resume(struct radeon_device *rdev)
{ … }

u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
{ … }

static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
{ … }

static void si_set_clk_bypass_mode(struct radeon_device *rdev)
{ … }

static void si_spll_powerdown(struct radeon_device *rdev)
{ … }

static void si_gpu_pci_config_reset(struct radeon_device *rdev)
{ … }

int si_asic_reset(struct radeon_device *rdev, bool hard)
{ … }

/**
 * si_gfx_is_lockup - Check if the GFX engine is locked up
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Check if the GFX engine is locked up.
 * Returns true if the engine appears to be locked up, false if not.
 */
bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
{ … }

/* MC */
static void si_mc_program(struct radeon_device *rdev)
{ … }

void si_vram_gtt_location(struct radeon_device *rdev,
			  struct radeon_mc *mc)
{ … }

static int si_mc_init(struct radeon_device *rdev)
{ … }

/*
 * GART
 */
void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
{ … }

static int si_pcie_gart_enable(struct radeon_device *rdev)
{ … }

static void si_pcie_gart_disable(struct radeon_device *rdev)
{ … }

static void si_pcie_gart_fini(struct radeon_device *rdev)
{ … }

/* vm parser */
static bool si_vm_reg_valid(u32 reg)
{ … }

static int si_vm_packet3_ce_check(struct radeon_device *rdev,
				  u32 *ib, struct radeon_cs_packet *pkt)
{ … }

static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
{ … }

static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
				   u32 *ib, struct radeon_cs_packet *pkt)
{ … }

static int si_vm_packet3_compute_check(struct radeon_device *rdev,
				       u32 *ib, struct radeon_cs_packet *pkt)
{ … }

int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
{ … }

/*
 * vm
 */
int si_vm_init(struct radeon_device *rdev)
{ … }

void si_vm_fini(struct radeon_device *rdev)
{ … }

/**
 * si_vm_decode_fault - print human readable fault info
 *
 * @rdev: radeon_device pointer
 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
 *
 * Print human readable fault information (SI).
 */
static void si_vm_decode_fault(struct radeon_device *rdev,
			       u32 status, u32 addr)
{ … }

void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
		 unsigned vm_id, uint64_t pd_addr)
{ … }

/*
 *  Power and clock gating
 */
static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
{ … }

static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
					 bool enable)
{ … }

static void si_set_uvd_dcm(struct radeon_device *rdev,
			   bool sw_mode)
{ … }

void si_init_uvd_internal_cg(struct radeon_device *rdev)
{ … }

static u32 si_halt_rlc(struct radeon_device *rdev)
{ … }

static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
{ … }

static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
{ … }

static void si_init_dma_pg(struct radeon_device *rdev)
{ … }

static void si_enable_gfx_cgpg(struct radeon_device *rdev,
			       bool enable)
{ … }

static void si_init_gfx_cgpg(struct radeon_device *rdev)
{ … }

static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
{ … }

static void si_init_ao_cu_mask(struct radeon_device *rdev)
{ … }

static void si_enable_cgcg(struct radeon_device *rdev,
			   bool enable)
{ … }

static void si_enable_mgcg(struct radeon_device *rdev,
			   bool enable)
{ … }

static void si_enable_uvd_mgcg(struct radeon_device *rdev,
			       bool enable)
{ … }

static const u32 mc_cg_registers[] = …;

static void si_enable_mc_ls(struct radeon_device *rdev,
			    bool enable)
{ … }

static void si_enable_mc_mgcg(struct radeon_device *rdev,
			       bool enable)
{ … }

static void si_enable_dma_mgcg(struct radeon_device *rdev,
			       bool enable)
{ … }

static void si_enable_bif_mgls(struct radeon_device *rdev,
			       bool enable)
{ … }

static void si_enable_hdp_mgcg(struct radeon_device *rdev,
			       bool enable)
{ … }

static void si_enable_hdp_ls(struct radeon_device *rdev,
			     bool enable)
{ … }

static void si_update_cg(struct radeon_device *rdev,
			 u32 block, bool enable)
{ … }

static void si_init_cg(struct radeon_device *rdev)
{ … }

static void si_fini_cg(struct radeon_device *rdev)
{ … }

u32 si_get_csb_size(struct radeon_device *rdev)
{ … }

void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
{ … }

static void si_init_pg(struct radeon_device *rdev)
{ … }

static void si_fini_pg(struct radeon_device *rdev)
{ … }

/*
 * RLC
 */
void si_rlc_reset(struct radeon_device *rdev)
{ … }

static void si_rlc_stop(struct radeon_device *rdev)
{ … }

static void si_rlc_start(struct radeon_device *rdev)
{ … }

static bool si_lbpw_supported(struct radeon_device *rdev)
{ … }

static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
{ … }

static int si_rlc_resume(struct radeon_device *rdev)
{ … }

static void si_enable_interrupts(struct radeon_device *rdev)
{ … }

static void si_disable_interrupts(struct radeon_device *rdev)
{ … }

static void si_disable_interrupt_state(struct radeon_device *rdev)
{ … }

static int si_irq_init(struct radeon_device *rdev)
{ … }

/* The order we write back each register here is important */
int si_irq_set(struct radeon_device *rdev)
{ … }

/* The order we write back each register here is important */
static inline void si_irq_ack(struct radeon_device *rdev)
{ … }

static void si_irq_disable(struct radeon_device *rdev)
{ … }

static void si_irq_suspend(struct radeon_device *rdev)
{ … }

static void si_irq_fini(struct radeon_device *rdev)
{ … }

static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
{ … }

/*        SI IV Ring
 * Each IV ring entry is 128 bits:
 * [7:0]    - interrupt source id
 * [31:8]   - reserved
 * [59:32]  - interrupt source data
 * [63:60]  - reserved
 * [71:64]  - RINGID
 * [79:72]  - VMID
 * [127:80] - reserved
 */
int si_irq_process(struct radeon_device *rdev)
{ … }

/*
 * startup/shutdown callbacks
 */
static void si_uvd_init(struct radeon_device *rdev)
{ … }

static void si_uvd_start(struct radeon_device *rdev)
{ … }

static void si_uvd_resume(struct radeon_device *rdev)
{ … }

static void si_vce_init(struct radeon_device *rdev)
{ … }

static void si_vce_start(struct radeon_device *rdev)
{ … }

static void si_vce_resume(struct radeon_device *rdev)
{ … }

static int si_startup(struct radeon_device *rdev)
{ … }

int si_resume(struct radeon_device *rdev)
{ … }

int si_suspend(struct radeon_device *rdev)
{ … }

/* Plan is to move initialization in that function and use
 * helper function so that radeon_device_init pretty much
 * do nothing more than calling asic specific function. This
 * should also allow to remove a bunch of callback function
 * like vram_info.
 */
int si_init(struct radeon_device *rdev)
{ … }

void si_fini(struct radeon_device *rdev)
{ … }

/**
 * si_get_gpu_clock_counter - return GPU clock counter snapshot
 *
 * @rdev: radeon_device pointer
 *
 * Fetches a GPU clock counter snapshot (SI).
 * Returns the 64 bit clock counter snapshot.
 */
uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
{ … }

int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
{ … }

static void si_pcie_gen3_enable(struct radeon_device *rdev)
{ … }

static void si_program_aspm(struct radeon_device *rdev)
{ … }

static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
{ … }

int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
{ … }