linux/drivers/gpu/drm/radeon/r600.c

/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */

#include <linux/debugfs.h>
#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/seq_file.h>
#include <linux/slab.h>

#include <drm/drm_device.h>
#include <drm/drm_vblank.h>
#include <drm/radeon_drm.h>

#include "atom.h"
#include "avivod.h"
#include "evergreen.h"
#include "r600.h"
#include "r600d.h"
#include "rv770.h"
#include "radeon.h"
#include "radeon_asic.h"
#include "radeon_audio.h"
#include "radeon_mode.h"
#include "radeon_ucode.h"

/* Firmware Names */
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;
MODULE_FIRMWARE(…) …;

static const u32 crtc_offsets[2] = …;

static void r600_debugfs_mc_info_init(struct radeon_device *rdev);

/* r600,rv610,rv630,rv620,rv635,rv670 */
int r600_mc_wait_for_idle(struct radeon_device *rdev);
static void r600_gpu_init(struct radeon_device *rdev);
void r600_fini(struct radeon_device *rdev);
void r600_irq_disable(struct radeon_device *rdev);
static void r600_pcie_gen2_enable(struct radeon_device *rdev);

/*
 * Indirect registers accessor
 */
u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg)
{ … }

void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v)
{ … }

u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg)
{ … }

void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v)
{ … }

/**
 * r600_get_allowed_info_register - fetch the register for the info ioctl
 *
 * @rdev: radeon_device pointer
 * @reg: register offset in bytes
 * @val: register value
 *
 * Returns 0 for success or -EINVAL for an invalid register
 *
 */
int r600_get_allowed_info_register(struct radeon_device *rdev,
				   u32 reg, u32 *val)
{ … }

/**
 * r600_get_xclk - get the xclk
 *
 * @rdev: radeon_device pointer
 *
 * Returns the reference clock used by the gfx engine
 * (r6xx, IGPs, APUs).
 */
u32 r600_get_xclk(struct radeon_device *rdev)
{ … }

int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
{ … }

void dce3_program_fmt(struct drm_encoder *encoder)
{ … }

/* get temperature in millidegrees */
int rv6xx_get_temp(struct radeon_device *rdev)
{ … }

void r600_pm_get_dynpm_state(struct radeon_device *rdev)
{ … }

void rs780_pm_init_profile(struct radeon_device *rdev)
{ … }

void r600_pm_init_profile(struct radeon_device *rdev)
{ … }

void r600_pm_misc(struct radeon_device *rdev)
{ … }

bool r600_gui_idle(struct radeon_device *rdev)
{ … }

/* hpd for digital panel detect/disconnect */
bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
{ … }

void r600_hpd_set_polarity(struct radeon_device *rdev,
			   enum radeon_hpd_id hpd)
{ … }

void r600_hpd_init(struct radeon_device *rdev)
{ … }

void r600_hpd_fini(struct radeon_device *rdev)
{ … }

/*
 * R600 PCIE GART
 */
void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
{ … }

int r600_pcie_gart_init(struct radeon_device *rdev)
{ … }

static int r600_pcie_gart_enable(struct radeon_device *rdev)
{ … }

static void r600_pcie_gart_disable(struct radeon_device *rdev)
{ … }

static void r600_pcie_gart_fini(struct radeon_device *rdev)
{ … }

static void r600_agp_enable(struct radeon_device *rdev)
{ … }

int r600_mc_wait_for_idle(struct radeon_device *rdev)
{ … }

uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
{ … }

void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
{ … }

static void r600_mc_program(struct radeon_device *rdev)
{ … }

/**
 * r600_vram_gtt_location - try to find VRAM & GTT location
 * @rdev: radeon device structure holding all necessary informations
 * @mc: memory controller structure holding memory informations
 *
 * Function will place try to place VRAM at same place as in CPU (PCI)
 * address space as some GPU seems to have issue when we reprogram at
 * different address space.
 *
 * If there is not enough space to fit the unvisible VRAM after the
 * aperture then we limit the VRAM size to the aperture.
 *
 * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 * them to be in one from GPU point of view so that we can program GPU to
 * catch access outside them (weird GPU policy see ??).
 *
 * This function will never fails, worst case are limiting VRAM or GTT.
 *
 * Note: GTT start, end, size should be initialized before calling this
 * function on AGP platform.
 */
static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
{ … }

static int r600_mc_init(struct radeon_device *rdev)
{ … }

int r600_vram_scratch_init(struct radeon_device *rdev)
{ … }

void r600_vram_scratch_fini(struct radeon_device *rdev)
{ … }

void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
{ … }

static void r600_print_gpu_status_regs(struct radeon_device *rdev)
{ … }

static bool r600_is_display_hung(struct radeon_device *rdev)
{ … }

u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
{ … }

static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
{ … }

static void r600_gpu_pci_config_reset(struct radeon_device *rdev)
{ … }

int r600_asic_reset(struct radeon_device *rdev, bool hard)
{ … }

/**
 * r600_gfx_is_lockup - Check if the GFX engine is locked up
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Check if the GFX engine is locked up.
 * Returns true if the engine appears to be locked up, false if not.
 */
bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
{ … }

u32 r6xx_remap_render_backend(struct radeon_device *rdev,
			      u32 tiling_pipe_num,
			      u32 max_rb_num,
			      u32 total_max_rb_num,
			      u32 disabled_rb_mask)
{ … }

int r600_count_pipe_bits(uint32_t val)
{ … }

static void r600_gpu_init(struct radeon_device *rdev)
{ … }


/*
 * Indirect registers accessor
 */
u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
{ … }

void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
{ … }

/*
 * CP & Ring
 */
void r600_cp_stop(struct radeon_device *rdev)
{ … }

int r600_init_microcode(struct radeon_device *rdev)
{ … }

u32 r600_gfx_get_rptr(struct radeon_device *rdev,
		      struct radeon_ring *ring)
{ … }

u32 r600_gfx_get_wptr(struct radeon_device *rdev,
		      struct radeon_ring *ring)
{ … }

void r600_gfx_set_wptr(struct radeon_device *rdev,
		       struct radeon_ring *ring)
{ … }

static int r600_cp_load_microcode(struct radeon_device *rdev)
{ … }

int r600_cp_start(struct radeon_device *rdev)
{ … }

int r600_cp_resume(struct radeon_device *rdev)
{ … }

void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
{ … }

void r600_cp_fini(struct radeon_device *rdev)
{ … }

/*
 * GPU scratch registers helpers function.
 */
void r600_scratch_init(struct radeon_device *rdev)
{ … }

int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
{ … }

/*
 * CP fences/semaphores
 */

void r600_fence_ring_emit(struct radeon_device *rdev,
			  struct radeon_fence *fence)
{ … }

/**
 * r600_semaphore_ring_emit - emit a semaphore on the CP ring
 *
 * @rdev: radeon_device pointer
 * @ring: radeon ring buffer object
 * @semaphore: radeon semaphore object
 * @emit_wait: Is this a semaphore wait?
 *
 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
 * from running ahead of semaphore waits.
 */
bool r600_semaphore_ring_emit(struct radeon_device *rdev,
			      struct radeon_ring *ring,
			      struct radeon_semaphore *semaphore,
			      bool emit_wait)
{ … }

/**
 * r600_copy_cpdma - copy pages using the CP DMA engine
 *
 * @rdev: radeon_device pointer
 * @src_offset: src GPU address
 * @dst_offset: dst GPU address
 * @num_gpu_pages: number of GPU pages to xfer
 * @resv: DMA reservation object to manage fences
 *
 * Copy GPU paging using the CP DMA engine (r6xx+).
 * Used by the radeon ttm implementation to move pages if
 * registered as the asic copy callback.
 */
struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev,
				     uint64_t src_offset, uint64_t dst_offset,
				     unsigned num_gpu_pages,
				     struct dma_resv *resv)
{ … }

int r600_set_surface_reg(struct radeon_device *rdev, int reg,
			 uint32_t tiling_flags, uint32_t pitch,
			 uint32_t offset, uint32_t obj_size)
{ … }

void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
{ … }

static void r600_uvd_init(struct radeon_device *rdev)
{ … }

static void r600_uvd_start(struct radeon_device *rdev)
{ … }

static void r600_uvd_resume(struct radeon_device *rdev)
{ … }

static int r600_startup(struct radeon_device *rdev)
{ … }

void r600_vga_set_state(struct radeon_device *rdev, bool state)
{ … }

int r600_resume(struct radeon_device *rdev)
{ … }

int r600_suspend(struct radeon_device *rdev)
{ … }

/* Plan is to move initialization in that function and use
 * helper function so that radeon_device_init pretty much
 * do nothing more than calling asic specific function. This
 * should also allow to remove a bunch of callback function
 * like vram_info.
 */
int r600_init(struct radeon_device *rdev)
{ … }

void r600_fini(struct radeon_device *rdev)
{ … }


/*
 * CS stuff
 */
void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
{ … }

int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
{ … }

/*
 * Interrupts
 *
 * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
 * the same as the CP ring buffer, but in reverse.  Rather than the CPU
 * writing to the ring and the GPU consuming, the GPU writes to the ring
 * and host consumes.  As the host irq handler processes interrupts, it
 * increments the rptr.  When the rptr catches up with the wptr, all the
 * current interrupts have been processed.
 */

void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
{ … }

int r600_ih_ring_alloc(struct radeon_device *rdev)
{ … }

void r600_ih_ring_fini(struct radeon_device *rdev)
{ … }

void r600_rlc_stop(struct radeon_device *rdev)
{ … }

static void r600_rlc_start(struct radeon_device *rdev)
{ … }

static int r600_rlc_resume(struct radeon_device *rdev)
{ … }

static void r600_enable_interrupts(struct radeon_device *rdev)
{ … }

void r600_disable_interrupts(struct radeon_device *rdev)
{ … }

static void r600_disable_interrupt_state(struct radeon_device *rdev)
{ … }

int r600_irq_init(struct radeon_device *rdev)
{ … }

void r600_irq_suspend(struct radeon_device *rdev)
{ … }

void r600_irq_fini(struct radeon_device *rdev)
{ … }

int r600_irq_set(struct radeon_device *rdev)
{ … }

static void r600_irq_ack(struct radeon_device *rdev)
{ … }

void r600_irq_disable(struct radeon_device *rdev)
{ … }

static u32 r600_get_ih_wptr(struct radeon_device *rdev)
{ … }

/*        r600 IV Ring
 * Each IV ring entry is 128 bits:
 * [7:0]    - interrupt source id
 * [31:8]   - reserved
 * [59:32]  - interrupt source data
 * [127:60]  - reserved
 *
 * The basic interrupt vector entries
 * are decoded as follows:
 * src_id  src_data  description
 *      1         0  D1 Vblank
 *      1         1  D1 Vline
 *      5         0  D2 Vblank
 *      5         1  D2 Vline
 *     19         0  FP Hot plug detection A
 *     19         1  FP Hot plug detection B
 *     19         2  DAC A auto-detection
 *     19         3  DAC B auto-detection
 *     21         4  HDMI block A
 *     21         5  HDMI block B
 *    176         -  CP_INT RB
 *    177         -  CP_INT IB1
 *    178         -  CP_INT IB2
 *    181         -  EOP Interrupt
 *    233         -  GUI Idle
 *
 * Note, these are based on r600 and may need to be
 * adjusted or added to on newer asics
 */

int r600_irq_process(struct radeon_device *rdev)
{ … }

/*
 * Debugfs info
 */
#if defined(CONFIG_DEBUG_FS)

static int r600_debugfs_mc_info_show(struct seq_file *m, void *unused)
{ … }

DEFINE_SHOW_ATTRIBUTE(…);
#endif

static void r600_debugfs_mc_info_init(struct radeon_device *rdev)
{ … }

/**
 * r600_mmio_hdp_flush - flush Host Data Path cache via MMIO
 * @rdev: radeon device structure
 *
 * Some R6XX/R7XX don't seem to take into account HDP flushes performed
 * through the ring buffer. This leads to corruption in rendering, see
 * http://bugzilla.kernel.org/show_bug.cgi?id=15186 . To avoid this, we
 * directly perform the HDP flush by writing the register through MMIO.
 */
void r600_mmio_hdp_flush(struct radeon_device *rdev)
{ … }

void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
{ … }

int r600_get_pcie_lanes(struct radeon_device *rdev)
{ … }

static void r600_pcie_gen2_enable(struct radeon_device *rdev)
{ … }

/**
 * r600_get_gpu_clock_counter - return GPU clock counter snapshot
 *
 * @rdev: radeon_device pointer
 *
 * Fetches a GPU clock counter snapshot (R6xx-cayman).
 * Returns the 64 bit clock counter snapshot.
 */
uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
{ … }