linux/drivers/gpu/drm/i915/gt/gen8_engine_cs.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2014 Intel Corporation
 */

#include "gen8_engine_cs.h"
#include "intel_engine_regs.h"
#include "intel_gpu_commands.h"
#include "intel_gt.h"
#include "intel_lrc.h"
#include "intel_ring.h"

int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode)
{ … }

int gen8_emit_flush_xcs(struct i915_request *rq, u32 mode)
{ … }

int gen11_emit_flush_rcs(struct i915_request *rq, u32 mode)
{ … }

static u32 preparser_disable(bool state)
{ … }

static i915_reg_t gen12_get_aux_inv_reg(struct intel_engine_cs *engine)
{ … }

static bool gen12_needs_ccs_aux_inv(struct intel_engine_cs *engine)
{ … }

u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)
{ … }

static int mtl_dummy_pipe_control(struct i915_request *rq)
{ … }

int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
{ … }

int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode)
{ … }

static u32 preempt_address(struct intel_engine_cs *engine)
{ … }

static u32 hwsp_offset(const struct i915_request *rq)
{ … }

int gen8_emit_init_breadcrumb(struct i915_request *rq)
{ … }

static int __xehp_emit_bb_start(struct i915_request *rq,
				u64 offset, u32 len,
				const unsigned int flags,
				u32 arb)
{ … }

int xehp_emit_bb_start_noarb(struct i915_request *rq,
			     u64 offset, u32 len,
			     const unsigned int flags)
{ … }

int xehp_emit_bb_start(struct i915_request *rq,
		       u64 offset, u32 len,
		       const unsigned int flags)
{ … }

int gen8_emit_bb_start_noarb(struct i915_request *rq,
			     u64 offset, u32 len,
			     const unsigned int flags)
{ … }

int gen8_emit_bb_start(struct i915_request *rq,
		       u64 offset, u32 len,
		       const unsigned int flags)
{ … }

static void assert_request_valid(struct i915_request *rq)
{ … }

/*
 * Reserve space for 2 NOOPs at the end of each request to be
 * used as a workaround for not being allowed to do lite
 * restore with HEAD==TAIL (WaIdleLiteRestore).
 */
static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)
{ … }

static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)
{ … }

static __always_inline u32*
gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)
{ … }

static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)
{ … }

u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)
{ … }

u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
{ … }

u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
{ … }

/*
 * Note that the CS instruction pre-parser will not stall on the breadcrumb
 * flush and will continue pre-fetching the instructions after it before the
 * memory sync is completed. On pre-gen12 HW, the pre-parser will stop at
 * BB_START/END instructions, so, even though we might pre-fetch the pre-amble
 * of the next request before the memory has been flushed, we're guaranteed that
 * we won't access the batch itself too early.
 * However, on gen12+ the parser can pre-fetch across the BB_START/END commands,
 * so, if the current request is modifying an instruction in the next request on
 * the same intel_context, we might pre-fetch and then execute the pre-update
 * instruction. To avoid this, the users of self-modifying code should either
 * disable the parser around the code emitting the memory writes, via a new flag
 * added to MI_ARB_CHECK, or emit the writes from a different intel_context. For
 * the in-kernel use-cases we've opted to use a separate context, see
 * reloc_gpu() as an example.
 * All the above applies only to the instructions themselves. Non-inline data
 * used by the instructions is not pre-fetched.
 */

static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)
{ … }

/* Wa_14014475959:dg2 */
/* Wa_16019325821 */
/* Wa_14019159160 */
#define HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET …
static u32 hold_switchout_semaphore_offset(struct i915_request *rq)
{ … }

/* Wa_14014475959:dg2 */
/* Wa_16019325821 */
/* Wa_14019159160 */
static u32 *hold_switchout_emit_wa_busywait(struct i915_request *rq, u32 *cs)
{ … }

static __always_inline u32*
gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)
{ … }

u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)
{ … }

u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
{ … }