// SPDX-License-Identifier: MIT /* * Copyright © 2014-2018 Intel Corporation */ #include "i915_drv.h" #include "i915_reg.h" #include "intel_context.h" #include "intel_engine_pm.h" #include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_ccs_mode.h" #include "intel_gt_mcr.h" #include "intel_gt_print.h" #include "intel_gt_regs.h" #include "intel_ring.h" #include "intel_workarounds.h" #include "display/intel_fbc_regs.h" /** * DOC: Hardware workarounds * * Hardware workarounds are register programming documented to be executed in * the driver that fall outside of the normal programming sequences for a * platform. There are some basic categories of workarounds, depending on * how/when they are applied: * * - Context workarounds: workarounds that touch registers that are * saved/restored to/from the HW context image. The list is emitted (via Load * Register Immediate commands) once when initializing the device and saved in * the default context. That default context is then used on every context * creation to have a "primed golden context", i.e. a context image that * already contains the changes needed to all the registers. * * Context workarounds should be implemented in the \*_ctx_workarounds_init() * variants respective to the targeted platforms. * * - Engine workarounds: the list of these WAs is applied whenever the specific * engine is reset. It's also possible that a set of engine classes share a * common power domain and they are reset together. This happens on some * platforms with render and compute engines. In this case (at least) one of * them need to keeep the workaround programming: the approach taken in the * driver is to tie those workarounds to the first compute/render engine that * is registered. When executing with GuC submission, engine resets are * outside of kernel driver control, hence the list of registers involved in * written once, on engine initialization, and then passed to GuC, that * saves/restores their values before/after the reset takes place. See * ``drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c`` for reference. * * Workarounds for registers specific to RCS and CCS should be implemented in * rcs_engine_wa_init() and ccs_engine_wa_init(), respectively; those for * registers belonging to BCS, VCS or VECS should be implemented in * xcs_engine_wa_init(). Workarounds for registers not belonging to a specific * engine's MMIO range but that are part of of the common RCS/CCS reset domain * should be implemented in general_render_compute_wa_init(). The settings * about the CCS load balancing should be added in ccs_engine_wa_mode(). * * - GT workarounds: the list of these WAs is applied whenever these registers * revert to their default values: on GPU reset, suspend/resume [1]_, etc. * * GT workarounds should be implemented in the \*_gt_workarounds_init() * variants respective to the targeted platforms. * * - Register whitelist: some workarounds need to be implemented in userspace, * but need to touch privileged registers. The whitelist in the kernel * instructs the hardware to allow the access to happen. From the kernel side, * this is just a special case of a MMIO workaround (as we write the list of * these to/be-whitelisted registers to some special HW registers). * * Register whitelisting should be done in the \*_whitelist_build() variants * respective to the targeted platforms. * * - Workaround batchbuffers: buffers that get executed automatically by the * hardware on every HW context restore. These buffers are created and * programmed in the default context so the hardware always go through those * programming sequences when switching contexts. The support for workaround * batchbuffers is enabled these hardware mechanisms: * * #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default * context, pointing the hardware to jump to that location when that offset * is reached in the context restore. Workaround batchbuffer in the driver * currently uses this mechanism for all platforms. * * #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context, * pointing the hardware to a buffer to continue executing after the * engine registers are restored in a context restore sequence. This is * currently not used in the driver. * * - Other: There are WAs that, due to their nature, cannot be applied from a * central place. Those are peppered around the rest of the code, as needed. * Workarounds related to the display IP are the main example. * * .. [1] Technically, some registers are powercontext saved & restored, so they * survive a suspend/resume. In practice, writing them again is not too * costly and simplifies things, so it's the approach taken in the driver. */ static void wa_init_start(struct i915_wa_list *wal, struct intel_gt *gt, const char *name, const char *engine_name) { … } #define WA_LIST_CHUNK … static void wa_init_finish(struct i915_wa_list *wal) { … } static enum forcewake_domains wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) { … } static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) { … } static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set, u32 read_mask, bool masked_reg) { … } static void wa_mcr_add(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clear, u32 set, u32 read_mask, bool masked_reg) { … } static void wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) { … } static void wa_mcr_write_clr_set(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clear, u32 set) { … } static void wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set) { … } static void wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) { … } static void wa_mcr_write_or(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set) { … } static void wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) { … } static void wa_mcr_write_clr(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clr) { … } /* * WA operations on "masked register". A masked register has the upper 16 bits * documented as "masked" in b-spec. Its purpose is to allow writing to just a * portion of the register without a rmw: you simply write in the upper 16 bits * the mask of bits you are going to modify. * * The wa_masked_* family of functions already does the necessary operations to * calculate the mask based on the parameters passed, so user only has to * provide the lower 16 bits of that register. */ static void wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) { … } static void wa_mcr_masked_en(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val) { … } static void wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) { … } static void wa_mcr_masked_dis(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val) { … } static void wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, u32 val) { … } static void wa_mcr_masked_field_set(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 mask, u32 val) { … } static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void chv_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void skl_tune_iz_hashing(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void skl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void glk_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } /* * These settings aren't actually workarounds, but general tuning settings that * need to be programmed on dg2 platform. */ static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void xelpg_ctx_gt_tuning_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void xelpg_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } /* * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround * defined by the hardware team, but it programming general context registers. * Adding those context register programming in context workaround * allow us to use the wa framework for proper application and validation. */ static void gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, struct i915_wa_list *wal, const char *name) { … } void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) { … } int intel_engine_emit_ctx_wa(struct i915_request *rq) { … } static void gen4_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) { … } static void gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void __set_mcr_steering(struct i915_wa_list *wal, i915_reg_t steering_reg, unsigned int slice, unsigned int subslice) { … } static void debug_dump_steering(struct intel_gt *gt) { … } static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal, unsigned int slice, unsigned int subslice) { … } static void icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } /* * Though there are per-engine instances of these registers, * they retain their value through engine resets and should * only be provided on the GT workaround list rather than * the engine-specific workaround list. */ static void wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void wa_16021867713(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { … } /* * The bspec performance guide has recommended MMIO tuning settings. These * aren't truly "workarounds" but we want to program them through the * workaround infrastructure to make sure they're (re)applied at the proper * times. * * The programming in this function is for settings that persist through * engine resets and also are not part of any engine's register state context. * I.e., settings that only need to be re-applied in the event of a full GT * reset. */ static void gt_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal) { … } void intel_gt_init_workarounds(struct intel_gt *gt) { … } static bool wa_verify(struct intel_gt *gt, const struct i915_wa *wa, u32 cur, const char *name, const char *from) { … } static void wa_list_apply(const struct i915_wa_list *wal) { … } void intel_gt_apply_workarounds(struct intel_gt *gt) { … } static bool wa_list_verify(struct intel_gt *gt, const struct i915_wa_list *wal, const char *from) { … } bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from) { … } __maybe_unused static bool is_nonpriv_flags_valid(u32 flags) { … } static void whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags) { … } static void whitelist_mcr_reg_ext(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 flags) { … } static void whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) { … } static void whitelist_mcr_reg(struct i915_wa_list *wal, i915_mcr_reg_t reg) { … } static void gen9_whitelist_build(struct i915_wa_list *w) { … } static void skl_whitelist_build(struct intel_engine_cs *engine) { … } static void bxt_whitelist_build(struct intel_engine_cs *engine) { … } static void kbl_whitelist_build(struct intel_engine_cs *engine) { … } static void glk_whitelist_build(struct intel_engine_cs *engine) { … } static void cfl_whitelist_build(struct intel_engine_cs *engine) { … } static void allow_read_ctx_timestamp(struct intel_engine_cs *engine) { … } static void cml_whitelist_build(struct intel_engine_cs *engine) { … } static void icl_whitelist_build(struct intel_engine_cs *engine) { … } static void tgl_whitelist_build(struct intel_engine_cs *engine) { … } static void dg2_whitelist_build(struct intel_engine_cs *engine) { … } static void xelpg_whitelist_build(struct intel_engine_cs *engine) { … } void intel_engine_init_whitelist(struct intel_engine_cs *engine) { … } void intel_engine_apply_whitelist(struct intel_engine_cs *engine) { … } /* * engine_fake_wa_init(), a place holder to program the registers * which are not part of an official workaround defined by the * hardware team. * Adding programming of those register inside workaround will * allow utilizing wa framework to proper application and verification. */ static void engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } /* * The bspec performance guide has recommended MMIO tuning settings. These * aren't truly "workarounds" but we want to program them with the same * workaround infrastructure to ensure that they're automatically added to * the GuC save/restore lists, re-applied at the right times, and checked for * any conflicting programming requested by real workarounds. * * Programming settings should be added here only if their registers are not * part of an engine's register state context. If a register is part of a * context, then any tuning settings should be programmed in an appropriate * function invoked by __intel_engine_init_ctx_wa(). */ static void add_render_compute_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal) { … } static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } /* * The workarounds in this function apply to shared registers in * the general render reset domain that aren't tied to a * specific engine. Since all render+compute engines get reset * together, and the contents of these registers are lost during * the shared render domain reset, we'll define such workarounds * here and then add them to just a single RCS or CCS engine's * workaround list (whichever engine has the XXXX flag). */ static void general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } static void engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) { … } void intel_engine_init_workarounds(struct intel_engine_cs *engine) { … } void intel_engine_apply_workarounds(struct intel_engine_cs *engine) { … } static const struct i915_range mcr_ranges_gen8[] = …; static const struct i915_range mcr_ranges_gen12[] = …; static const struct i915_range mcr_ranges_xehp[] = …; static bool mcr_range(struct drm_i915_private *i915, u32 offset) { … } static int wa_list_srm(struct i915_request *rq, const struct i915_wa_list *wal, struct i915_vma *vma) { … } static int engine_wa_list_verify(struct intel_context *ce, const struct i915_wa_list * const wal, const char *from) { … } int intel_engine_verify_workarounds(struct intel_engine_cs *engine, const char *from) { … } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_workarounds.c" #endif
Codebase Browser
linux