// SPDX-License-Identifier: MIT /* * Copyright © 2020 Intel Corporation */ #include <asm/set_memory.h> #include <asm/smp.h> #include <linux/types.h> #include <linux/stop_machine.h> #include <drm/drm_managed.h> #include <drm/intel/i915_drm.h> #include <drm/intel/intel-gtt.h> #include "gem/i915_gem_lmem.h" #include "intel_context.h" #include "intel_ggtt_gmch.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_regs.h" #include "intel_pci_config.h" #include "intel_ring.h" #include "i915_drv.h" #include "i915_pci.h" #include "i915_reg.h" #include "i915_request.h" #include "i915_scatterlist.h" #include "i915_utils.h" #include "i915_vgpu.h" #include "intel_gtt.h" #include "gen8_ppgtt.h" #include "intel_engine_pm.h" static void i915_ggtt_color_adjust(const struct drm_mm_node *node, unsigned long color, u64 *start, u64 *end) { … } static int ggtt_init_hw(struct i915_ggtt *ggtt) { … } /** * i915_ggtt_init_hw - Initialize GGTT hardware * @i915: i915 device */ int i915_ggtt_init_hw(struct drm_i915_private *i915) { … } /** * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM * @vm: The VM to suspend the mappings for * * Suspend the memory mappings for all objects mapped to HW via the GGTT or a * DPT page table. */ void i915_ggtt_suspend_vm(struct i915_address_space *vm) { … } void i915_ggtt_suspend(struct i915_ggtt *ggtt) { … } void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) { … } static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915) { … } static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) { … } static void guc_ggtt_ct_invalidate(struct intel_gt *gt) { … } static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) { … } static u64 mtl_ggtt_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } u64 gen8_ggtt_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static bool should_update_ggtt_with_bind(struct i915_ggtt *ggtt) { … } static struct intel_context *gen8_ggtt_bind_get_ce(struct i915_ggtt *ggtt, intel_wakeref_t *wakeref) { … } static void gen8_ggtt_bind_put_ce(struct intel_context *ce, intel_wakeref_t wakeref) { … } static bool gen8_ggtt_bind_ptes(struct i915_ggtt *ggtt, u32 offset, struct sg_table *pages, u32 num_entries, const gen8_pte_t pte) { … } static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) { … } static void gen8_ggtt_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, unsigned int pat_index, u32 flags) { … } static void gen8_ggtt_insert_page_bind(struct i915_address_space *vm, dma_addr_t addr, u64 offset, unsigned int pat_index, u32 flags) { … } static void gen8_ggtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static bool __gen8_ggtt_insert_entries_bind(struct i915_address_space *vm, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static void gen8_ggtt_insert_entries_bind(struct i915_address_space *vm, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static void gen8_ggtt_clear_range(struct i915_address_space *vm, u64 start, u64 length) { … } static void gen8_ggtt_scratch_range_bind(struct i915_address_space *vm, u64 start, u64 length) { … } static void gen6_ggtt_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, unsigned int pat_index, u32 flags) { … } /* * Binds an object into the global gtt with the specified cache level. * The object will be accessible to the GPU via commands whose operands * reference offsets within the global GTT as well as accessible by the GPU * through the GMADR mapped BAR (i915->mm.gtt->gtt). */ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static void nop_clear_range(struct i915_address_space *vm, u64 start, u64 length) { … } static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) { … } struct insert_page { … }; static int bxt_vtd_ggtt_insert_page__cb(void *_arg) { … } static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, dma_addr_t addr, u64 offset, unsigned int pat_index, u32 unused) { … } struct insert_entries { … }; static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) { … } static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static void gen6_ggtt_clear_range(struct i915_address_space *vm, u64 start, u64 length) { … } void intel_ggtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } void intel_ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res) { … } /* * Reserve the top of the GuC address space for firmware images. Addresses * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC, * which makes for a suitable range to hold GuC/HuC firmware images if the * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk * of the same size anyway, which is far more than needed, to keep the logic * in uc_fw_ggtt_offset() simple. */ #define GUC_TOP_RESERVE_SIZE … static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) { … } static void ggtt_release_guc_top(struct i915_ggtt *ggtt) { … } static void cleanup_init_ggtt(struct i915_ggtt *ggtt) { … } static int init_ggtt(struct i915_ggtt *ggtt) { … } static void aliasing_gtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags) { … } static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res) { … } static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) { … } static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) { … } int i915_init_ggtt(struct drm_i915_private *i915) { … } static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) { … } /** * i915_ggtt_driver_release - Clean up GGTT hardware initialization * @i915: i915 device */ void i915_ggtt_driver_release(struct drm_i915_private *i915) { … } /** * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after * all free objects have been drained. * @i915: i915 device */ void i915_ggtt_driver_late_release(struct drm_i915_private *i915) { … } static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) { … } static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) { … } static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) { … } static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) { … } static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) { … } static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) { … } static void gen6_gmch_remove(struct i915_address_space *vm) { … } static struct resource pci_resource(struct pci_dev *pdev, int bar) { … } static int gen8_gmch_probe(struct i915_ggtt *ggtt) { … } /* * For pre-gen8 platforms pat_index is the same as enum i915_cache_level, * so the switch-case statements in these PTE encode functions are still valid. * See translation table LEGACY_CACHELEVEL. */ static u64 snb_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static u64 ivb_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static u64 byt_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static u64 hsw_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static u64 iris_pte_encode(dma_addr_t addr, unsigned int pat_index, u32 flags) { … } static int gen6_gmch_probe(struct i915_ggtt *ggtt) { … } static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) { … } /** * i915_ggtt_probe_hw - Probe GGTT hardware location * @i915: i915 device */ int i915_ggtt_probe_hw(struct drm_i915_private *i915) { … } struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915) { … } int i915_ggtt_enable_hw(struct drm_i915_private *i915) { … } /** * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM * @vm: The VM to restore the mappings for * * Restore the memory mappings for all objects mapped to HW via the GGTT or a * DPT page table. * * Returns %true if restoring the mapping for any object that was in a write * domain before suspend. */ bool i915_ggtt_resume_vm(struct i915_address_space *vm) { … } void i915_ggtt_resume(struct i915_ggtt *ggtt) { … }
Codebase Browser
linux