/* SPDX-License-Identifier: MIT */ /* * Copyright © 2020 Intel Corporation * * Please try to maintain the following order within this file unless it makes * sense to do otherwise. From top to bottom: * 1. typedefs * 2. #defines, and macros * 3. structure definitions * 4. function prototypes * * Within each section, please try to order by generation in ascending order, * from top to bottom (ie. gen6 on the top, gen8 on the bottom). */ #ifndef __INTEL_GTT_H__ #define __INTEL_GTT_H__ #include <linux/io-mapping.h> #include <linux/kref.h> #include <linux/mm.h> #include <linux/pagevec.h> #include <linux/scatterlist.h> #include <linux/workqueue.h> #include <drm/drm_mm.h> #include "gt/intel_reset.h" #include "i915_selftest.h" #include "i915_vma_resource.h" #include "i915_vma_types.h" #include "i915_params.h" #include "intel_memory_region.h" #define I915_GFP_ALLOW_FAIL … #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GTT) #define GTT_TRACE … #else #define GTT_TRACE(...) … #endif #define NALLOC … #define I915_GTT_PAGE_SIZE_4K … #define I915_GTT_PAGE_SIZE_64K … #define I915_GTT_PAGE_SIZE_2M … #define I915_GTT_PAGE_SIZE … #define I915_GTT_MAX_PAGE_SIZE … #define I915_GTT_PAGE_MASK … #define I915_GTT_MIN_ALIGNMENT … #define I915_FENCE_REG_NONE … #define I915_MAX_NUM_FENCES … /* 32 fences + sign bit for FENCE_REG_NONE */ #define I915_MAX_NUM_FENCE_BITS … gen6_pte_t; gen8_pte_t; #define ggtt_total_entries(ggtt) … #define I915_PTES(pte_len) … #define I915_PTE_MASK(pte_len) … #define I915_PDES … #define I915_PDE_MASK … /* gen6-hsw has bit 11-4 for physical addr bit 39-32 */ #define GEN6_GTT_ADDR_ENCODE(addr) … #define GEN6_PTE_ADDR_ENCODE(addr) … #define GEN6_PDE_ADDR_ENCODE(addr) … #define GEN6_PTE_CACHE_LLC … #define GEN6_PTE_UNCACHED … #define GEN6_PTE_VALID … #define GEN6_PTES … #define GEN6_PD_SIZE … #define GEN6_PD_ALIGN … #define GEN6_PDE_SHIFT … #define GEN6_PDE_VALID … #define NUM_PTE(pde_shift) … #define GEN7_PTE_CACHE_L3_LLC … #define BYT_PTE_SNOOPED_BY_CPU_CACHES … #define BYT_PTE_WRITEABLE … #define MTL_PPGTT_PTE_PAT3 … #define GEN12_PPGTT_PTE_LM … #define GEN12_PPGTT_PTE_PAT2 … #define GEN12_PPGTT_PTE_PAT1 … #define GEN12_PPGTT_PTE_PAT0 … #define GEN12_GGTT_PTE_LM … #define MTL_GGTT_PTE_PAT0 … #define MTL_GGTT_PTE_PAT1 … #define GEN12_GGTT_PTE_ADDR_MASK … #define MTL_GGTT_PTE_PAT_MASK … #define GEN12_PDE_64K … #define GEN12_PTE_PS64 … /* * Cacheability Control is a 4-bit value. The low three bits are stored in bits * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. */ #define HSW_CACHEABILITY_CONTROL(bits) … #define HSW_WB_LLC_AGE3 … #define HSW_WB_LLC_AGE0 … #define HSW_WB_ELLC_LLC_AGE3 … #define HSW_WB_ELLC_LLC_AGE0 … #define HSW_WT_ELLC_LLC_AGE3 … #define HSW_WT_ELLC_LLC_AGE0 … #define HSW_PTE_UNCACHED … #define HSW_GTT_ADDR_ENCODE(addr) … #define HSW_PTE_ADDR_ENCODE(addr) … /* * GEN8 32b style address is defined as a 3 level page table: * 31:30 | 29:21 | 20:12 | 11:0 * PDPE | PDE | PTE | offset * The difference as compared to normal x86 3 level page table is the PDPEs are * programmed via register. * * GEN8 48b style address is defined as a 4 level page table: * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 * PML4E | PDPE | PDE | PTE | offset */ #define GEN8_3LVL_PDPES … #define PPAT_UNCACHED … #define PPAT_CACHED_PDE … #define PPAT_CACHED … #define PPAT_DISPLAY_ELLC … #define CHV_PPAT_SNOOP … #define GEN8_PPAT_AGE(x) … #define GEN8_PPAT_LLCeLLC … #define GEN8_PPAT_LLCELLC … #define GEN8_PPAT_LLC … #define GEN8_PPAT_WB … #define GEN8_PPAT_WT … #define GEN8_PPAT_WC … #define GEN8_PPAT_UC … #define GEN8_PPAT_ELLC_OVERRIDE … #define GEN8_PPAT(i, x) … #define GEN8_PAGE_PRESENT … #define GEN8_PAGE_RW … #define GEN8_PDE_IPS_64K … #define GEN8_PDE_PS_2M … #define MTL_PPAT_L4_CACHE_POLICY_MASK … #define MTL_PAT_INDEX_COH_MODE_MASK … #define MTL_PPAT_L4_3_UC … #define MTL_PPAT_L4_1_WT … #define MTL_PPAT_L4_0_WB … #define MTL_3_COH_2W … #define MTL_2_COH_1W … struct drm_i915_gem_object; struct i915_fence_reg; struct i915_vma; struct intel_gt; #define for_each_sgt_daddr(__dp, __iter, __sgt) … #define for_each_sgt_daddr_next(__dp, __iter) … struct i915_page_table { … }; struct i915_page_directory { … }; #define __px_choose_expr(x, type, expr, other) … #define px_base(px) … struct page *__px_page(struct drm_i915_gem_object *p); dma_addr_t __px_dma(struct drm_i915_gem_object *p); #define px_dma(px) … void *__px_vaddr(struct drm_i915_gem_object *p); #define px_vaddr(px) … #define px_pt(px) … #define px_used(px) … struct i915_vm_pt_stash { … }; struct i915_vma_ops { … }; struct i915_address_space { … }; /* * The Graphics Translation Table is the way in which GEN hardware translates a * Graphics Virtual Address into a Physical Address. In addition to the normal * collateral associated with any va->pa translations GEN hardware also has a * portion of the GTT which can be mapped by the CPU and remain both coherent * and correct (in cases like swizzling). That region is referred to as GMADR in * the spec. */ struct i915_ggtt { … }; struct i915_ppgtt { … }; #define i915_is_ggtt(vm) … #define i915_is_dpt(vm) … #define i915_is_ggtt_or_dpt(vm) … bool intel_vm_no_concurrent_access_wa(struct drm_i915_private *i915); int __must_check i915_vm_lock_objects(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww); static inline bool i915_vm_is_4lvl(const struct i915_address_space *vm) { … } static inline bool i915_vm_has_scratch_64K(struct i915_address_space *vm) { … } static inline u64 i915_vm_min_alignment(struct i915_address_space *vm, enum intel_memory_type type) { … } static inline u64 i915_vm_obj_min_alignment(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { … } static inline bool i915_vm_has_cache_coloring(struct i915_address_space *vm) { … } static inline struct i915_ggtt * i915_vm_to_ggtt(struct i915_address_space *vm) { … } static inline struct i915_ppgtt * i915_vm_to_ppgtt(struct i915_address_space *vm) { … } static inline struct i915_address_space * i915_vm_get(struct i915_address_space *vm) { … } static inline struct i915_address_space * i915_vm_tryget(struct i915_address_space *vm) { … } static inline void assert_vm_alive(struct i915_address_space *vm) { … } /** * i915_vm_resv_get - Obtain a reference on the vm's reservation lock * @vm: The vm whose reservation lock we want to share. * * Return: A pointer to the vm's reservation lock. */ static inline struct dma_resv *i915_vm_resv_get(struct i915_address_space *vm) { … } void i915_vm_release(struct kref *kref); void i915_vm_resv_release(struct kref *kref); static inline void i915_vm_put(struct i915_address_space *vm) { … } /** * i915_vm_resv_put - Release a reference on the vm's reservation lock * @vm: The vm whose reservation lock reference we want to release */ static inline void i915_vm_resv_put(struct i915_address_space *vm) { … } void i915_address_space_init(struct i915_address_space *vm, int subclass); void i915_address_space_fini(struct i915_address_space *vm); static inline u32 i915_pte_index(u64 address, unsigned int pde_shift) { … } /* * Helper to counts the number of PTEs within the given length. This count * does not cross a page table boundary, so the max value would be * GEN6_PTES for GEN6, and GEN8_PTES for GEN8. */ static inline u32 i915_pte_count(u64 addr, u64 length, unsigned int pde_shift) { … } static inline u32 i915_pde_index(u64 addr, u32 shift) { … } static inline struct i915_page_table * i915_pt_entry(const struct i915_page_directory * const pd, const unsigned short n) { … } static inline struct i915_page_directory * i915_pd_entry(const struct i915_page_directory * const pdp, const unsigned short n) { … } static inline dma_addr_t i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) { … } void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt, unsigned long lmem_pt_obj_flags); void intel_ggtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags); void intel_ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res); int i915_ggtt_probe_hw(struct drm_i915_private *i915); int i915_ggtt_init_hw(struct drm_i915_private *i915); int i915_ggtt_enable_hw(struct drm_i915_private *i915); int i915_init_ggtt(struct drm_i915_private *i915); void i915_ggtt_driver_release(struct drm_i915_private *i915); void i915_ggtt_driver_late_release(struct drm_i915_private *i915); struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915); static inline bool i915_ggtt_has_aperture(const struct i915_ggtt *ggtt) { … } int i915_ppgtt_init_hw(struct intel_gt *gt); struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt, unsigned long lmem_pt_obj_flags); void i915_ggtt_suspend_vm(struct i915_address_space *vm); bool i915_ggtt_resume_vm(struct i915_address_space *vm); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); void fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count); #define fill_px(px, v) … #define fill32_px(px, v) … int setup_scratch_page(struct i915_address_space *vm); void free_scratch(struct i915_address_space *vm); struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz); struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz); struct i915_page_directory *alloc_pd(struct i915_address_space *vm); struct i915_page_directory *__alloc_pd(int npde); int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj); void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl); #define free_pt(vm, px) … #define free_pd(vm, px) … void __set_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, struct i915_page_table *pt, u64 (*encode)(const dma_addr_t, const enum i915_cache_level)); #define set_pd_entry(pd, idx, to) … void clear_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, const struct drm_i915_gem_object * const scratch); bool release_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, struct i915_page_table * const pt, const struct drm_i915_gem_object * const scratch); void gen6_ggtt_invalidate(struct i915_ggtt *ggtt); void ppgtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, unsigned int pat_index, u32 flags); void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res); void gtt_write_workarounds(struct intel_gt *gt); void setup_private_pat(struct intel_gt *gt); int i915_vm_alloc_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, u64 size); int i915_vm_map_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); void i915_vm_free_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); struct i915_vma * __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size); struct i915_vma * __vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size); static inline struct sgt_dma { … } bool i915_ggtt_require_binder(struct drm_i915_private *i915); #endif
Codebase Browser
linux