linux/drivers/gpu/drm/i915/gt/gen8_ppgtt.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2020 Intel Corporation
 */

#include <linux/log2.h>

#include "gem/i915_gem_internal.h"
#include "gem/i915_gem_lmem.h"

#include "gen8_ppgtt.h"
#include "i915_scatterlist.h"
#include "i915_trace.h"
#include "i915_pvinfo.h"
#include "i915_vgpu.h"
#include "intel_gt.h"
#include "intel_gtt.h"

static u64 gen8_pde_encode(const dma_addr_t addr,
			   const enum i915_cache_level level)
{ … }

static u64 gen8_pte_encode(dma_addr_t addr,
			   unsigned int pat_index,
			   u32 flags)
{ … }

static u64 gen12_pte_encode(dma_addr_t addr,
			    unsigned int pat_index,
			    u32 flags)
{ … }

static void gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
{ … }

/* Index shifts into the pagetable are offset by GEN8_PTE_SHIFT [12] */
#define GEN8_PAGE_SIZE …
#define GEN8_PTE_SHIFT …
#define GEN8_PDES …
#define gen8_pd_shift(lvl) …
#define gen8_pd_index(i, lvl) …
#define __gen8_pte_shift(lvl) …
#define __gen8_pte_index(a, lvl) …

#define as_pd(x) …

static unsigned int
gen8_pd_range(u64 start, u64 end, int lvl, unsigned int *idx)
{ … }

static bool gen8_pd_contains(u64 start, u64 end, int lvl)
{ … }

static unsigned int gen8_pt_count(u64 start, u64 end)
{ … }

static unsigned int gen8_pd_top_count(const struct i915_address_space *vm)
{ … }

static struct i915_page_directory *
gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx)
{ … }

static struct i915_page_directory *
gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr)
{ … }

static void __gen8_ppgtt_cleanup(struct i915_address_space *vm,
				 struct i915_page_directory *pd,
				 int count, int lvl)
{ … }

static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
{ … }

static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm,
			      struct i915_page_directory * const pd,
			      u64 start, const u64 end, int lvl)
{ … }

static void gen8_ppgtt_clear(struct i915_address_space *vm,
			     u64 start, u64 length)
{ … }

static void __gen8_ppgtt_alloc(struct i915_address_space * const vm,
			       struct i915_vm_pt_stash *stash,
			       struct i915_page_directory * const pd,
			       u64 * const start, const u64 end, int lvl)
{ … }

static void gen8_ppgtt_alloc(struct i915_address_space *vm,
			     struct i915_vm_pt_stash *stash,
			     u64 start, u64 length)
{ … }

static void __gen8_ppgtt_foreach(struct i915_address_space *vm,
				 struct i915_page_directory *pd,
				 u64 *start, u64 end, int lvl,
				 void (*fn)(struct i915_address_space *vm,
					    struct i915_page_table *pt,
					    void *data),
				 void *data)
{ … }

static void gen8_ppgtt_foreach(struct i915_address_space *vm,
			       u64 start, u64 length,
			       void (*fn)(struct i915_address_space *vm,
					  struct i915_page_table *pt,
					  void *data),
			       void *data)
{ … }

static __always_inline u64
gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt,
		      struct i915_page_directory *pdp,
		      struct sgt_dma *iter,
		      u64 idx,
		      unsigned int pat_index,
		      u32 flags)
{ … }

static void
xehp_ppgtt_insert_huge(struct i915_address_space *vm,
		       struct i915_vma_resource *vma_res,
		       struct sgt_dma *iter,
		       unsigned int pat_index,
		       u32 flags)
{ … }

static void gen8_ppgtt_insert_huge(struct i915_address_space *vm,
				   struct i915_vma_resource *vma_res,
				   struct sgt_dma *iter,
				   unsigned int pat_index,
				   u32 flags)
{ … }

static void gen8_ppgtt_insert(struct i915_address_space *vm,
			      struct i915_vma_resource *vma_res,
			      unsigned int pat_index,
			      u32 flags)
{ … }

static void gen8_ppgtt_insert_entry(struct i915_address_space *vm,
				    dma_addr_t addr,
				    u64 offset,
				    unsigned int pat_index,
				    u32 flags)
{ … }

static void xehp_ppgtt_insert_entry_lm(struct i915_address_space *vm,
				       dma_addr_t addr,
				       u64 offset,
				       unsigned int pat_index,
				       u32 flags)
{ … }

static void xehp_ppgtt_insert_entry(struct i915_address_space *vm,
				    dma_addr_t addr,
				    u64 offset,
				    unsigned int pat_index,
				    u32 flags)
{ … }

static int gen8_init_scratch(struct i915_address_space *vm)
{ … }

static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
{ … }

static struct i915_page_directory *
gen8_alloc_top_pd(struct i915_address_space *vm)
{ … }

static int gen8_init_rsvd(struct i915_address_space *vm)
{ … }

/*
 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 * with a net effect resembling a 2-level page table in normal x86 terms. Each
 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 * space.
 *
 */
struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt,
				     unsigned long lmem_pt_obj_flags)
{ … }