// SPDX-License-Identifier: MIT /* * Copyright © 2021 Intel Corporation */ #include "xe_ggtt.h" #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/sizes.h> #include <drm/drm_drv.h> #include <drm/drm_managed.h> #include <drm/intel/i915_drm.h> #include <generated/xe_wa_oob.h> #include "regs/xe_gt_regs.h" #include "regs/xe_gtt_defs.h" #include "regs/xe_regs.h" #include "xe_assert.h" #include "xe_bo.h" #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_printk.h" #include "xe_gt_sriov_vf.h" #include "xe_gt_tlb_invalidation.h" #include "xe_map.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_sriov.h" #include "xe_wa.h" #include "xe_wopcm.h" /** * DOC: Global Graphics Translation Table (GGTT) * * Xe GGTT implements the support for a Global Virtual Address space that is used * for resources that are accessible to privileged (i.e. kernel-mode) processes, * and not tied to a specific user-level process. For example, the Graphics * micro-Controller (GuC) and Display Engine (if present) utilize this Global * address space. * * The Global GTT (GGTT) translates from the Global virtual address to a physical * address that can be accessed by HW. The GGTT is a flat, single-level table. * * Xe implements a simplified version of the GGTT specifically managing only a * certain range of it that goes from the Write Once Protected Content Memory (WOPCM) * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space * is limited on both ends of the GGTT, because the GuC shim HW redirects * accesses to those addresses to other HW areas instead of going through the * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size, * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things * simple, instead of checking each object to see if they are accessed by GuC or * not, we just exclude those areas from the allocator. Additionally, to simplify * the driver load, we use the maximum WOPCM size in this logic instead of the * programmed one, so we don't need to wait until the actual size to be * programmed is determined (which requires FW fetch) before initializing the * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs * depending on the platform) but we can live with this. Another benefit of this * is the GuC bootrom can't access anything below the WOPCM max size so anything * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max * give us the correct placement for free. */ static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, u16 pat_index) { … } static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, u16 pat_index) { … } static unsigned int probe_gsm_size(struct pci_dev *pdev) { … } static void ggtt_update_access_counter(struct xe_ggtt *ggtt) { … } static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte) { … } static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte) { … } static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) { … } static void ggtt_fini_early(struct drm_device *drm, void *arg) { … } static void ggtt_fini(void *arg) { … } static void primelockdep(struct xe_ggtt *ggtt) { … } static const struct xe_ggtt_pt_ops xelp_pt_ops = …; static const struct xe_ggtt_pt_ops xelpg_pt_ops = …; static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = …; /** * xe_ggtt_init_early - Early GGTT initialization * @ggtt: the &xe_ggtt to be initialized * * It allows to create new mappings usable by the GuC. * Mappings are not usable by the HW engines, as it doesn't have scratch nor * initial clear done to it yet. That will happen in the regular, non-early * GGTT initialization. * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_init_early(struct xe_ggtt *ggtt) { … } static void xe_ggtt_invalidate(struct xe_ggtt *ggtt); static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) { … } static void ggtt_node_remove(struct xe_ggtt_node *node) { … } static void ggtt_node_remove_work_func(struct work_struct *work) { … } /** * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT * @node: the &xe_ggtt_node to be removed * @invalidate: if node needs invalidation upon removal */ void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate) { … } /** * xe_ggtt_init - Regular non-early GGTT initialization * @ggtt: the &xe_ggtt to be initialized * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_init(struct xe_ggtt *ggtt) { … } static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) { … } static void xe_ggtt_invalidate(struct xe_ggtt *ggtt) { … } static void xe_ggtt_dump_node(struct xe_ggtt *ggtt, const struct drm_mm_node *node, const char *description) { … } /** * xe_ggtt_node_insert_balloon - prevent allocation of specified GGTT addresses * @node: the &xe_ggtt_node to hold reserved GGTT node * @start: the starting GGTT address of the reserved region * @end: then end GGTT address of the reserved region * * Use xe_ggtt_node_remove_balloon() to release a reserved GGTT node. * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_node_insert_balloon(struct xe_ggtt_node *node, u64 start, u64 end) { … } /** * xe_ggtt_node_remove_balloon - release a reserved GGTT region * @node: the &xe_ggtt_node with reserved GGTT region * * See xe_ggtt_node_insert_balloon() for details. */ void xe_ggtt_node_remove_balloon(struct xe_ggtt_node *node) { … } /** * xe_ggtt_node_insert_locked - Locked version to insert a &xe_ggtt_node into the GGTT * @node: the &xe_ggtt_node to be inserted * @size: size of the node * @align: alignment constrain of the node * @mm_flags: flags to control the node behavior * * It cannot be called without first having called xe_ggtt_init() once. * To be used in cases where ggtt->lock is already taken. * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node, u32 size, u32 align, u32 mm_flags) { … } /** * xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT * @node: the &xe_ggtt_node to be inserted * @size: size of the node * @align: alignment constrain of the node * * It cannot be called without first having called xe_ggtt_init() once. * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align) { … } /** * xe_ggtt_node_init - Initialize %xe_ggtt_node struct * @ggtt: the &xe_ggtt where the new node will later be inserted/reserved. * * This function will allocated the struct %xe_ggtt_node and return it's pointer. * This struct will then be freed after the node removal upon xe_ggtt_node_remove() * or xe_ggtt_node_remove_balloon(). * Having %xe_ggtt_node struct allocated doesn't mean that the node is already allocated * in GGTT. Only the xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(), * xe_ggtt_node_insert_balloon() will ensure the node is inserted or reserved in GGTT. * * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise. **/ struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt) { … } /** * xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct * @node: the &xe_ggtt_node to be freed * * If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(), * or xe_ggtt_node_insert_balloon(); and this @node is not going to be reused, then, * this function needs to be called to free the %xe_ggtt_node struct **/ void xe_ggtt_node_fini(struct xe_ggtt_node *node) { … } /** * xe_ggtt_node_allocated - Check if node is allocated in GGTT * @node: the &xe_ggtt_node to be inspected * * Return: True if allocated, False otherwise. */ bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node) { … } /** * xe_ggtt_map_bo - Map the BO into GGTT * @ggtt: the &xe_ggtt where node will be mapped * @bo: the &xe_bo to be mapped */ void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) { … } static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, u64 start, u64 end) { … } /** * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space * @ggtt: the &xe_ggtt where bo will be inserted * @bo: the &xe_bo to be inserted * @start: address where it will be inserted * @end: end of the range where it will be inserted * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, u64 start, u64 end) { … } /** * xe_ggtt_insert_bo - Insert BO into GGTT * @ggtt: the &xe_ggtt where bo will be inserted * @bo: the &xe_bo to be inserted * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) { … } /** * xe_ggtt_remove_bo - Remove a BO from the GGTT * @ggtt: the &xe_ggtt where node will be removed * @bo: the &xe_bo to be removed */ void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) { … } /** * xe_ggtt_largest_hole - Largest GGTT hole * @ggtt: the &xe_ggtt that will be inspected * @alignment: minimum alignment * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare * * Return: size of the largest continuous GGTT region */ u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare) { … } #ifdef CONFIG_PCI_IOV static u64 xe_encode_vfid_pte(u16 vfid) { … } static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid) { … } /** * xe_ggtt_assign - assign a GGTT region to the VF * @node: the &xe_ggtt_node to update * @vfid: the VF identifier * * This function is used by the PF driver to assign a GGTT region to the VF. * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some * platforms VFs can't modify that either. */ void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid) { … } #endif /** * xe_ggtt_dump - Dump GGTT for debug * @ggtt: the &xe_ggtt to be dumped * @p: the &drm_mm_printer helper handle to be used to dump the information * * Return: 0 on success or a negative error code on failure. */ int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p) { … } /** * xe_ggtt_print_holes - Print holes * @ggtt: the &xe_ggtt to be inspected * @alignment: min alignment * @p: the &drm_printer * * Print GGTT ranges that are available and return total size available. * * Return: Total available size. */ u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p) { … }
Codebase Browser
linux