// SPDX-License-Identifier: MIT
/*
* Copyright © 2021 Intel Corporation
*/
#include "xe_ggtt.h"
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/sizes.h>
#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/intel/i915_drm.h>
#include <generated/xe_wa_oob.h>
#include "regs/xe_gt_regs.h"
#include "regs/xe_gtt_defs.h"
#include "regs/xe_regs.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_gt.h"
#include "xe_gt_printk.h"
#include "xe_gt_sriov_vf.h"
#include "xe_gt_tlb_invalidation.h"
#include "xe_map.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_sriov.h"
#include "xe_wa.h"
#include "xe_wopcm.h"
/**
* DOC: Global Graphics Translation Table (GGTT)
*
* Xe GGTT implements the support for a Global Virtual Address space that is used
* for resources that are accessible to privileged (i.e. kernel-mode) processes,
* and not tied to a specific user-level process. For example, the Graphics
* micro-Controller (GuC) and Display Engine (if present) utilize this Global
* address space.
*
* The Global GTT (GGTT) translates from the Global virtual address to a physical
* address that can be accessed by HW. The GGTT is a flat, single-level table.
*
* Xe implements a simplified version of the GGTT specifically managing only a
* certain range of it that goes from the Write Once Protected Content Memory (WOPCM)
* Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to
* the GuC (Graphics Microcontroller) hardware limitations. The GuC address space
* is limited on both ends of the GGTT, because the GuC shim HW redirects
* accesses to those addresses to other HW areas instead of going through the
* GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size,
* while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things
* simple, instead of checking each object to see if they are accessed by GuC or
* not, we just exclude those areas from the allocator. Additionally, to simplify
* the driver load, we use the maximum WOPCM size in this logic instead of the
* programmed one, so we don't need to wait until the actual size to be
* programmed is determined (which requires FW fetch) before initializing the
* GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs
* depending on the platform) but we can live with this. Another benefit of this
* is the GuC bootrom can't access anything below the WOPCM max size so anything
* the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT
* above the WOPCM max size. Starting the GGTT allocations above the WOPCM max
* give us the correct placement for free.
*/
static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
u16 pat_index)
{
u64 pte;
pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
pte |= XE_PAGE_PRESENT;
if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
pte |= XE_GGTT_PTE_DM;
return pte;
}
static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
u16 pat_index)
{
struct xe_device *xe = xe_bo_device(bo);
u64 pte;
pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
xe_assert(xe, pat_index <= 3);
if (pat_index & BIT(0))
pte |= XELPG_GGTT_PTE_PAT0;
if (pat_index & BIT(1))
pte |= XELPG_GGTT_PTE_PAT1;
return pte;
}
static unsigned int probe_gsm_size(struct pci_dev *pdev)
{
u16 gmch_ctl, ggms;
pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
return ggms ? SZ_1M << ggms : 0;
}
static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
{
struct xe_gt *gt = XE_WA(ggtt->tile->primary_gt, 22019338487) ? ggtt->tile->primary_gt :
ggtt->tile->media_gt;
u32 max_gtt_writes = XE_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63;
/*
* Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
* to wait for completion of prior GTT writes before letting this through.
* This needs to be done for all GGTT writes originating from the CPU.
*/
lockdep_assert_held(&ggtt->lock);
if ((++ggtt->access_count % max_gtt_writes) == 0) {
xe_mmio_write32(gt, GMD_ID, 0x0);
ggtt->access_count = 0;
}
}
static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
{
xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
xe_tile_assert(ggtt->tile, addr < ggtt->size);
writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
}
static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
{
xe_ggtt_set_pte(ggtt, addr, pte);
ggtt_update_access_counter(ggtt);
}
static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
{
u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
u64 end = start + size - 1;
u64 scratch_pte;
xe_tile_assert(ggtt->tile, start < end);
if (ggtt->scratch)
scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
pat_index);
else
scratch_pte = 0;
while (start < end) {
ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
start += XE_PAGE_SIZE;
}
}
static void ggtt_fini_early(struct drm_device *drm, void *arg)
{
struct xe_ggtt *ggtt = arg;
destroy_workqueue(ggtt->wq);
mutex_destroy(&ggtt->lock);
drm_mm_takedown(&ggtt->mm);
}
static void ggtt_fini(void *arg)
{
struct xe_ggtt *ggtt = arg;
ggtt->scratch = NULL;
}
static void primelockdep(struct xe_ggtt *ggtt)
{
if (!IS_ENABLED(CONFIG_LOCKDEP))
return;
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&ggtt->lock);
fs_reclaim_release(GFP_KERNEL);
}
static const struct xe_ggtt_pt_ops xelp_pt_ops = {
.pte_encode_bo = xelp_ggtt_pte_encode_bo,
.ggtt_set_pte = xe_ggtt_set_pte,
};
static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
.ggtt_set_pte = xe_ggtt_set_pte,
};
static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
.ggtt_set_pte = xe_ggtt_set_pte_and_flush,
};
/**
* xe_ggtt_init_early - Early GGTT initialization
* @ggtt: the &xe_ggtt to be initialized
*
* It allows to create new mappings usable by the GuC.
* Mappings are not usable by the HW engines, as it doesn't have scratch nor
* initial clear done to it yet. That will happen in the regular, non-early
* GGTT initialization.
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_init_early(struct xe_ggtt *ggtt)
{
struct xe_device *xe = tile_to_xe(ggtt->tile);
struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
unsigned int gsm_size;
int err;
if (IS_SRIOV_VF(xe))
gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
else
gsm_size = probe_gsm_size(pdev);
if (gsm_size == 0) {
drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
return -ENOMEM;
}
ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
ggtt->flags |= XE_GGTT_FLAGS_64K;
if (ggtt->size > GUC_GGTT_TOP)
ggtt->size = GUC_GGTT_TOP;
if (GRAPHICS_VERx100(xe) >= 1270)
ggtt->pt_ops = (ggtt->tile->media_gt &&
XE_WA(ggtt->tile->media_gt, 22019338487)) ||
XE_WA(ggtt->tile->primary_gt, 22019338487) ?
&xelpg_pt_wa_ops : &xelpg_pt_ops;
else
ggtt->pt_ops = &xelp_pt_ops;
ggtt->wq = alloc_workqueue("xe-ggtt-wq", 0, 0);
drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
ggtt->size - xe_wopcm_size(xe));
mutex_init(&ggtt->lock);
primelockdep(ggtt);
err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
if (err)
return err;
if (IS_SRIOV_VF(xe)) {
err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
if (err)
return err;
}
return 0;
}
static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
{
struct drm_mm_node *hole;
u64 start, end;
/* Display may have allocated inside ggtt, so be careful with clearing here */
mutex_lock(&ggtt->lock);
drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
xe_ggtt_clear(ggtt, start, end - start);
xe_ggtt_invalidate(ggtt);
mutex_unlock(&ggtt->lock);
}
static void ggtt_node_remove(struct xe_ggtt_node *node)
{
struct xe_ggtt *ggtt = node->ggtt;
struct xe_device *xe = tile_to_xe(ggtt->tile);
bool bound;
int idx;
bound = drm_dev_enter(&xe->drm, &idx);
mutex_lock(&ggtt->lock);
if (bound)
xe_ggtt_clear(ggtt, node->base.start, node->base.size);
drm_mm_remove_node(&node->base);
node->base.size = 0;
mutex_unlock(&ggtt->lock);
if (!bound)
goto free_node;
if (node->invalidate_on_remove)
xe_ggtt_invalidate(ggtt);
drm_dev_exit(idx);
free_node:
xe_ggtt_node_fini(node);
}
static void ggtt_node_remove_work_func(struct work_struct *work)
{
struct xe_ggtt_node *node = container_of(work, typeof(*node),
delayed_removal_work);
struct xe_device *xe = tile_to_xe(node->ggtt->tile);
xe_pm_runtime_get(xe);
ggtt_node_remove(node);
xe_pm_runtime_put(xe);
}
/**
* xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT
* @node: the &xe_ggtt_node to be removed
* @invalidate: if node needs invalidation upon removal
*/
void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
{
struct xe_ggtt *ggtt;
struct xe_device *xe;
if (!node || !node->ggtt)
return;
ggtt = node->ggtt;
xe = tile_to_xe(ggtt->tile);
node->invalidate_on_remove = invalidate;
if (xe_pm_runtime_get_if_active(xe)) {
ggtt_node_remove(node);
xe_pm_runtime_put(xe);
} else {
queue_work(ggtt->wq, &node->delayed_removal_work);
}
}
/**
* xe_ggtt_init - Regular non-early GGTT initialization
* @ggtt: the &xe_ggtt to be initialized
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_init(struct xe_ggtt *ggtt)
{
struct xe_device *xe = tile_to_xe(ggtt->tile);
unsigned int flags;
int err;
/*
* So we don't need to worry about 64K GGTT layout when dealing with
* scratch entires, rather keep the scratch page in system memory on
* platforms where 64K pages are needed for VRAM.
*/
flags = XE_BO_FLAG_PINNED;
if (ggtt->flags & XE_GGTT_FLAGS_64K)
flags |= XE_BO_FLAG_SYSTEM;
else
flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
if (IS_ERR(ggtt->scratch)) {
err = PTR_ERR(ggtt->scratch);
goto err;
}
xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
xe_ggtt_initial_clear(ggtt);
return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt);
err:
ggtt->scratch = NULL;
return err;
}
static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
{
int err;
if (!gt)
return;
err = xe_gt_tlb_invalidation_ggtt(gt);
if (err)
drm_warn(>_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
}
static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
{
/* Each GT in a tile has its own TLB to cache GGTT lookups */
ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
}
static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
const struct drm_mm_node *node, const char *description)
{
char buf[10];
if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
node->start, node->start + node->size, buf, description);
}
}
/**
* xe_ggtt_node_insert_balloon - prevent allocation of specified GGTT addresses
* @node: the &xe_ggtt_node to hold reserved GGTT node
* @start: the starting GGTT address of the reserved region
* @end: then end GGTT address of the reserved region
*
* Use xe_ggtt_node_remove_balloon() to release a reserved GGTT node.
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_node_insert_balloon(struct xe_ggtt_node *node, u64 start, u64 end)
{
struct xe_ggtt *ggtt = node->ggtt;
int err;
xe_tile_assert(ggtt->tile, start < end);
xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(&node->base));
node->base.color = 0;
node->base.start = start;
node->base.size = end - start;
mutex_lock(&ggtt->lock);
err = drm_mm_reserve_node(&ggtt->mm, &node->base);
mutex_unlock(&ggtt->lock);
if (xe_gt_WARN(ggtt->tile->primary_gt, err,
"Failed to balloon GGTT %#llx-%#llx (%pe)\n",
node->base.start, node->base.start + node->base.size, ERR_PTR(err)))
return err;
xe_ggtt_dump_node(ggtt, &node->base, "balloon");
return 0;
}
/**
* xe_ggtt_node_remove_balloon - release a reserved GGTT region
* @node: the &xe_ggtt_node with reserved GGTT region
*
* See xe_ggtt_node_insert_balloon() for details.
*/
void xe_ggtt_node_remove_balloon(struct xe_ggtt_node *node)
{
if (!node || !node->ggtt)
return;
if (!drm_mm_node_allocated(&node->base))
goto free_node;
xe_ggtt_dump_node(node->ggtt, &node->base, "remove-balloon");
mutex_lock(&node->ggtt->lock);
drm_mm_remove_node(&node->base);
mutex_unlock(&node->ggtt->lock);
free_node:
xe_ggtt_node_fini(node);
}
/**
* xe_ggtt_node_insert_locked - Locked version to insert a &xe_ggtt_node into the GGTT
* @node: the &xe_ggtt_node to be inserted
* @size: size of the node
* @align: alignment constrain of the node
* @mm_flags: flags to control the node behavior
*
* It cannot be called without first having called xe_ggtt_init() once.
* To be used in cases where ggtt->lock is already taken.
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node,
u32 size, u32 align, u32 mm_flags)
{
return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
mm_flags);
}
/**
* xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT
* @node: the &xe_ggtt_node to be inserted
* @size: size of the node
* @align: alignment constrain of the node
*
* It cannot be called without first having called xe_ggtt_init() once.
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align)
{
int ret;
if (!node || !node->ggtt)
return -ENOENT;
mutex_lock(&node->ggtt->lock);
ret = xe_ggtt_node_insert_locked(node, size, align,
DRM_MM_INSERT_HIGH);
mutex_unlock(&node->ggtt->lock);
return ret;
}
/**
* xe_ggtt_node_init - Initialize %xe_ggtt_node struct
* @ggtt: the &xe_ggtt where the new node will later be inserted/reserved.
*
* This function will allocated the struct %xe_ggtt_node and return it's pointer.
* This struct will then be freed after the node removal upon xe_ggtt_node_remove()
* or xe_ggtt_node_remove_balloon().
* Having %xe_ggtt_node struct allocated doesn't mean that the node is already allocated
* in GGTT. Only the xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
* xe_ggtt_node_insert_balloon() will ensure the node is inserted or reserved in GGTT.
*
* Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
**/
struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
{
struct xe_ggtt_node *node = kzalloc(sizeof(*node), GFP_NOFS);
if (!node)
return ERR_PTR(-ENOMEM);
INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func);
node->ggtt = ggtt;
return node;
}
/**
* xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct
* @node: the &xe_ggtt_node to be freed
*
* If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
* or xe_ggtt_node_insert_balloon(); and this @node is not going to be reused, then,
* this function needs to be called to free the %xe_ggtt_node struct
**/
void xe_ggtt_node_fini(struct xe_ggtt_node *node)
{
kfree(node);
}
/**
* xe_ggtt_node_allocated - Check if node is allocated in GGTT
* @node: the &xe_ggtt_node to be inspected
*
* Return: True if allocated, False otherwise.
*/
bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node)
{
if (!node || !node->ggtt)
return false;
return drm_mm_node_allocated(&node->base);
}
/**
* xe_ggtt_map_bo - Map the BO into GGTT
* @ggtt: the &xe_ggtt where node will be mapped
* @bo: the &xe_bo to be mapped
*/
void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
u64 start;
u64 offset, pte;
if (XE_WARN_ON(!bo->ggtt_node))
return;
start = bo->ggtt_node->base.start;
for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
ggtt->pt_ops->ggtt_set_pte(ggtt, start + offset, pte);
}
}
static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
u64 start, u64 end)
{
int err;
u64 alignment = XE_PAGE_SIZE;
if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
alignment = SZ_64K;
if (XE_WARN_ON(bo->ggtt_node)) {
/* Someone's already inserted this BO in the GGTT */
xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
return 0;
}
err = xe_bo_validate(bo, NULL, false);
if (err)
return err;
xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
bo->ggtt_node = xe_ggtt_node_init(ggtt);
if (IS_ERR(bo->ggtt_node)) {
err = PTR_ERR(bo->ggtt_node);
bo->ggtt_node = NULL;
goto out;
}
mutex_lock(&ggtt->lock);
err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node->base, bo->size,
alignment, 0, start, end, 0);
if (err) {
xe_ggtt_node_fini(bo->ggtt_node);
bo->ggtt_node = NULL;
} else {
xe_ggtt_map_bo(ggtt, bo);
}
mutex_unlock(&ggtt->lock);
if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
xe_ggtt_invalidate(ggtt);
out:
xe_pm_runtime_put(tile_to_xe(ggtt->tile));
return err;
}
/**
* xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space
* @ggtt: the &xe_ggtt where bo will be inserted
* @bo: the &xe_bo to be inserted
* @start: address where it will be inserted
* @end: end of the range where it will be inserted
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
u64 start, u64 end)
{
return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
}
/**
* xe_ggtt_insert_bo - Insert BO into GGTT
* @ggtt: the &xe_ggtt where bo will be inserted
* @bo: the &xe_bo to be inserted
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
}
/**
* xe_ggtt_remove_bo - Remove a BO from the GGTT
* @ggtt: the &xe_ggtt where node will be removed
* @bo: the &xe_bo to be removed
*/
void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
if (XE_WARN_ON(!bo->ggtt_node))
return;
/* This BO is not currently in the GGTT */
xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
xe_ggtt_node_remove(bo->ggtt_node,
bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
}
/**
* xe_ggtt_largest_hole - Largest GGTT hole
* @ggtt: the &xe_ggtt that will be inspected
* @alignment: minimum alignment
* @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare
*
* Return: size of the largest continuous GGTT region
*/
u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare)
{
const struct drm_mm *mm = &ggtt->mm;
const struct drm_mm_node *entry;
u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
u64 hole_start, hole_end, hole_size;
u64 max_hole = 0;
mutex_lock(&ggtt->lock);
drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
hole_start = max(hole_start, hole_min_start);
hole_start = ALIGN(hole_start, alignment);
hole_end = ALIGN_DOWN(hole_end, alignment);
if (hole_start >= hole_end)
continue;
hole_size = hole_end - hole_start;
if (spare)
*spare -= min3(*spare, hole_size, max_hole);
max_hole = max(max_hole, hole_size);
}
mutex_unlock(&ggtt->lock);
return max_hole;
}
#ifdef CONFIG_PCI_IOV
static u64 xe_encode_vfid_pte(u16 vfid)
{
return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
}
static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
{
u64 start = node->start;
u64 size = node->size;
u64 end = start + size - 1;
u64 pte = xe_encode_vfid_pte(vfid);
lockdep_assert_held(&ggtt->lock);
if (!drm_mm_node_allocated(node))
return;
while (start < end) {
ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
start += XE_PAGE_SIZE;
}
xe_ggtt_invalidate(ggtt);
}
/**
* xe_ggtt_assign - assign a GGTT region to the VF
* @node: the &xe_ggtt_node to update
* @vfid: the VF identifier
*
* This function is used by the PF driver to assign a GGTT region to the VF.
* In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
* platforms VFs can't modify that either.
*/
void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
{
mutex_lock(&node->ggtt->lock);
xe_ggtt_assign_locked(node->ggtt, &node->base, vfid);
mutex_unlock(&node->ggtt->lock);
}
#endif
/**
* xe_ggtt_dump - Dump GGTT for debug
* @ggtt: the &xe_ggtt to be dumped
* @p: the &drm_mm_printer helper handle to be used to dump the information
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
{
int err;
err = mutex_lock_interruptible(&ggtt->lock);
if (err)
return err;
drm_mm_print(&ggtt->mm, p);
mutex_unlock(&ggtt->lock);
return err;
}
/**
* xe_ggtt_print_holes - Print holes
* @ggtt: the &xe_ggtt to be inspected
* @alignment: min alignment
* @p: the &drm_printer
*
* Print GGTT ranges that are available and return total size available.
*
* Return: Total available size.
*/
u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p)
{
const struct drm_mm *mm = &ggtt->mm;
const struct drm_mm_node *entry;
u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
u64 hole_start, hole_end, hole_size;
u64 total = 0;
char buf[10];
mutex_lock(&ggtt->lock);
drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
hole_start = max(hole_start, hole_min_start);
hole_start = ALIGN(hole_start, alignment);
hole_end = ALIGN_DOWN(hole_end, alignment);
if (hole_start >= hole_end)
continue;
hole_size = hole_end - hole_start;
total += hole_size;
string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
hole_start, hole_end - 1, buf);
}
mutex_unlock(&ggtt->lock);
return total;
}
Codebase Browser
linux