linux/drivers/gpu/drm/xe/xe_hmm.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2024 Intel Corporation
 */

#include <linux/scatterlist.h>
#include <linux/mmu_notifier.h>
#include <linux/dma-mapping.h>
#include <linux/memremap.h>
#include <linux/swap.h>
#include <linux/hmm.h>
#include <linux/mm.h>
#include "xe_hmm.h"
#include "xe_vm.h"
#include "xe_bo.h"

static u64 xe_npages_in_range(unsigned long start, unsigned long end)
{ … }

/*
 * xe_mark_range_accessed() - mark a range is accessed, so core mm
 * have such information for memory eviction or write back to
 * hard disk
 *
 * @range: the range to mark
 * @write: if write to this range, we mark pages in this range
 * as dirty
 */
static void xe_mark_range_accessed(struct hmm_range *range, bool write)
{ … }

/*
 * xe_build_sg() - build a scatter gather table for all the physical pages/pfn
 * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table
 * and will be used to program GPU page table later.
 *
 * @xe: the xe device who will access the dma-address in sg table
 * @range: the hmm range that we build the sg table from. range->hmm_pfns[]
 * has the pfn numbers of pages that back up this hmm address range.
 * @st: pointer to the sg table.
 * @write: whether we write to this range. This decides dma map direction
 * for system pages. If write we map it bi-diretional; otherwise
 * DMA_TO_DEVICE
 *
 * All the contiguous pfns will be collapsed into one entry in
 * the scatter gather table. This is for the purpose of efficiently
 * programming GPU page table.
 *
 * The dma_address in the sg table will later be used by GPU to
 * access memory. So if the memory is system memory, we need to
 * do a dma-mapping so it can be accessed by GPU/DMA.
 *
 * FIXME: This function currently only support pages in system
 * memory. If the memory is GPU local memory (of the GPU who
 * is going to access memory), we need gpu dpa (device physical
 * address), and there is no need of dma-mapping. This is TBD.
 *
 * FIXME: dma-mapping for peer gpu device to access remote gpu's
 * memory. Add this when you support p2p
 *
 * This function allocates the storage of the sg table. It is
 * caller's responsibility to free it calling sg_free_table.
 *
 * Returns 0 if successful; -ENOMEM if fails to allocate memory
 */
static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
		       struct sg_table *st, bool write)
{ … }

/*
 * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
 *
 * @uvma: the userptr vma which hold the scatter gather table
 *
 * With function xe_userptr_populate_range, we allocate storage of
 * the userptr sg table. This is a helper function to free this
 * sg table, and dma unmap the address in the table.
 */
void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
{ … }

/**
 * xe_hmm_userptr_populate_range() - Populate physical pages of a virtual
 * address range
 *
 * @uvma: userptr vma which has information of the range to populate.
 * @is_mm_mmap_locked: True if mmap_read_lock is already acquired by caller.
 *
 * This function populate the physical pages of a virtual
 * address range. The populated physical pages is saved in
 * userptr's sg table. It is similar to get_user_pages but call
 * hmm_range_fault.
 *
 * This function also read mmu notifier sequence # (
 * mmu_interval_read_begin), for the purpose of later
 * comparison (through mmu_interval_read_retry).
 *
 * This must be called with mmap read or write lock held.
 *
 * This function allocates the storage of the userptr sg table.
 * It is caller's responsibility to free it calling sg_free_table.
 *
 * returns: 0 for succuss; negative error no on failure
 */
int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
				  bool is_mm_mmap_locked)
{ … }