// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_vm.h" /* * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt */ struct amdgpu_vm_pt_cursor { … }; /** * amdgpu_vm_pt_level_shift - return the addr shift for each level * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of bits the pfn needs to be right shifted for a level. */ static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, unsigned int level) { … } /** * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of entries in a page directory or page table. */ static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev, unsigned int level) { … } /** * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The mask to extract the entry number of a PD/PT from an address. */ static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev, unsigned int level) { … } /** * amdgpu_vm_pt_size - returns the size of the page table in bytes * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The size of the BO for a page directory or page table in bytes. */ static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev, unsigned int level) { … } /** * amdgpu_vm_pt_parent - get the parent page directory * * @pt: child page table * * Helper to get the parent entry for the child page table. NULL if we are at * the root page directory. */ static struct amdgpu_vm_bo_base * amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt) { … } /** * amdgpu_vm_pt_start - start PD/PT walk * * @adev: amdgpu_device pointer * @vm: amdgpu_vm structure * @start: start address of the walk * @cursor: state to initialize * * Initialize a amdgpu_vm_pt_cursor to start a walk. */ static void amdgpu_vm_pt_start(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t start, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_descendant - go to child node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the child node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_sibling - go to sibling node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the sibling node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_ancestor - go to parent node * * @cursor: current state * * Walk to the parent node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_next - get next PD/PT in hieratchy * * @adev: amdgpu_device pointer * @cursor: current state * * Walk the PD/PT tree to the next node. */ static void amdgpu_vm_pt_next(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_first_dfs - start a deep first search * * @adev: amdgpu_device structure * @vm: amdgpu_vm structure * @start: optional cursor to start with * @cursor: state to initialize * * Starts a deep first traversal of the PD/PT tree. */ static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue * * @start: starting point for the search * @entry: current entry * * Returns: * True when the search should continue, false otherwise. */ static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_bo_base *entry) { … } /** * amdgpu_vm_pt_next_dfs - get the next node for a deep first search * * @adev: amdgpu_device structure * @cursor: current state * * Move the cursor to the next node in a deep first search. */ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { … } /* * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs */ #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) … /** * amdgpu_vm_pt_clear - initially clear the PDs/PTs * * @adev: amdgpu_device pointer * @vm: VM to clear BO from * @vmbo: BO to clear * @immediate: use an immediate update * * Root PD needs to be reserved when calling this. * * Returns: * 0 on success, errno otherwise. */ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_bo_vm *vmbo, bool immediate) { … } /** * amdgpu_vm_pt_create - create bo for PD/PT * * @adev: amdgpu_device pointer * @vm: requesting vm * @level: the page table level * @immediate: use a immediate update * @vmbo: pointer to the buffer object pointer * @xcp_id: GPU partition id */ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, int level, bool immediate, struct amdgpu_bo_vm **vmbo, int32_t xcp_id) { … } /** * amdgpu_vm_pt_alloc - Allocate a specific page table * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @cursor: Which page table to allocate * @immediate: use an immediate update * * Make sure a specific page table or directory is allocated. * * Returns: * 1 if page table needed to be allocated, 0 if page table was already * allocated, negative errno if an error occurred. */ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *cursor, bool immediate) { … } /** * amdgpu_vm_pt_free - free one PD/PT * * @entry: PDE to free */ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) { … } void amdgpu_vm_pt_free_work(struct work_struct *work) { … } /** * amdgpu_vm_pt_free_list - free PD/PT levels * * @adev: amdgpu device structure * @params: see amdgpu_vm_update_params definition * * Free the page directory objects saved in the flush list */ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev, struct amdgpu_vm_update_params *params) { … } /** * amdgpu_vm_pt_add_list - add PD/PT level to the flush list * * @params: parameters for the update * @cursor: first PT entry to start DF search from, non NULL * * This list will be freed after TLB flush. */ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, struct amdgpu_vm_pt_cursor *cursor) { … } /** * amdgpu_vm_pt_free_root - free root PD * @adev: amdgpu device structure * @vm: amdgpu vm structure * * Free the root page directory and everything below it. */ void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm) { … } /** * amdgpu_vm_pde_update - update a single level in the hierarchy * * @params: parameters for the update * @entry: entry to update * * Makes sure the requested entry in parent is up to date. */ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, struct amdgpu_vm_bo_base *entry) { … } /** * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags * * @adev: amdgpu_device pointer * @flags: pointer to PTE flags * * Update PTE no-retry flags when TF is enabled. */ static void amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device *adev, uint64_t *flags) { … } /* * amdgpu_vm_pte_update_flags - figure out flags for PTE updates * * Make sure to set the right flags for the PTEs at the desired level. */ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, struct amdgpu_bo_vm *pt, unsigned int level, uint64_t pe, uint64_t addr, unsigned int count, uint32_t incr, uint64_t flags) { … } /** * amdgpu_vm_pte_fragment - get fragment for PTEs * * @params: see amdgpu_vm_update_params definition * @start: first PTE to handle * @end: last PTE to handle * @flags: hw mapping flags * @frag: resulting fragment size * @frag_end: end of this fragment * * Returns the first possible fragment for the start and end address. */ static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params, uint64_t start, uint64_t end, uint64_t flags, unsigned int *frag, uint64_t *frag_end) { … } /** * amdgpu_vm_ptes_update - make sure that page tables are valid * * @params: see amdgpu_vm_update_params definition * @start: start of GPU address range * @end: end of GPU address range * @dst: destination address to map to, the next dst inside the function * @flags: mapping flags * * Update the page tables in the range @start - @end. * * Returns: * 0 for success, -EINVAL for failure. */ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, uint64_t start, uint64_t end, uint64_t dst, uint64_t flags) { … } /** * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible * @adev: amdgpu device structure * @vm: amdgpu vm structure * * make root page directory and everything below it cpu accessible. */ int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm) { … }
Codebase Browser
linux