// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/mutex.h> #include <linux/log2.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include <linux/mmu_context.h> #include <linux/slab.h> #include <linux/notifier.h> #include <linux/compat.h> #include <linux/mman.h> #include <linux/file.h> #include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" struct mm_struct; #include "kfd_priv.h" #include "kfd_device_queue_manager.h" #include "kfd_svm.h" #include "kfd_smi_events.h" #include "kfd_debug.h" /* * List of struct kfd_process (field kfd_process). * Unique/indexed by mm_struct* */ DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); DEFINE_MUTEX(…) …; DEFINE_SRCU(…); /* For process termination handling */ static struct workqueue_struct *kfd_process_wq; /* Ordered, single-threaded workqueue for restoring evicted * processes. Restoring multiple processes concurrently under memory * pressure can lead to processes blocking each other from validating * their BOs and result in a live-lock situation where processes * remain evicted indefinitely. */ static struct workqueue_struct *kfd_restore_wq; static struct kfd_process *find_process(const struct task_struct *thread, bool ref); static void kfd_process_ref_release(struct kref *ref); static struct kfd_process *create_process(const struct task_struct *thread); static void evict_process_worker(struct work_struct *work); static void restore_process_worker(struct work_struct *work); static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd); struct kfd_procfs_tree { … }; static struct kfd_procfs_tree procfs; /* * Structure for SDMA activity tracking */ struct kfd_sdma_activity_handler_workarea { … }; struct temp_sdma_queue_list { … }; static void kfd_sdma_activity_worker(struct work_struct *work) { … } /** * kfd_get_cu_occupancy - Collect number of waves in-flight on this device * by current process. Translates acquired wave count into number of compute units * that are occupied. * * @attr: Handle of attribute that allows reporting of wave count. The attribute * handle encapsulates GPU device it is associated with, thereby allowing collection * of waves in flight, etc * @buffer: Handle of user provided buffer updated with wave count * * Return: Number of bytes written to user buffer or an error value */ static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer) { … } static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, char *buffer) { … } static void kfd_procfs_kobj_release(struct kobject *kobj) { … } static const struct sysfs_ops kfd_procfs_ops = …; static const struct kobj_type procfs_type = …; void kfd_procfs_init(void) { … } void kfd_procfs_shutdown(void) { … } static ssize_t kfd_procfs_queue_show(struct kobject *kobj, struct attribute *attr, char *buffer) { … } static ssize_t kfd_procfs_stats_show(struct kobject *kobj, struct attribute *attr, char *buffer) { … } static ssize_t kfd_sysfs_counters_show(struct kobject *kobj, struct attribute *attr, char *buf) { … } static struct attribute attr_queue_size = …; static struct attribute attr_queue_type = …; static struct attribute attr_queue_gpuid = …; static struct attribute *procfs_queue_attrs[] = …; ATTRIBUTE_GROUPS(…); static const struct sysfs_ops procfs_queue_ops = …; static const struct kobj_type procfs_queue_type = …; static const struct sysfs_ops procfs_stats_ops = …; static const struct kobj_type procfs_stats_type = …; static const struct sysfs_ops sysfs_counters_ops = …; static const struct kobj_type sysfs_counters_type = …; int kfd_procfs_add_queue(struct queue *q) { … } static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr, char *name) { … } static void kfd_procfs_add_sysfs_stats(struct kfd_process *p) { … } static void kfd_procfs_add_sysfs_counters(struct kfd_process *p) { … } static void kfd_procfs_add_sysfs_files(struct kfd_process *p) { … } void kfd_procfs_del_queue(struct queue *q) { … } int kfd_process_create_wq(void) { … } void kfd_process_destroy_wq(void) { … } static void kfd_process_free_gpuvm(struct kgd_mem *mem, struct kfd_process_device *pdd, void **kptr) { … } /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process * This function should be only called right after the process * is created and when kfd_processes_mutex is still being held * to avoid concurrency. Because of that exclusiveness, we do * not need to take p->mutex. */ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, uint64_t gpu_va, uint32_t size, uint32_t flags, struct kgd_mem **mem, void **kptr) { … } /* kfd_process_device_reserve_ib_mem - Reserve memory inside the * process for IB usage The memory reserved is for KFD to submit * IB to AMDGPU from kernel. If the memory is reserved * successfully, ib_kaddr will have the CPU/kernel * address. Check ib_kaddr before accessing the memory. */ static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) { … } static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd) { … } struct kfd_process *kfd_create_process(struct task_struct *thread) { … } struct kfd_process *kfd_get_process(const struct task_struct *thread) { … } static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) { … } static struct kfd_process *find_process(const struct task_struct *thread, bool ref) { … } void kfd_unref_process(struct kfd_process *p) { … } /* This increments the process->ref counter. */ struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid) { … } static void kfd_process_device_free_bos(struct kfd_process_device *pdd) { … } /* * Just kunmap and unpin signal BO here. It will be freed in * kfd_process_free_outstanding_kfd_bos() */ static void kfd_process_kunmap_signal_bo(struct kfd_process *p) { … } static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p) { … } static void kfd_process_destroy_pdds(struct kfd_process *p) { … } static void kfd_process_remove_sysfs(struct kfd_process *p) { … } /* No process locking is needed in this function, because the process * is not findable any more. We must assume that no other thread is * using it any more, otherwise we couldn't safely free the process * structure in the end. */ static void kfd_process_wq_release(struct work_struct *work) { … } static void kfd_process_ref_release(struct kref *ref) { … } static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm) { … } static void kfd_process_free_notifier(struct mmu_notifier *mn) { … } static void kfd_process_notifier_release_internal(struct kfd_process *p) { … } static void kfd_process_notifier_release(struct mmu_notifier *mn, struct mm_struct *mm) { … } static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = …; /* * This code handles the case when driver is being unloaded before all * mm_struct are released. We need to safely free the kfd_process and * avoid race conditions with mmu_notifier that might try to free them. * */ void kfd_cleanup_processes(void) { … } int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep) { … } static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) { … } static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd) { … } void kfd_process_set_trap_handler(struct qcm_process_device *qpd, uint64_t tba_addr, uint64_t tma_addr) { … } bool kfd_process_xnack_mode(struct kfd_process *p, bool supported) { … } void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd, bool enabled) { … } /* * On return the kfd_process is fully operational and will be freed when the * mm is released */ static struct kfd_process *create_process(const struct task_struct *thread) { … } struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev, struct kfd_process *p) { … } struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev, struct kfd_process *p) { … } /** * kfd_process_device_init_vm - Initialize a VM for a process-device * * @pdd: The process-device * @drm_file: Optional pointer to a DRM file descriptor * * If @drm_file is specified, it will be used to acquire the VM from * that file descriptor. If successful, the @pdd takes ownership of * the file descriptor. * * If @drm_file is NULL, a new VM is created. * * Returns 0 on success, -errno on failure. */ int kfd_process_device_init_vm(struct kfd_process_device *pdd, struct file *drm_file) { … } /* * Direct the IOMMU to bind the process (specifically the pasid->mm) * to the device. * Unbinding occurs when the process dies or the device is removed. * * Assumes that the process lock is held. */ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev, struct kfd_process *p) { … } /* Create specific handle mapped to mem from process local memory idr * Assumes that the process lock is held. */ int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, void *mem) { … } /* Translate specific handle from process local memory idr * Assumes that the process lock is held. */ void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, int handle) { … } /* Remove specific handle from process local memory idr * Assumes that the process lock is held. */ void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, int handle) { … } /* This increments the process->ref counter. */ struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid) { … } /* This increments the process->ref counter. */ struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm) { … } /* kfd_process_evict_queues - Evict all user queues of a process * * Eviction is reference-counted per process-device. This means multiple * evictions from different sources can be nested safely. */ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger) { … } /* kfd_process_restore_queues - Restore all user queues of a process */ int kfd_process_restore_queues(struct kfd_process *p) { … } int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id) { … } int kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node, uint32_t *gpuid, uint32_t *gpuidx) { … } static int signal_eviction_fence(struct kfd_process *p) { … } static void evict_process_worker(struct work_struct *work) { … } static int restore_process_helper(struct kfd_process *p) { … } static void restore_process_worker(struct work_struct *work) { … } void kfd_suspend_all_processes(void) { … } int kfd_resume_all_processes(void) { … } int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process, struct vm_area_struct *vma) { … } /* assumes caller holds process lock. */ int kfd_process_drain_interrupts(struct kfd_process_device *pdd) { … } void kfd_process_close_interrupt_drain(unsigned int pasid) { … } struct send_exception_work_handler_workarea { … }; static void send_exception_work_handler(struct work_struct *work) { … } int kfd_send_exception_to_runtime(struct kfd_process *p, unsigned int queue_id, uint64_t error_reason) { … } struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id) { … } int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id) { … } #if defined(CONFIG_DEBUG_FS) int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data) { … } #endif
Codebase Browser
linux