// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/bsearch.h> #include <linux/pci.h> #include <linux/slab.h> #include "kfd_priv.h" #include "kfd_device_queue_manager.h" #include "kfd_pm4_headers_vi.h" #include "kfd_pm4_headers_aldebaran.h" #include "cwsr_trap_handler.h" #include "amdgpu_amdkfd.h" #include "kfd_smi_events.h" #include "kfd_svm.h" #include "kfd_migrate.h" #include "amdgpu.h" #include "amdgpu_xcp.h" #define MQD_SIZE_ALIGNED … /* * kfd_locked is used to lock the kfd driver during suspend or reset * once locked, kfd driver will stop any further GPU execution. * create process (open) will return -EAGAIN. */ static int kfd_locked; #ifdef CONFIG_DRM_AMDGPU_CIK extern const struct kfd2kgd_calls gfx_v7_kfd2kgd; #endif extern const struct kfd2kgd_calls gfx_v8_kfd2kgd; extern const struct kfd2kgd_calls gfx_v9_kfd2kgd; extern const struct kfd2kgd_calls arcturus_kfd2kgd; extern const struct kfd2kgd_calls aldebaran_kfd2kgd; extern const struct kfd2kgd_calls gc_9_4_3_kfd2kgd; extern const struct kfd2kgd_calls gfx_v10_kfd2kgd; extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd; extern const struct kfd2kgd_calls gfx_v11_kfd2kgd; extern const struct kfd2kgd_calls gfx_v12_kfd2kgd; static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, unsigned int chunk_size); static void kfd_gtt_sa_fini(struct kfd_dev *kfd); static int kfd_resume(struct kfd_node *kfd); static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd) { … } static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) { … } static void kfd_device_info_init(struct kfd_dev *kfd, bool vf, uint32_t gfx_target_version) { … } struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) { … } static void kfd_cwsr_init(struct kfd_dev *kfd) { … } static int kfd_gws_init(struct kfd_node *node) { … } static void kfd_smi_init(struct kfd_node *dev) { … } static int kfd_init_node(struct kfd_node *node) { … } static void kfd_cleanup_nodes(struct kfd_dev *kfd, unsigned int num_nodes) { … } static void kfd_setup_interrupt_bitmap(struct kfd_node *node, unsigned int kfd_node_idx) { … } bool kgd2kfd_device_init(struct kfd_dev *kfd, const struct kgd2kfd_shared_resources *gpu_resources) { … } void kgd2kfd_device_exit(struct kfd_dev *kfd) { … } int kgd2kfd_pre_reset(struct kfd_dev *kfd, struct amdgpu_reset_context *reset_context) { … } /* * Fix me. KFD won't be able to resume existing process for now. * We will keep all existing process in a evicted state and * wait the process to be terminated. */ int kgd2kfd_post_reset(struct kfd_dev *kfd) { … } bool kfd_is_locked(void) { … } void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) { … } int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) { … } static int kfd_resume(struct kfd_node *node) { … } static inline void kfd_queue_work(struct workqueue_struct *wq, struct work_struct *work) { … } /* This is called directly from KGD at ISR. */ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) { … } int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger) { … } int kgd2kfd_resume_mm(struct mm_struct *mm) { … } /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will * prepare for safe eviction of KFD BOs that belong to the specified * process. * * @mm: mm_struct that identifies the specified KFD process * @fence: eviction fence attached to KFD process BOs * */ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, struct dma_fence *fence) { … } static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, unsigned int chunk_size) { … } static void kfd_gtt_sa_fini(struct kfd_dev *kfd) { … } static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr, unsigned int bit_num, unsigned int chunk_size) { … } static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr, unsigned int bit_num, unsigned int chunk_size) { … } int kfd_gtt_sa_allocate(struct kfd_node *node, unsigned int size, struct kfd_mem_obj **mem_obj) { … } int kfd_gtt_sa_free(struct kfd_node *node, struct kfd_mem_obj *mem_obj) { … } void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) { … } void kfd_inc_compute_active(struct kfd_node *node) { … } void kfd_dec_compute_active(struct kfd_node *node) { … } void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask) { … } /* kfd_get_num_sdma_engines returns the number of PCIe optimized SDMA and * kfd_get_num_xgmi_sdma_engines returns the number of XGMI SDMA. * When the device has more than two engines, we reserve two for PCIe to enable * full-duplex and the rest are used as XGMI. */ unsigned int kfd_get_num_sdma_engines(struct kfd_node *node) { … } unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *node) { … } int kgd2kfd_check_and_lock_kfd(void) { … } void kgd2kfd_unlock_kfd(void) { … } #if defined(CONFIG_DEBUG_FS) /* This function will send a package to HIQ to hang the HWS * which will trigger a GPU reset and bring the HWS back to normal state */ int kfd_debugfs_hang_hws(struct kfd_node *dev) { … } #endif
Codebase Browser
linux