linux/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright 2014-2022 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <linux/device.h>
#include <linux/export.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/compat.h>
#include <uapi/linux/kfd_ioctl.h>
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/ptrace.h>
#include <linux/dma-buf.h>
#include <linux/fdtable.h>
#include <linux/processor.h>
#include "kfd_priv.h"
#include "kfd_device_queue_manager.h"
#include "kfd_svm.h"
#include "amdgpu_amdkfd.h"
#include "kfd_smi_events.h"
#include "amdgpu_dma_buf.h"
#include "kfd_debug.h"

static long kfd_ioctl(struct file *, unsigned int, unsigned long);
static int kfd_open(struct inode *, struct file *);
static int kfd_release(struct inode *, struct file *);
static int kfd_mmap(struct file *, struct vm_area_struct *);

static const char kfd_dev_name[] = …;

static const struct file_operations kfd_fops = …;

static int kfd_char_dev_major = …;
struct device *kfd_device;
static const struct class kfd_class = …;

static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
{ … }

static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
{ … }

int kfd_chardev_init(void)
{ … }

void kfd_chardev_exit(void)
{ … }


static int kfd_open(struct inode *inode, struct file *filep)
{ … }

static int kfd_release(struct inode *inode, struct file *filep)
{ … }

static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
					void *data)
{ … }

static int set_queue_properties_from_user(struct queue_properties *q_properties,
				struct kfd_ioctl_create_queue_args *args)
{ … }

static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_get_queue_wave_state(struct file *filep,
					  struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_set_memory_policy(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_set_trap_handler(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_dbg_register(struct file *filep,
				struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_dbg_unregister(struct file *filep,
				struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_dbg_address_watch(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

/* Parse and generate fixed size data structure for wave control */
static int kfd_ioctl_dbg_wave_control(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_get_clock_counters(struct file *filep,
				struct kfd_process *p, void *data)
{ … }


static int kfd_ioctl_get_process_apertures(struct file *filp,
				struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_get_process_apertures_new(struct file *filp,
				struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
					void *data)
{ … }

static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
				void *data)
{ … }

static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
				void *data)
{ … }

static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
				void *data)
{ … }
static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_get_tile_config(struct file *filep,
		struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
				void *data)
{ … }

bool kfd_dev_is_large_bar(struct kfd_node *dev)
{ … }

static int kfd_ioctl_get_available_memory(struct file *filep,
					  struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
					struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_alloc_queue_gws(struct file *filep,
		struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_get_dmabuf_info(struct file *filep,
		struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_import_dmabuf(struct file *filep,
				   struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_export_dmabuf(struct file *filep,
				   struct kfd_process *p, void *data)
{ … }

/* Handle requests for watching SMI events */
static int kfd_ioctl_smi_events(struct file *filep,
				struct kfd_process *p, void *data)
{ … }

#if IS_ENABLED(CONFIG_HSA_AMD_SVM)

static int kfd_ioctl_set_xnack_mode(struct file *filep,
				    struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
{ … }
#else
static int kfd_ioctl_set_xnack_mode(struct file *filep,
				    struct kfd_process *p, void *data)
{
	return -EPERM;
}
static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
{
	return -EPERM;
}
#endif

static int criu_checkpoint_process(struct kfd_process *p,
			     uint8_t __user *user_priv_data,
			     uint64_t *priv_offset)
{ … }

static int criu_checkpoint_devices(struct kfd_process *p,
			     uint32_t num_devices,
			     uint8_t __user *user_addr,
			     uint8_t __user *user_priv_data,
			     uint64_t *priv_offset)
{ … }

static uint32_t get_process_num_bos(struct kfd_process *p)
{ … }

static int criu_get_prime_handle(struct kgd_mem *mem,
				 int flags, u32 *shared_fd)
{ … }

static int criu_checkpoint_bos(struct kfd_process *p,
			       uint32_t num_bos,
			       uint8_t __user *user_bos,
			       uint8_t __user *user_priv_data,
			       uint64_t *priv_offset)
{ … }

static int criu_get_process_object_info(struct kfd_process *p,
					uint32_t *num_devices,
					uint32_t *num_bos,
					uint32_t *num_objects,
					uint64_t *objs_priv_size)
{ … }

static int criu_checkpoint(struct file *filep,
			   struct kfd_process *p,
			   struct kfd_ioctl_criu_args *args)
{ … }

static int criu_restore_process(struct kfd_process *p,
				struct kfd_ioctl_criu_args *args,
				uint64_t *priv_offset,
				uint64_t max_priv_data_size)
{ … }

static int criu_restore_devices(struct kfd_process *p,
				struct kfd_ioctl_criu_args *args,
				uint64_t *priv_offset,
				uint64_t max_priv_data_size)
{ … }

static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
				      struct kfd_criu_bo_bucket *bo_bucket,
				      struct kfd_criu_bo_priv_data *bo_priv,
				      struct kgd_mem **kgd_mem)
{ … }

static int criu_restore_bo(struct kfd_process *p,
			   struct kfd_criu_bo_bucket *bo_bucket,
			   struct kfd_criu_bo_priv_data *bo_priv)
{ … }

static int criu_restore_bos(struct kfd_process *p,
			    struct kfd_ioctl_criu_args *args,
			    uint64_t *priv_offset,
			    uint64_t max_priv_data_size)
{ … }

static int criu_restore_objects(struct file *filep,
				struct kfd_process *p,
				struct kfd_ioctl_criu_args *args,
				uint64_t *priv_offset,
				uint64_t max_priv_data_size)
{ … }

static int criu_restore(struct file *filep,
			struct kfd_process *p,
			struct kfd_ioctl_criu_args *args)
{ … }

static int criu_unpause(struct file *filep,
			struct kfd_process *p,
			struct kfd_ioctl_criu_args *args)
{ … }

static int criu_resume(struct file *filep,
			struct kfd_process *p,
			struct kfd_ioctl_criu_args *args)
{ … }

static int criu_process_info(struct file *filep,
				struct kfd_process *p,
				struct kfd_ioctl_criu_args *args)
{ … }

static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
{ … }

static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
			bool enable_ttmp_setup)
{ … }

static int runtime_disable(struct kfd_process *p)
{ … }

static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
{ … }

static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
{ … }

#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) …

/** Ioctl table */
static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = …;

#define AMDKFD_CORE_IOCTL_COUNT …

static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
{ … }

static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
		      struct vm_area_struct *vma)
{ … }


static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
{ … }