kfd_events.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright 2014-2022 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <linux/mm_types.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/mman.h>
#include <linux/memory.h>
#include "kfd_priv.h"
#include "kfd_events.h"
#include <linux/device.h>

/*
 * Wrapper around wait_queue_entry_t
 */
struct kfd_event_waiter { … };

/*
 * Each signal event needs a 64-bit signal slot where the signaler will write
 * a 1 before sending an interrupt. (This is needed because some interrupts
 * do not contain enough spare data bits to identify an event.)
 * We get whole pages and map them to the process VA.
 * Individual signal events use their event_id as slot index.
 */
struct kfd_signal_page { … };

static uint64_t *page_slots(struct kfd_signal_page *page)
{ … }

static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
{ … }

static int allocate_event_notification_slot(struct kfd_process *p,
					    struct kfd_event *ev,
					    const int *restore_id)
{ … }

/*
 * Assumes that p->event_mutex or rcu_readlock is held and of course that p is
 * not going away.
 */
static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
{ … }

/**
 * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID
 * @p:     Pointer to struct kfd_process
 * @id:    ID to look up
 * @bits:  Number of valid bits in @id
 *
 * Finds the first signaled event with a matching partial ID. If no
 * matching signaled event is found, returns NULL. In that case the
 * caller should assume that the partial ID is invalid and do an
 * exhaustive search of all siglaned events.
 *
 * If multiple events with the same partial ID signal at the same
 * time, they will be found one interrupt at a time, not necessarily
 * in the same order the interrupts occurred. As long as the number of
 * interrupts is correct, all signaled events will be seen by the
 * driver.
 */
static struct kfd_event *lookup_signaled_event_by_partial_id(
	struct kfd_process *p, uint32_t id, uint32_t bits)
{ … }

static int create_signal_event(struct file *devkfd, struct kfd_process *p,
				struct kfd_event *ev, const int *restore_id)
{ … }

static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)
{ … }

int kfd_event_init_process(struct kfd_process *p)
{ … }

static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
{ … }

static void destroy_events(struct kfd_process *p)
{ … }

/*
 * We assume that the process is being destroyed and there is no need to
 * unmap the pages or keep bookkeeping data in order.
 */
static void shutdown_signal_page(struct kfd_process *p)
{ … }

void kfd_event_free_process(struct kfd_process *p)
{ … }

static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
{ … }

static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
{ … }

static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
		       uint64_t size, uint64_t user_handle)
{ … }

int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
{ … }

int kfd_event_create(struct file *devkfd, struct kfd_process *p,
		     uint32_t event_type, bool auto_reset, uint32_t node_id,
		     uint32_t *event_id, uint32_t *event_trigger_data,
		     uint64_t *event_page_offset, uint32_t *event_slot_index)
{ … }

int kfd_criu_restore_event(struct file *devkfd,
			   struct kfd_process *p,
			   uint8_t __user *user_priv_ptr,
			   uint64_t *priv_data_offset,
			   uint64_t max_priv_data_size)
{ … }

int kfd_criu_checkpoint_events(struct kfd_process *p,
			 uint8_t __user *user_priv_data,
			 uint64_t *priv_data_offset)
{ … }

int kfd_get_num_events(struct kfd_process *p)
{ … }

/* Assumes that p is current. */
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
{ … }

static void set_event(struct kfd_event *ev)
{ … }

/* Assumes that p is current. */
int kfd_set_event(struct kfd_process *p, uint32_t event_id)
{ … }

static void reset_event(struct kfd_event *ev)
{ … }

/* Assumes that p is current. */
int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
{ … }

static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
{ … }

static void set_event_from_interrupt(struct kfd_process *p,
					struct kfd_event *ev)
{ … }

void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
				uint32_t valid_id_bits)
{ … }

static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
{ … }

static int init_event_waiter(struct kfd_process *p,
		struct kfd_event_waiter *waiter,
		struct kfd_event_data *event_data)
{ … }

/* test_event_condition - Test condition of events being waited for
 * @all:           Return completion only if all events have signaled
 * @num_events:    Number of events to wait for
 * @event_waiters: Array of event waiters, one per event
 *
 * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have
 * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all)
 * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of
 * the events have been destroyed.
 */
static uint32_t test_event_condition(bool all, uint32_t num_events,
				struct kfd_event_waiter *event_waiters)
{ … }

/*
 * Copy event specific data, if defined.
 * Currently only memory exception events have additional data to copy to user
 */
static int copy_signaled_event_data(uint32_t num_events,
		struct kfd_event_waiter *event_waiters,
		struct kfd_event_data __user *data)
{ … }

static long user_timeout_to_jiffies(uint32_t user_timeout_ms)
{ … }

static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters,
			 bool undo_auto_reset)
{ … }

int kfd_wait_on_events(struct kfd_process *p,
		       uint32_t num_events, void __user *data,
		       bool all, uint32_t *user_timeout_ms,
		       uint32_t *wait_result)
{ … }

int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
{ … }

/*
 * Assumes that p is not going away.
 */
static void lookup_events_by_type_and_signal(struct kfd_process *p,
		int type, void *event_data)
{ … }

void kfd_signal_hw_exception_event(u32 pasid)
{ … }

void kfd_signal_vm_fault_event(struct kfd_node *dev, u32 pasid,
				struct kfd_vm_fault_info *info,
				struct kfd_hsa_memory_exception_data *data)
{ … }

void kfd_signal_reset_event(struct kfd_node *dev)
{ … }

void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid)
{ … }
linux/drivers/gpu/drm/amd/amdkfd/kfd_events.c
