nouveau_svm.c | Explore in Territory

/*
 * Copyright 2018 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */
#include "nouveau_svm.h"
#include "nouveau_drv.h"
#include "nouveau_chan.h"
#include "nouveau_dmem.h"

#include <nvif/event.h>
#include <nvif/object.h>
#include <nvif/vmm.h>

#include <nvif/class.h>
#include <nvif/clb069.h>
#include <nvif/ifc00d.h>

#include <linux/sched/mm.h>
#include <linux/sort.h>
#include <linux/hmm.h>
#include <linux/memremap.h>
#include <linux/rmap.h>

struct nouveau_svm { … };

#define FAULT_ACCESS_READ …
#define FAULT_ACCESS_WRITE …
#define FAULT_ACCESS_ATOMIC …
#define FAULT_ACCESS_PREFETCH …

#define SVM_DBG(s,f,a...) …
#define SVM_ERR(s,f,a...) …

struct nouveau_pfnmap_args { … };

struct nouveau_ivmm { … };

static struct nouveau_ivmm *
nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst)
{ … }

#define SVMM_DBG(s,f,a...) …
#define SVMM_ERR(s,f,a...) …

int
nouveau_svmm_bind(struct drm_device *dev, void *data,
		  struct drm_file *file_priv)
{ … }

/* Unlink channel instance from SVMM. */
void
nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst)
{ … }

/* Link channel instance to SVMM. */
int
nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst)
{ … }

/* Invalidate SVMM address-range on GPU. */
void
nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)
{ … }

static int
nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn,
				    const struct mmu_notifier_range *update)
{ … }

static void nouveau_svmm_free_notifier(struct mmu_notifier *mn)
{ … }

static const struct mmu_notifier_ops nouveau_mn_ops = …;

void
nouveau_svmm_fini(struct nouveau_svmm **psvmm)
{ … }

int
nouveau_svmm_init(struct drm_device *dev, void *data,
		  struct drm_file *file_priv)
{ … }

/* Issue fault replay for GPU to retry accesses that faulted previously. */
static void
nouveau_svm_fault_replay(struct nouveau_svm *svm)
{ … }

/* Cancel a replayable fault that could not be handled.
 *
 * Cancelling the fault will trigger recovery to reset the engine
 * and kill the offending channel (ie. GPU SIGSEGV).
 */
static void
nouveau_svm_fault_cancel(struct nouveau_svm *svm,
			 u64 inst, u8 hub, u8 gpc, u8 client)
{ … }

static void
nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm,
			       struct nouveau_svm_fault *fault)
{ … }

static int
nouveau_svm_fault_priority(u8 fault)
{ … }

static int
nouveau_svm_fault_cmp(const void *a, const void *b)
{ … }

static void
nouveau_svm_fault_cache(struct nouveau_svm *svm,
			struct nouveau_svm_fault_buffer *buffer, u32 offset)
{ … }

struct svm_notifier { … };

static bool nouveau_svm_range_invalidate(struct mmu_interval_notifier *mni,
					 const struct mmu_notifier_range *range,
					 unsigned long cur_seq)
{ … }

static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = …;

static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm,
				    struct hmm_range *range,
				    struct nouveau_pfnmap_args *args)
{ … }

static int nouveau_atomic_range_fault(struct nouveau_svmm *svmm,
			       struct nouveau_drm *drm,
			       struct nouveau_pfnmap_args *args, u32 size,
			       struct svm_notifier *notifier)
{ … }

static int nouveau_range_fault(struct nouveau_svmm *svmm,
			       struct nouveau_drm *drm,
			       struct nouveau_pfnmap_args *args, u32 size,
			       unsigned long hmm_flags,
			       struct svm_notifier *notifier)
{ … }

static void
nouveau_svm_fault(struct work_struct *work)
{ … }

static int
nouveau_svm_event(struct nvif_event *event, void *argv, u32 argc)
{ … }

static struct nouveau_pfnmap_args *
nouveau_pfns_to_args(void *pfns)
{ … }

u64 *
nouveau_pfns_alloc(unsigned long npages)
{ … }

void
nouveau_pfns_free(u64 *pfns)
{ … }

void
nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm,
		 unsigned long addr, u64 *pfns, unsigned long npages)
{ … }

static void
nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id)
{ … }

static int
nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id)
{ … }

static void
nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id)
{ … }

static int
nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id)
{ … }

void
nouveau_svm_resume(struct nouveau_drm *drm)
{ … }

void
nouveau_svm_suspend(struct nouveau_drm *drm)
{ … }

void
nouveau_svm_fini(struct nouveau_drm *drm)
{ … }

void
nouveau_svm_init(struct nouveau_drm *drm)
{ … }
linux/drivers/gpu/drm/nouveau/nouveau_svm.c
