linux/drivers/vfio/pci/mlx5/main.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
 */

#include <linux/device.h>
#include <linux/eventfd.h>
#include <linux/file.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/sched/mm.h>
#include <linux/anon_inodes.h>

#include "cmd.h"

/* Device specification max LOAD size */
#define MAX_LOAD_SIZE …

#define MAX_CHUNK_SIZE …

static struct mlx5vf_pci_core_device *mlx5vf_drvdata(struct pci_dev *pdev)
{ … }

struct page *
mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
			  unsigned long offset)
{ … }

static void mlx5vf_disable_fd(struct mlx5_vf_migration_file *migf)
{ … }

static int mlx5vf_release_file(struct inode *inode, struct file *filp)
{ … }

static struct mlx5_vhca_data_buffer *
mlx5vf_get_data_buff_from_pos(struct mlx5_vf_migration_file *migf, loff_t pos,
			      bool *end_of_data)
{ … }

static void mlx5vf_buf_read_done(struct mlx5_vhca_data_buffer *vhca_buf)
{ … }

static ssize_t mlx5vf_buf_read(struct mlx5_vhca_data_buffer *vhca_buf,
			       char __user **buf, size_t *len, loff_t *pos)
{ … }

static ssize_t mlx5vf_save_read(struct file *filp, char __user *buf, size_t len,
			       loff_t *pos)
{ … }

static __poll_t mlx5vf_save_poll(struct file *filp,
				 struct poll_table_struct *wait)
{ … }

/*
 * FD is exposed and user can use it after receiving an error.
 * Mark migf in error, and wake the user.
 */
static void mlx5vf_mark_err(struct mlx5_vf_migration_file *migf)
{ … }

void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
				   u8 chunk_num, size_t next_required_umem_size)
{ … }

static struct mlx5_vhca_data_buffer *
mlx5vf_mig_file_get_stop_copy_buf(struct mlx5_vf_migration_file *migf,
				  u8 index, size_t required_length)
{ … }

static void mlx5vf_mig_file_save_work(struct work_struct *_work)
{ … }

static int mlx5vf_add_stop_copy_header(struct mlx5_vf_migration_file *migf,
				       bool track)
{ … }

static int mlx5vf_prep_stop_copy(struct mlx5vf_pci_core_device *mvdev,
				 struct mlx5_vf_migration_file *migf,
				 size_t state_size, u64 full_size,
				 bool track)
{ … }

static long mlx5vf_precopy_ioctl(struct file *filp, unsigned int cmd,
				 unsigned long arg)
{ … }

static const struct file_operations mlx5vf_save_fops = …;

static int mlx5vf_pci_save_device_inc_data(struct mlx5vf_pci_core_device *mvdev)
{ … }

static struct mlx5_vf_migration_file *
mlx5vf_pci_save_device_data(struct mlx5vf_pci_core_device *mvdev, bool track)
{ … }

static int
mlx5vf_append_page_to_mig_buf(struct mlx5_vhca_data_buffer *vhca_buf,
			      const char __user **buf, size_t *len,
			      loff_t *pos, ssize_t *done)
{ … }

static ssize_t
mlx5vf_resume_read_image(struct mlx5_vf_migration_file *migf,
			 struct mlx5_vhca_data_buffer *vhca_buf,
			 size_t image_size, const char __user **buf,
			 size_t *len, loff_t *pos, ssize_t *done,
			 bool *has_work)
{ … }

static int
mlx5vf_resume_read_header_data(struct mlx5_vf_migration_file *migf,
			       struct mlx5_vhca_data_buffer *vhca_buf,
			       const char __user **buf, size_t *len,
			       loff_t *pos, ssize_t *done)
{ … }

static int
mlx5vf_resume_read_header(struct mlx5_vf_migration_file *migf,
			  struct mlx5_vhca_data_buffer *vhca_buf,
			  const char __user **buf,
			  size_t *len, loff_t *pos,
			  ssize_t *done, bool *has_work)
{ … }

static ssize_t mlx5vf_resume_write(struct file *filp, const char __user *buf,
				   size_t len, loff_t *pos)
{ … }

static const struct file_operations mlx5vf_resume_fops = …;

static struct mlx5_vf_migration_file *
mlx5vf_pci_resume_device_data(struct mlx5vf_pci_core_device *mvdev)
{ … }

void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,
			enum mlx5_vf_migf_state *last_save_state)
{ … }

static struct file *
mlx5vf_pci_step_device_state_locked(struct mlx5vf_pci_core_device *mvdev,
				    u32 new)
{ … }

/*
 * This function is called in all state_mutex unlock cases to
 * handle a 'deferred_reset' if exists.
 */
void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)
{ … }

static struct file *
mlx5vf_pci_set_device_state(struct vfio_device *vdev,
			    enum vfio_device_mig_state new_state)
{ … }

static int mlx5vf_pci_get_data_size(struct vfio_device *vdev,
				    unsigned long *stop_copy_length)
{ … }

static int mlx5vf_pci_get_device_state(struct vfio_device *vdev,
				       enum vfio_device_mig_state *curr_state)
{ … }

static void mlx5vf_pci_aer_reset_done(struct pci_dev *pdev)
{ … }

static int mlx5vf_pci_open_device(struct vfio_device *core_vdev)
{ … }

static void mlx5vf_pci_close_device(struct vfio_device *core_vdev)
{ … }

static const struct vfio_migration_ops mlx5vf_pci_mig_ops = …;

static const struct vfio_log_ops mlx5vf_pci_log_ops = …;

static int mlx5vf_pci_init_dev(struct vfio_device *core_vdev)
{ … }

static void mlx5vf_pci_release_dev(struct vfio_device *core_vdev)
{ … }

static const struct vfio_device_ops mlx5vf_pci_ops = …;

static int mlx5vf_pci_probe(struct pci_dev *pdev,
			    const struct pci_device_id *id)
{ … }

static void mlx5vf_pci_remove(struct pci_dev *pdev)
{ … }

static const struct pci_device_id mlx5vf_pci_table[] = …;

MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table);

static const struct pci_error_handlers mlx5vf_err_handlers = …;

static struct pci_driver mlx5vf_pci_driver = …;

module_pci_driver(…) …;

MODULE_IMPORT_NS(…);
MODULE_LICENSE(…) …;
MODULE_AUTHOR(…) …;
MODULE_AUTHOR(…) …;
MODULE_DESCRIPTION(…) …;