// SPDX-License-Identifier: GPL-2.0-only /* * VDUSE: vDPA Device in Userspace * * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. * * Author: Xie Yongji <[email protected]> * */ #include "linux/virtio_net.h" #include <linux/init.h> #include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> #include <linux/eventfd.h> #include <linux/slab.h> #include <linux/wait.h> #include <linux/dma-map-ops.h> #include <linux/poll.h> #include <linux/file.h> #include <linux/uio.h> #include <linux/vdpa.h> #include <linux/nospec.h> #include <linux/vmalloc.h> #include <linux/sched/mm.h> #include <uapi/linux/vduse.h> #include <uapi/linux/vdpa.h> #include <uapi/linux/virtio_config.h> #include <uapi/linux/virtio_ids.h> #include <uapi/linux/virtio_blk.h> #include <uapi/linux/virtio_ring.h> #include <linux/mod_devicetable.h> #include "iova_domain.h" #define DRV_AUTHOR … #define DRV_DESC … #define DRV_LICENSE … #define VDUSE_DEV_MAX … #define VDUSE_MAX_BOUNCE_SIZE … #define VDUSE_MIN_BOUNCE_SIZE … #define VDUSE_BOUNCE_SIZE … /* 128 MB reserved for virtqueue creation */ #define VDUSE_IOVA_SIZE … #define VDUSE_MSG_DEFAULT_TIMEOUT … #define IRQ_UNBOUND … struct vduse_virtqueue { … }; struct vduse_dev; struct vduse_vdpa { … }; struct vduse_umem { … }; struct vduse_dev { … }; struct vduse_dev_msg { … }; struct vduse_control { … }; static DEFINE_MUTEX(vduse_lock); static DEFINE_IDR(vduse_idr); static dev_t vduse_major; static struct cdev vduse_ctrl_cdev; static struct cdev vduse_cdev; static struct workqueue_struct *vduse_irq_wq; static struct workqueue_struct *vduse_irq_bound_wq; static u32 allowed_device_id[] = …; static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) { … } static inline struct vduse_dev *dev_to_vduse(struct device *dev) { … } static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, uint32_t request_id) { … } static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) { … } static void vduse_enqueue_msg(struct list_head *head, struct vduse_dev_msg *msg) { … } static void vduse_dev_broken(struct vduse_dev *dev) { … } static int vduse_dev_msg_sync(struct vduse_dev *dev, struct vduse_dev_msg *msg) { … } static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, struct vduse_virtqueue *vq, struct vdpa_vq_state_packed *packed) { … } static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, struct vduse_virtqueue *vq, struct vdpa_vq_state_split *split) { … } static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) { … } static int vduse_dev_update_iotlb(struct vduse_dev *dev, u64 start, u64 last) { … } static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) { … } static bool is_mem_zero(const char *ptr, int size) { … } static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) { … } static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) { … } static void vduse_dev_reset(struct vduse_dev *dev) { … } static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, u64 desc_area, u64 driver_area, u64 device_area) { … } static void vduse_vq_kick(struct vduse_virtqueue *vq) { … } static void vduse_vq_kick_work(struct work_struct *work) { … } static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) { … } static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, struct vdpa_callback *cb) { … } static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) { … } static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx) { … } static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) { … } static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) { … } static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, const struct vdpa_vq_state *state) { … } static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, struct vdpa_vq_state *state) { … } static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) { … } static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) { … } static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) { … } static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) { … } static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, struct vdpa_callback *cb) { … } static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) { … } static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) { … } static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) { … } static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) { … } static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) { … } static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) { … } static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, void *buf, unsigned int len) { … } static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, const void *buf, unsigned int len) { … } static int vduse_vdpa_reset(struct vdpa_device *vdpa) { … } static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) { … } static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, const struct cpumask *cpu_mask) { … } static const struct cpumask * vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx) { … } static int vduse_vdpa_set_map(struct vdpa_device *vdpa, unsigned int asid, struct vhost_iotlb *iotlb) { … } static void vduse_vdpa_free(struct vdpa_device *vdpa) { … } static const struct vdpa_config_ops vduse_vdpa_config_ops = …; static void vduse_dev_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { … } static void vduse_dev_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { … } static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { … } static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { … } static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, unsigned long attrs) { … } static void vduse_dev_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { … } static size_t vduse_dev_max_mapping_size(struct device *dev) { … } static const struct dma_map_ops vduse_dev_dma_ops = …; static unsigned int perm_to_file_flags(u8 perm) { … } static int vduse_kickfd_setup(struct vduse_dev *dev, struct vduse_vq_eventfd *eventfd) { … } static bool vduse_dev_is_ready(struct vduse_dev *dev) { … } static void vduse_dev_irq_inject(struct work_struct *work) { … } static void vduse_vq_irq_inject(struct work_struct *work) { … } static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq) { … } static int vduse_dev_queue_irq_work(struct vduse_dev *dev, struct work_struct *irq_work, int irq_effective_cpu) { … } static int vduse_dev_dereg_umem(struct vduse_dev *dev, u64 iova, u64 size) { … } static int vduse_dev_reg_umem(struct vduse_dev *dev, u64 iova, u64 uaddr, u64 size) { … } static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq) { … } static long vduse_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { … } static int vduse_dev_release(struct inode *inode, struct file *file) { … } static struct vduse_dev *vduse_dev_get_from_minor(int minor) { … } static int vduse_dev_open(struct inode *inode, struct file *file) { … } static const struct file_operations vduse_dev_fops = …; static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf) { … } static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq, const char *buf, size_t count) { … } struct vq_sysfs_entry { … }; static struct vq_sysfs_entry irq_cb_affinity_attr = …; static struct attribute *vq_attrs[] = …; ATTRIBUTE_GROUPS(…); static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) { … } static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { … } static const struct sysfs_ops vq_sysfs_ops = …; static void vq_release(struct kobject *kobj) { … } static const struct kobj_type vq_type = …; static char *vduse_devnode(const struct device *dev, umode_t *mode) { … } static const struct class vduse_class = …; static void vduse_dev_deinit_vqs(struct vduse_dev *dev) { … } static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) { … } static struct vduse_dev *vduse_dev_create(void) { … } static void vduse_dev_destroy(struct vduse_dev *dev) { … } static struct vduse_dev *vduse_find_dev(const char *name) { … } static int vduse_destroy_dev(char *name) { … } static bool device_is_allowed(u32 device_id) { … } static bool features_is_valid(struct vduse_dev_config *config) { … } static bool vduse_validate_config(struct vduse_dev_config *config) { … } static ssize_t msg_timeout_show(struct device *device, struct device_attribute *attr, char *buf) { … } static ssize_t msg_timeout_store(struct device *device, struct device_attribute *attr, const char *buf, size_t count) { … } static DEVICE_ATTR_RW(msg_timeout); static ssize_t bounce_size_show(struct device *device, struct device_attribute *attr, char *buf) { … } static ssize_t bounce_size_store(struct device *device, struct device_attribute *attr, const char *buf, size_t count) { … } static DEVICE_ATTR_RW(bounce_size); static struct attribute *vduse_dev_attrs[] = …; ATTRIBUTE_GROUPS(…); static int vduse_create_dev(struct vduse_dev_config *config, void *config_buf, u64 api_version) { … } static long vduse_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { … } static int vduse_release(struct inode *inode, struct file *file) { … } static int vduse_open(struct inode *inode, struct file *file) { … } static const struct file_operations vduse_ctrl_fops = …; struct vduse_mgmt_dev { … }; static struct vduse_mgmt_dev *vduse_mgmt; static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) { … } static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, const struct vdpa_dev_set_config *config) { … } static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) { … } static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = …; static struct virtio_device_id id_table[] = …; static void vduse_mgmtdev_release(struct device *dev) { … } static int vduse_mgmtdev_init(void) { … } static void vduse_mgmtdev_exit(void) { … } static int vduse_init(void) { … } module_init(…) …; static void vduse_exit(void) { … } module_exit(vduse_exit); MODULE_LICENSE(…); MODULE_AUTHOR(…); MODULE_DESCRIPTION(…);
Codebase Browser
linux