// SPDX-License-Identifier: GPL-2.0-only //#define DEBUG #include <linux/spinlock.h> #include <linux/slab.h> #include <linux/blkdev.h> #include <linux/hdreg.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/interrupt.h> #include <linux/virtio.h> #include <linux/virtio_blk.h> #include <linux/scatterlist.h> #include <linux/string_helpers.h> #include <linux/idr.h> #include <linux/blk-mq.h> #include <linux/blk-mq-virtio.h> #include <linux/numa.h> #include <linux/vmalloc.h> #include <uapi/linux/virtio_ring.h> #define PART_BITS … #define VQ_NAME_LEN … #define MAX_DISCARD_SEGMENTS … /* The maximum number of sg elements that fit into a virtqueue */ #define VIRTIO_BLK_MAX_SG_ELEMS … #ifdef CONFIG_ARCH_NO_SG_CHAIN #define VIRTIO_BLK_INLINE_SG_CNT … #else #define VIRTIO_BLK_INLINE_SG_CNT … #endif static unsigned int num_request_queues; module_param(num_request_queues, uint, 0644); MODULE_PARM_DESC(…) …; static unsigned int poll_queues; module_param(poll_queues, uint, 0644); MODULE_PARM_DESC(…) …; static int major; static DEFINE_IDA(vd_index_ida); static struct workqueue_struct *virtblk_wq; struct virtio_blk_vq { … } ____cacheline_aligned_in_smp; struct virtio_blk { … }; struct virtblk_req { … }; static inline blk_status_t virtblk_result(u8 status) { … } static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx) { … } static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr) { … } static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap) { … } static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr) { … } static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req, struct virtblk_req *vbr) { … } static void virtblk_cleanup_cmd(struct request *req) { … } static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, struct request *req, struct virtblk_req *vbr) { … } /* * The status byte is always the last byte of the virtblk request * in-header. This helper fetches its value for all in-header formats * that are currently defined. */ static inline u8 virtblk_vbr_status(struct virtblk_req *vbr) { … } static inline void virtblk_request_done(struct request *req) { … } static void virtblk_done(struct virtqueue *vq) { … } static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) { … } static blk_status_t virtblk_fail_to_queue(struct request *req, int rc) { … } static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx, struct virtio_blk *vblk, struct request *req, struct virtblk_req *vbr) { … } static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { … } static bool virtblk_prep_rq_batch(struct request *req) { … } static bool virtblk_add_req_batch(struct virtio_blk_vq *vq, struct request **rqlist) { … } static void virtio_queue_rqs(struct request **rqlist) { … } #ifdef CONFIG_BLK_DEV_ZONED static void *virtblk_alloc_report_buffer(struct virtio_blk *vblk, unsigned int nr_zones, size_t *buflen) { … } static int virtblk_submit_zone_report(struct virtio_blk *vblk, char *report_buf, size_t report_len, sector_t sector) { … } static int virtblk_parse_zone(struct virtio_blk *vblk, struct virtio_blk_zone_descriptor *entry, unsigned int idx, report_zones_cb cb, void *data) { … } static int virtblk_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data) { … } static int virtblk_read_zoned_limits(struct virtio_blk *vblk, struct queue_limits *lim) { … } #else /* * Zoned block device support is not configured in this kernel, host-managed * zoned devices can't be supported. */ #define virtblk_report_zones … static inline int virtblk_read_zoned_limits(struct virtio_blk *vblk, struct queue_limits *lim) { dev_err(&vblk->vdev->dev, "virtio_blk: zoned devices are not supported"); return -EOPNOTSUPP; } #endif /* CONFIG_BLK_DEV_ZONED */ /* return id (s/n) string for *disk to *id_str */ static int virtblk_get_id(struct gendisk *disk, char *id_str) { … } /* We provide getgeo only to please some old bootloader/partitioning tools */ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) { … } static void virtblk_free_disk(struct gendisk *disk) { … } static const struct block_device_operations virtblk_fops = …; static int index_to_minor(int index) { … } static int minor_to_index(int minor) { … } static ssize_t serial_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(serial); /* The queue's logical block size must be set before calling this */ static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) { … } static void virtblk_config_changed_work(struct work_struct *work) { … } static void virtblk_config_changed(struct virtio_device *vdev) { … } static int init_vq(struct virtio_blk *vblk) { … } /* * Legacy naming scheme used for virtio devices. We are stuck with it for * virtio blk but don't ever use it for any new driver. */ static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) { … } static int virtblk_get_cache_mode(struct virtio_device *vdev) { … } static const char *const virtblk_cache_types[] = …; static ssize_t cache_type_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } static ssize_t cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RW(cache_type); static struct attribute *virtblk_attrs[] = …; static umode_t virtblk_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n) { … } static const struct attribute_group virtblk_attr_group = …; static const struct attribute_group *virtblk_attr_groups[] = …; static void virtblk_map_queues(struct blk_mq_tag_set *set) { … } static void virtblk_complete_batch(struct io_comp_batch *iob) { … } static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { … } static const struct blk_mq_ops virtio_mq_ops = …; static unsigned int virtblk_queue_depth; module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); static int virtblk_read_limits(struct virtio_blk *vblk, struct queue_limits *lim) { … } static int virtblk_probe(struct virtio_device *vdev) { … } static void virtblk_remove(struct virtio_device *vdev) { … } #ifdef CONFIG_PM_SLEEP static int virtblk_freeze(struct virtio_device *vdev) { … } static int virtblk_restore(struct virtio_device *vdev) { … } #endif static const struct virtio_device_id id_table[] = …; static unsigned int features_legacy[] = … ; static unsigned int features[] = …; static struct virtio_driver virtio_blk = …; static int __init virtio_blk_init(void) { … } static void __exit virtio_blk_fini(void) { … } module_init(…) …; module_exit(virtio_blk_fini); MODULE_DEVICE_TABLE(virtio, id_table); MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux