// SPDX-License-Identifier: GPL-2.0-or-later /* * RDMA Network Block Driver * * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. */ #undef pr_fmt #define pr_fmt(fmt) … #include <linux/module.h> #include <linux/blkdev.h> #include <linux/hdreg.h> #include <linux/scatterlist.h> #include <linux/idr.h> #include "rnbd-clt.h" MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; static int rnbd_client_major; static DEFINE_IDA(index_ida); static DEFINE_MUTEX(sess_lock); static LIST_HEAD(sess_list); static struct workqueue_struct *rnbd_clt_wq; /* * Maximum number of partitions an instance can have. * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself) */ #define RNBD_PART_BITS … static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess) { … } static void free_sess(struct rnbd_clt_session *sess); static void rnbd_clt_put_sess(struct rnbd_clt_session *sess) { … } static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev) { … } static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev) { … } static void rnbd_clt_change_capacity(struct rnbd_clt_dev *dev, sector_t new_nsectors) { … } static int process_msg_open_rsp(struct rnbd_clt_dev *dev, struct rnbd_msg_open_rsp *rsp) { … } int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, sector_t newsize) { … } static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q) { … } enum { … }; /** * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun * @sess: Session to find a queue for * @cpu: Cpu to start the search from * * Description: * Each CPU has a list of HW queues, which needs to be rerun. If a list * is not empty - it is marked with a bit. This function finds first * set bit in a bitmap and returns corresponding CPU list. */ static struct rnbd_cpu_qlist * rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu) { … } static inline int nxt_cpu(int cpu) { … } /** * rnbd_rerun_if_needed() - rerun next queue marked as stopped * @sess: Session to rerun a queue on * * Description: * Each CPU has it's own list of HW queues, which should be rerun. * Function finds such list with HW queues, takes a list lock, picks up * the first HW queue out of the list and requeues it. * * Return: * True if the queue was requeued, false otherwise. * * Context: * Does not matter. */ static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess) { … } /** * rnbd_rerun_all_if_idle() - rerun all queues left in the list if * session is idling (there are no requests * in-flight). * @sess: Session to rerun the queues on * * Description: * This function tries to rerun all stopped queues if there are no * requests in-flight anymore. This function tries to solve an obvious * problem, when number of tags < than number of queues (hctx), which * are stopped and put to sleep. If last permit, which has been just put, * does not wake up all left queues (hctxs), IO requests hang forever. * * That can happen when all number of permits, say N, have been exhausted * from one CPU, and we have many block devices per session, say M. * Each block device has it's own queue (hctx) for each CPU, so eventually * we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids. * If number of permits N < M x nr_cpu_ids finally we will get an IO hang. * * To avoid this hang last caller of rnbd_put_permit() (last caller is the * one who observes sess->busy == 0) must wake up all remaining queues. * * Context: * Does not matter. */ static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess) { … } static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess, enum rtrs_clt_con_type con_type, enum wait_type wait) { … } static void rnbd_put_permit(struct rnbd_clt_session *sess, struct rtrs_permit *permit) { … } static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess, enum rtrs_clt_con_type con_type, enum wait_type wait) { … } static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu) { … } static void rnbd_softirq_done_fn(struct request *rq) { … } static void msg_io_conf(void *priv, int errno) { … } static void wake_up_iu_comp(struct rnbd_iu *iu, int errno) { … } static void msg_conf(void *priv, int errno) { … } static int send_usr_msg(struct rtrs_clt_sess *rtrs, int dir, struct rnbd_iu *iu, struct kvec *vec, size_t len, struct scatterlist *sg, unsigned int sg_len, void (*conf)(struct work_struct *work), int *errno, int wait) { … } static void msg_close_conf(struct work_struct *work) { … } static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, enum wait_type wait) { … } static void msg_open_conf(struct work_struct *work) { … } static void msg_sess_info_conf(struct work_struct *work) { … } static int send_msg_open(struct rnbd_clt_dev *dev, enum wait_type wait) { … } static int send_msg_sess_info(struct rnbd_clt_session *sess, enum wait_type wait) { … } static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess) { … } static void remap_devs(struct rnbd_clt_session *sess) { … } static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev) { … } static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues) { … } static void destroy_mq_tags(struct rnbd_clt_session *sess) { … } static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess) { … } static void close_rtrs(struct rnbd_clt_session *sess) { … } static void free_sess(struct rnbd_clt_session *sess) { … } static struct rnbd_clt_session *alloc_sess(const char *sessname) { … } static int wait_for_rtrs_connection(struct rnbd_clt_session *sess) { … } static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess) __releases(&sess_lock) __acquires(&sess_lock) { … } static struct rnbd_clt_session *__find_and_get_sess(const char *sessname) __releases(&sess_lock) __acquires(&sess_lock) { … } /* caller is responsible for initializing 'first' to false */ static struct rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first) { … } static int rnbd_client_open(struct gendisk *disk, blk_mode_t mode) { … } static void rnbd_client_release(struct gendisk *gen) { … } static int rnbd_client_getgeo(struct block_device *block_device, struct hd_geometry *geo) { … } static const struct block_device_operations rnbd_client_ops = …; /* The amount of data that belongs to an I/O and the amount of data that * should be read or written to the disk (bi_size) can differ. * * E.g. When WRITE_SAME is used, only a small amount of data is * transferred that is then written repeatedly over a lot of sectors. * * Get the size of data to be transferred via RTRS by summing up the size * of the scather-gather list entries. */ static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len) { … } static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev, struct request *rq, struct rnbd_iu *iu) { … } /** * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy * @dev: Device to be checked * @q: Queue to be added to the requeue list if required * * Description: * If session is busy, that means someone will requeue us when resources * are freed. If session is not doing anything - device is not added to * the list and @false is returned. */ static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev, struct rnbd_queue *q) { … } static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev, struct blk_mq_hw_ctx *hctx, int delay) { … } static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { … } static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { … } static void rnbd_rdma_map_queues(struct blk_mq_tag_set *set) { … } static struct blk_mq_ops rnbd_mq_ops = …; static int setup_mq_tags(struct rnbd_clt_session *sess) { … } static struct rnbd_clt_session * find_and_get_or_create_sess(const char *sessname, const struct rtrs_addr *paths, size_t path_cnt, u16 port_nr, u32 nr_poll_queues) { … } static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev, struct rnbd_queue *q, struct blk_mq_hw_ctx *hctx) { … } static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev) { … } static int rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, struct rnbd_msg_open_rsp *rsp, int idx) { … } static int rnbd_client_setup_device(struct rnbd_clt_dev *dev, struct rnbd_msg_open_rsp *rsp) { … } static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, enum rnbd_access_mode access_mode, const char *pathname, u32 nr_poll_queues) { … } static bool __exists_dev(const char *pathname, const char *sessname) { … } static bool exists_devpath(const char *pathname, const char *sessname) { … } static bool insert_dev_if_not_exists_devpath(struct rnbd_clt_dev *dev) { … } static void delete_dev(struct rnbd_clt_dev *dev) { … } struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname, struct rtrs_addr *paths, size_t path_cnt, u16 port_nr, const char *pathname, enum rnbd_access_mode access_mode, u32 nr_poll_queues) { … } static void destroy_gen_disk(struct rnbd_clt_dev *dev) { … } static void destroy_sysfs(struct rnbd_clt_dev *dev, const struct attribute *sysfs_self) { … } int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force, const struct attribute *sysfs_self) { … } int rnbd_clt_remap_device(struct rnbd_clt_dev *dev) { … } static void unmap_device_work(struct work_struct *work) { … } static void rnbd_destroy_sessions(void) { … } static int __init rnbd_client_init(void) { … } static void __exit rnbd_client_exit(void) { … } module_init(…) …; module_exit(rnbd_client_exit);
Codebase Browser
linux