// SPDX-License-Identifier: GPL-2.0-or-later /* * RDMA Transport Layer * * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. */ #undef pr_fmt #define pr_fmt(fmt) … #include <linux/module.h> #include <linux/rculist.h> #include <linux/random.h> #include "rtrs-clt.h" #include "rtrs-log.h" #include "rtrs-clt-trace.h" #define RTRS_CONNECT_TIMEOUT_MS … /* * Wait a bit before trying to reconnect after a failure * in order to give server time to finish clean up which * leads to "false positives" failed reconnect attempts */ #define RTRS_RECONNECT_BACKOFF … /* * Wait for additional random time between 0 and 8 seconds * before starting to reconnect to avoid clients reconnecting * all at once in case of a major network outage */ #define RTRS_RECONNECT_SEED … #define FIRST_CONN … /* limit to 128 * 4k = 512k max IO */ #define RTRS_MAX_SEGMENTS … MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; static const struct rtrs_rdma_dev_pd_ops dev_pd_ops; static struct rtrs_rdma_dev_pd dev_pd = …; static struct workqueue_struct *rtrs_wq; static const struct class rtrs_clt_dev_class = …; static inline bool rtrs_clt_is_connected(const struct rtrs_clt_sess *clt) { … } static struct rtrs_permit * __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type) { … } static inline void __rtrs_put_permit(struct rtrs_clt_sess *clt, struct rtrs_permit *permit) { … } /** * rtrs_clt_get_permit() - allocates permit for future RDMA operation * @clt: Current session * @con_type: Type of connection to use with the permit * @can_wait: Wait type * * Description: * Allocates permit for the following RDMA operation. Permit is used * to preallocate all resources and to propagate memory pressure * up earlier. * * Context: * Can sleep if @wait == RTRS_PERMIT_WAIT */ struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type, enum wait_type can_wait) { … } EXPORT_SYMBOL(…); /** * rtrs_clt_put_permit() - puts allocated permit * @clt: Current session * @permit: Permit to be freed * * Context: * Does not matter */ void rtrs_clt_put_permit(struct rtrs_clt_sess *clt, struct rtrs_permit *permit) { … } EXPORT_SYMBOL(…); /** * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit * @clt_path: client path pointer * @permit: permit for the allocation of the RDMA buffer * Note: * IO connection starts from 1. * 0 connection is for user messages. */ static struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_path *clt_path, struct rtrs_permit *permit) { … } /** * rtrs_clt_change_state() - change the session state through session state * machine. * * @clt_path: client path to change the state of. * @new_state: state to change to. * * returns true if sess's state is changed to new state, otherwise return false. * * Locks: * state_wq lock must be hold. */ static bool rtrs_clt_change_state(struct rtrs_clt_path *clt_path, enum rtrs_clt_state new_state) { … } static bool rtrs_clt_change_state_from_to(struct rtrs_clt_path *clt_path, enum rtrs_clt_state old_state, enum rtrs_clt_state new_state) { … } static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path); static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) { … } static void rtrs_clt_fast_reg_done(struct ib_cq *cq, struct ib_wc *wc) { … } static struct ib_cqe fast_reg_cqe = …; static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, bool notify, bool can_wait); static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int rtrs_inv_rkey(struct rtrs_clt_io_req *req) { … } static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, bool notify, bool can_wait) { … } static int rtrs_post_send_rdma(struct rtrs_clt_con *con, struct rtrs_clt_io_req *req, struct rtrs_rbuf *rbuf, u32 off, u32 imm, struct ib_send_wr *wr) { … } static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, s16 errno, bool w_inval) { … } static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc) { … } static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) { … } static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc); static struct ib_cqe io_comp_cqe = …; /* * Post x2 empty WRs: first is for this RDMA with IMM, * second is for RECV with INV, which happened earlier. */ static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe) { … } static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int post_recv_io(struct rtrs_clt_con *con, size_t q_size) { … } static int post_recv_path(struct rtrs_clt_path *clt_path) { … } struct path_it { … }; /* * rtrs_clt_get_next_path_or_null - get clt path from the list or return NULL * @head: the head for the list. * @clt_path: The element to take the next clt_path from. * * Next clt path returned in round-robin fashion, i.e. head will be skipped, * but if list is observed as empty, NULL will be returned. * * This function may safely run concurrently with the _rcu list-mutation * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). */ static inline struct rtrs_clt_path * rtrs_clt_get_next_path_or_null(struct list_head *head, struct rtrs_clt_path *clt_path) { … } /** * get_next_path_rr() - Returns path in round-robin fashion. * @it: the path pointer * * Related to @MP_POLICY_RR * * Locks: * rcu_read_lock() must be held. */ static struct rtrs_clt_path *get_next_path_rr(struct path_it *it) { … } /** * get_next_path_min_inflight() - Returns path with minimal inflight count. * @it: the path pointer * * Related to @MP_POLICY_MIN_INFLIGHT * * Locks: * rcu_read_lock() must be hold. */ static struct rtrs_clt_path *get_next_path_min_inflight(struct path_it *it) { … } /** * get_next_path_min_latency() - Returns path with minimal latency. * @it: the path pointer * * Return: a path with the lowest latency or NULL if all paths are tried * * Locks: * rcu_read_lock() must be hold. * * Related to @MP_POLICY_MIN_LATENCY * * This DOES skip an already-tried path. * There is a skip-list to skip a path if the path has tried but failed. * It will try the minimum latency path and then the second minimum latency * path and so on. Finally it will return NULL if all paths are tried. * Therefore the caller MUST check the returned * path is NULL and trigger the IO error. */ static struct rtrs_clt_path *get_next_path_min_latency(struct path_it *it) { … } static inline void path_it_init(struct path_it *it, struct rtrs_clt_sess *clt) { … } static inline void path_it_deinit(struct path_it *it) { … } /** * rtrs_clt_init_req() - Initialize an rtrs_clt_io_req holding information * about an inflight IO. * The user buffer holding user control message (not data) is copied into * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will * also hold the control message of rtrs. * @req: an io request holding information about IO. * @clt_path: client path * @conf: conformation callback function to notify upper layer. * @permit: permit for allocation of RDMA remote buffer * @priv: private pointer * @vec: kernel vector containing control message * @usr_len: length of the user message * @sg: scater list for IO data * @sg_cnt: number of scater list entries * @data_len: length of the IO data * @dir: direction of the IO. */ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, struct rtrs_clt_path *clt_path, void (*conf)(void *priv, int errno), struct rtrs_permit *permit, void *priv, const struct kvec *vec, size_t usr_len, struct scatterlist *sg, size_t sg_cnt, size_t data_len, int dir) { … } static struct rtrs_clt_io_req * rtrs_clt_get_req(struct rtrs_clt_path *clt_path, void (*conf)(void *priv, int errno), struct rtrs_permit *permit, void *priv, const struct kvec *vec, size_t usr_len, struct scatterlist *sg, size_t sg_cnt, size_t data_len, int dir) { … } static struct rtrs_clt_io_req * rtrs_clt_get_copy_req(struct rtrs_clt_path *alive_path, struct rtrs_clt_io_req *fail_req) { … } static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, struct rtrs_clt_io_req *req, struct rtrs_rbuf *rbuf, bool fr_en, u32 count, u32 size, u32 imm, struct ib_send_wr *wr, struct ib_send_wr *tail) { … } static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count) { … } static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) { … } static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) { … } /** * rtrs_clt_failover_req() - Try to find an active path for a failed request * @clt: clt context * @fail_req: a failed io request. */ static int rtrs_clt_failover_req(struct rtrs_clt_sess *clt, struct rtrs_clt_io_req *fail_req) { … } static void fail_all_outstanding_reqs(struct rtrs_clt_path *clt_path) { … } static void free_path_reqs(struct rtrs_clt_path *clt_path) { … } static int alloc_path_reqs(struct rtrs_clt_path *clt_path) { … } static int alloc_permits(struct rtrs_clt_sess *clt) { … } static void free_permits(struct rtrs_clt_sess *clt) { … } static void query_fast_reg_mode(struct rtrs_clt_path *clt_path) { … } static bool rtrs_clt_change_state_get_old(struct rtrs_clt_path *clt_path, enum rtrs_clt_state new_state, enum rtrs_clt_state *old_state) { … } static void rtrs_clt_hb_err_handler(struct rtrs_con *c) { … } static void rtrs_clt_init_hb(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_reconnect_work(struct work_struct *work); static void rtrs_clt_close_work(struct work_struct *work); static void rtrs_clt_err_recovery_work(struct work_struct *work) { … } static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, const struct rtrs_addr *path, size_t con_num, u32 nr_poll_queues) { … } void free_path(struct rtrs_clt_path *clt_path) { … } static int create_con(struct rtrs_clt_path *clt_path, unsigned int cid) { … } static void destroy_con(struct rtrs_clt_con *con) { … } static int create_con_cq_qp(struct rtrs_clt_con *con) { … } static void destroy_con_cq_qp(struct rtrs_clt_con *con) { … } static void stop_cm(struct rtrs_clt_con *con) { … } static void destroy_cm(struct rtrs_clt_con *con) { … } static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) { … } static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) { … } static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, struct rdma_cm_event *ev) { … } static inline void flag_success_on_conn(struct rtrs_clt_con *con) { … } static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, struct rdma_cm_event *ev) { … } void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) { … } static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err) { … } static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *ev) { … } /* The caller should do the cleanup in case of error */ static int create_cm(struct rtrs_clt_con *con) { … } static void rtrs_clt_path_up(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_path_down(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_add_path_to_arr(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_close_work(struct work_struct *work) { … } static int init_conns(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int process_info_rsp(struct rtrs_clt_path *clt_path, const struct rtrs_msg_info_rsp *msg) { … } static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int rtrs_send_path_info(struct rtrs_clt_path *clt_path) { … } /** * init_path() - establishes all path connections and does handshake * @clt_path: client path. * In case of error full close or reconnect procedure should be taken, * because reconnect or close async works can be started. */ static int init_path(struct rtrs_clt_path *clt_path) { … } static void rtrs_clt_reconnect_work(struct work_struct *work) { … } static void rtrs_clt_dev_release(struct device *dev) { … } static struct rtrs_clt_sess *alloc_clt(const char *sessname, size_t paths_num, u16 port, size_t pdu_sz, void *priv, void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev), unsigned int reconnect_delay_sec, unsigned int max_reconnect_attempts) { … } static void free_clt(struct rtrs_clt_sess *clt) { … } /** * rtrs_clt_open() - Open a path to an RTRS server * @ops: holds the link event callback and the private pointer. * @pathname: name of the path to an RTRS server * @paths: Paths to be established defined by their src and dst addresses * @paths_num: Number of elements in the @paths array * @port: port to be used by the RTRS session * @pdu_sz: Size of extra payload which can be accessed after permit allocation. * @reconnect_delay_sec: time between reconnect tries * @max_reconnect_attempts: Number of times to reconnect on error before giving * up, 0 for * disabled, -1 for forever * @nr_poll_queues: number of polling mode connection using IB_POLL_DIRECT flag * * Starts session establishment with the rtrs_server. The function can block * up to ~2000ms before it returns. * * Return a valid pointer on success otherwise PTR_ERR. */ struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, const char *pathname, const struct rtrs_addr *paths, size_t paths_num, u16 port, size_t pdu_sz, u8 reconnect_delay_sec, s16 max_reconnect_attempts, u32 nr_poll_queues) { … } EXPORT_SYMBOL(…); /** * rtrs_clt_close() - Close a path * @clt: Session handle. Session is freed upon return. */ void rtrs_clt_close(struct rtrs_clt_sess *clt) { … } EXPORT_SYMBOL(…); int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *clt_path) { … } int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *clt_path, const struct attribute *sysfs_self) { … } void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value) { … } int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt) { … } /** * rtrs_clt_request() - Request data transfer to/from server via RDMA. * * @dir: READ/WRITE * @ops: callback function to be called as confirmation, and the pointer. * @clt: Session * @permit: Preallocated permit * @vec: Message that is sent to server together with the request. * Sum of len of all @vec elements limited to <= IO_MSG_SIZE. * Since the msg is copied internally it can be allocated on stack. * @nr: Number of elements in @vec. * @data_len: length of data sent to/from server * @sg: Pages to be sent/received to/from server. * @sg_cnt: Number of elements in the @sg * * Return: * 0: Success * <0: Error * * On dir=READ rtrs client will request a data transfer from Server to client. * The data that the server will respond with will be stored in @sg when * the user receives an %RTRS_CLT_RDMA_EV_RDMA_REQUEST_WRITE_COMPL event. * On dir=WRITE rtrs client will rdma write data in sg to server side. */ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, struct rtrs_clt_sess *clt, struct rtrs_permit *permit, const struct kvec *vec, size_t nr, size_t data_len, struct scatterlist *sg, unsigned int sg_cnt) { … } EXPORT_SYMBOL(…); int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index) { … } EXPORT_SYMBOL(…); /** * rtrs_clt_query() - queries RTRS session attributes *@clt: session pointer *@attr: query results for session attributes. * Returns: * 0 on success * -ECOMM no connection to the server */ int rtrs_clt_query(struct rtrs_clt_sess *clt, struct rtrs_attrs *attr) { … } EXPORT_SYMBOL(…); int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt, struct rtrs_addr *addr) { … } static int rtrs_clt_ib_dev_init(struct rtrs_ib_dev *dev) { … } static const struct rtrs_rdma_dev_pd_ops dev_pd_ops = …; static int __init rtrs_client_init(void) { … } static void __exit rtrs_client_exit(void) { … } module_init(…) …; module_exit(rtrs_client_exit);
Codebase Browser
linux