// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2017, Microsoft Corporation. * Copyright (C) 2018, LG Electronics. * * Author(s): Long Li <[email protected]>, * Hyunchul Lee <[email protected]> */ #define SUBMOD_NAME … #include <linux/kthread.h> #include <linux/list.h> #include <linux/mempool.h> #include <linux/highmem.h> #include <linux/scatterlist.h> #include <rdma/ib_verbs.h> #include <rdma/rdma_cm.h> #include <rdma/rw.h> #include "glob.h" #include "connection.h" #include "smb_common.h" #include "smbstatus.h" #include "transport_rdma.h" #define SMB_DIRECT_PORT_IWARP … #define SMB_DIRECT_PORT_INFINIBAND … #define SMB_DIRECT_VERSION_LE … /* SMB_DIRECT negotiation timeout in seconds */ #define SMB_DIRECT_NEGOTIATE_TIMEOUT … #define SMB_DIRECT_MAX_SEND_SGES … #define SMB_DIRECT_MAX_RECV_SGES … /* * Default maximum number of RDMA read/write outstanding on this connection * This value is possibly decreased during QP creation on hardware limit */ #define SMB_DIRECT_CM_INITIATOR_DEPTH … /* Maximum number of retries on data transfer operations */ #define SMB_DIRECT_CM_RETRY … /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */ #define SMB_DIRECT_CM_RNR_RETRY … /* * User configurable initial values per SMB_DIRECT transport connection * as defined in [MS-SMBD] 3.1.1.1 * Those may change after a SMB_DIRECT negotiation */ /* Set 445 port to SMB Direct port by default */ static int smb_direct_port = …; /* The local peer's maximum number of credits to grant to the peer */ static int smb_direct_receive_credit_max = …; /* The remote peer's credit request of local peer */ static int smb_direct_send_credit_target = …; /* The maximum single message size can be sent to remote peer */ static int smb_direct_max_send_size = …; /* The maximum fragmented upper-layer payload receive size supported */ static int smb_direct_max_fragmented_recv_size = …; /* The maximum single-message size which can be received */ static int smb_direct_max_receive_size = …; static int smb_direct_max_read_write_size = …; static LIST_HEAD(smb_direct_device_list); static DEFINE_RWLOCK(smb_direct_device_lock); struct smb_direct_device { … }; static struct smb_direct_listener { … } smb_direct_listener; static struct workqueue_struct *smb_direct_wq; enum smb_direct_status { … }; struct smb_direct_transport { … }; #define KSMBD_TRANS(t) … enum { … }; static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; struct smb_direct_send_ctx { … }; struct smb_direct_sendmsg { … }; struct smb_direct_recvmsg { … }; struct smb_direct_rdma_rw_msg { … }; void init_smbd_max_io_size(unsigned int sz) { … } unsigned int get_smbd_max_read_write_size(void) { … } static inline int get_buf_page_count(void *buf, int size) { … } static void smb_direct_destroy_pools(struct smb_direct_transport *transport); static void smb_direct_post_recv_credits(struct work_struct *work); static int smb_direct_post_send_data(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx, struct kvec *iov, int niov, int remaining_data_length); static inline struct smb_direct_transport * smb_trans_direct_transfort(struct ksmbd_transport *t) { … } static inline void *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg) { … } static inline bool is_receive_credit_post_required(int receive_credits, int avail_recvmsg_count) { … } static struct smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) { … } static void put_recvmsg(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg) { … } static struct smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t) { … } static void put_empty_recvmsg(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg) { … } static void enqueue_reassembly(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg, int data_length) { … } static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t) { … } static void smb_direct_disconnect_rdma_work(struct work_struct *work) { … } static void smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t) { … } static void smb_direct_send_immediate_work(struct work_struct *work) { … } static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) { … } static void free_transport(struct smb_direct_transport *t) { … } static struct smb_direct_sendmsg *smb_direct_alloc_sendmsg(struct smb_direct_transport *t) { … } static void smb_direct_free_sendmsg(struct smb_direct_transport *t, struct smb_direct_sendmsg *msg) { … } static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg) { … } static void recv_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int smb_direct_post_recv(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg) { … } static int smb_direct_read(struct ksmbd_transport *t, char *buf, unsigned int size, int unused) { … } static void smb_direct_post_recv_credits(struct work_struct *work) { … } static void send_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int manage_credits_prior_sending(struct smb_direct_transport *t) { … } static int smb_direct_post_send(struct smb_direct_transport *t, struct ib_send_wr *wr) { … } static void smb_direct_send_ctx_init(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx, bool need_invalidate_rkey, unsigned int remote_key) { … } static int smb_direct_flush_send_list(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx, bool is_last) { … } static int wait_for_credits(struct smb_direct_transport *t, wait_queue_head_t *waitq, atomic_t *total_credits, int needed) { … } static int wait_for_send_credits(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx) { … } static int wait_for_rw_credits(struct smb_direct_transport *t, int credits) { … } static int calc_rw_credits(struct smb_direct_transport *t, char *buf, unsigned int len) { … } static int smb_direct_create_header(struct smb_direct_transport *t, int size, int remaining_data_length, struct smb_direct_sendmsg **sendmsg_out) { … } static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries) { … } static int get_mapped_sg_list(struct ib_device *device, void *buf, int size, struct scatterlist *sg_list, int nentries, enum dma_data_direction dir) { … } static int post_sendmsg(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx, struct smb_direct_sendmsg *msg) { … } static int smb_direct_post_send_data(struct smb_direct_transport *t, struct smb_direct_send_ctx *send_ctx, struct kvec *iov, int niov, int remaining_data_length) { … } static int smb_direct_writev(struct ksmbd_transport *t, struct kvec *iov, int niovs, int buflen, bool need_invalidate, unsigned int remote_key) { … } static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t, struct smb_direct_rdma_rw_msg *msg, enum dma_data_direction dir) { … } static void read_write_done(struct ib_cq *cq, struct ib_wc *wc, enum dma_data_direction dir) { … } static void read_done(struct ib_cq *cq, struct ib_wc *wc) { … } static void write_done(struct ib_cq *cq, struct ib_wc *wc) { … } static int smb_direct_rdma_xmit(struct smb_direct_transport *t, void *buf, int buf_len, struct smb2_buffer_desc_v1 *desc, unsigned int desc_len, bool is_read) { … } static int smb_direct_rdma_write(struct ksmbd_transport *t, void *buf, unsigned int buflen, struct smb2_buffer_desc_v1 *desc, unsigned int desc_len) { … } static int smb_direct_rdma_read(struct ksmbd_transport *t, void *buf, unsigned int buflen, struct smb2_buffer_desc_v1 *desc, unsigned int desc_len) { … } static void smb_direct_disconnect(struct ksmbd_transport *t) { … } static void smb_direct_shutdown(struct ksmbd_transport *t) { … } static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { … } static void smb_direct_qpair_handler(struct ib_event *event, void *context) { … } static int smb_direct_send_negotiate_response(struct smb_direct_transport *t, int failed) { … } static int smb_direct_accept_client(struct smb_direct_transport *t) { … } static int smb_direct_prepare_negotiation(struct smb_direct_transport *t) { … } static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t) { … } static int smb_direct_init_params(struct smb_direct_transport *t, struct ib_qp_cap *cap) { … } static void smb_direct_destroy_pools(struct smb_direct_transport *t) { … } static int smb_direct_create_pools(struct smb_direct_transport *t) { … } static int smb_direct_create_qpair(struct smb_direct_transport *t, struct ib_qp_cap *cap) { … } static int smb_direct_prepare(struct ksmbd_transport *t) { … } static int smb_direct_connect(struct smb_direct_transport *st) { … } static bool rdma_frwr_is_supported(struct ib_device_attr *attrs) { … } static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) { … } static int smb_direct_listen_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { … } static int smb_direct_listen(int port) { … } static int smb_direct_ib_client_add(struct ib_device *ib_dev) { … } static void smb_direct_ib_client_remove(struct ib_device *ib_dev, void *client_data) { … } static struct ib_client smb_direct_ib_client = …; int ksmbd_rdma_init(void) { … } void ksmbd_rdma_destroy(void) { … } bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { … } static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = …;
Codebase Browser
linux