// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2017, Microsoft Corporation. * * Author(s): Long Li <[email protected]> */ #include <linux/module.h> #include <linux/highmem.h> #include "smbdirect.h" #include "cifs_debug.h" #include "cifsproto.h" #include "smb2proto.h" static struct smbd_response *get_empty_queue_buffer( struct smbd_connection *info); static struct smbd_response *get_receive_buffer( struct smbd_connection *info); static void put_receive_buffer( struct smbd_connection *info, struct smbd_response *response); static int allocate_receive_buffers(struct smbd_connection *info, int num_buf); static void destroy_receive_buffers(struct smbd_connection *info); static void put_empty_packet( struct smbd_connection *info, struct smbd_response *response); static void enqueue_reassembly( struct smbd_connection *info, struct smbd_response *response, int data_length); static struct smbd_response *_get_first_reassembly( struct smbd_connection *info); static int smbd_post_recv( struct smbd_connection *info, struct smbd_response *response); static int smbd_post_send_empty(struct smbd_connection *info); static void destroy_mr_list(struct smbd_connection *info); static int allocate_mr_list(struct smbd_connection *info); struct smb_extract_to_rdma { … }; static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len, struct smb_extract_to_rdma *rdma); /* SMBD version number */ #define SMBD_V1 … /* Port numbers for SMBD transport */ #define SMB_PORT … #define SMBD_PORT … /* Address lookup and resolve timeout in ms */ #define RDMA_RESOLVE_TIMEOUT … /* SMBD negotiation timeout in seconds */ #define SMBD_NEGOTIATE_TIMEOUT … /* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ #define SMBD_MIN_RECEIVE_SIZE … #define SMBD_MIN_FRAGMENTED_SIZE … /* * Default maximum number of RDMA read/write outstanding on this connection * This value is possibly decreased during QP creation on hardware limit */ #define SMBD_CM_RESPONDER_RESOURCES … /* Maximum number of retries on data transfer operations */ #define SMBD_CM_RETRY … /* No need to retry on Receiver Not Ready since SMBD manages credits */ #define SMBD_CM_RNR_RETRY … /* * User configurable initial values per SMBD transport connection * as defined in [MS-SMBD] 3.1.1.1 * Those may change after a SMBD negotiation */ /* The local peer's maximum number of credits to grant to the peer */ int smbd_receive_credit_max = …; /* The remote peer's credit request of local peer */ int smbd_send_credit_target = …; /* The maximum single message size can be sent to remote peer */ int smbd_max_send_size = …; /* The maximum fragmented upper-layer payload receive size supported */ int smbd_max_fragmented_recv_size = …; /* The maximum single-message size which can be received */ int smbd_max_receive_size = …; /* The timeout to initiate send of a keepalive message on idle */ int smbd_keep_alive_interval = …; /* * User configurable initial values for RDMA transport * The actual values used may be lower and are limited to hardware capabilities */ /* Default maximum number of pages in a single RDMA write/read */ int smbd_max_frmr_depth = …; /* If payload is less than this byte, use RDMA send/recv not read/write */ int rdma_readwrite_threshold = …; /* Transport logging functions * Logging are defined as classes. They can be OR'ed to define the actual * logging level via module parameter smbd_logging_class * e.g. cifs.smbd_logging_class=0xa0 will log all log_rdma_recv() and * log_rdma_event() */ #define LOG_OUTGOING … #define LOG_INCOMING … #define LOG_READ … #define LOG_WRITE … #define LOG_RDMA_SEND … #define LOG_RDMA_RECV … #define LOG_KEEP_ALIVE … #define LOG_RDMA_EVENT … #define LOG_RDMA_MR … static unsigned int smbd_logging_class; module_param(smbd_logging_class, uint, 0644); MODULE_PARM_DESC(…) …; #define ERR … #define INFO … static unsigned int smbd_logging_level = …; module_param(smbd_logging_level, uint, 0644); MODULE_PARM_DESC(…) …; #define log_rdma(level, class, fmt, args...) … #define log_outgoing(level, fmt, args...) … #define log_incoming(level, fmt, args...) … #define log_read(level, fmt, args...) … #define log_write(level, fmt, args...) … #define log_rdma_send(level, fmt, args...) … #define log_rdma_recv(level, fmt, args...) … #define log_keep_alive(level, fmt, args...) … #define log_rdma_event(level, fmt, args...) … #define log_rdma_mr(level, fmt, args...) … static void smbd_disconnect_rdma_work(struct work_struct *work) { … } static void smbd_disconnect_rdma_connection(struct smbd_connection *info) { … } /* Upcall from RDMA CM */ static int smbd_conn_upcall( struct rdma_cm_id *id, struct rdma_cm_event *event) { … } /* Upcall from RDMA QP */ static void smbd_qp_async_error_upcall(struct ib_event *event, void *context) { … } static inline void *smbd_request_payload(struct smbd_request *request) { … } static inline void *smbd_response_payload(struct smbd_response *response) { … } /* Called when a RDMA send is done */ static void send_done(struct ib_cq *cq, struct ib_wc *wc) { … } static void dump_smbd_negotiate_resp(struct smbd_negotiate_resp *resp) { … } /* * Process a negotiation response message, according to [MS-SMBD]3.1.5.7 * response, packet_length: the negotiation response message * return value: true if negotiation is a success, false if failed */ static bool process_negotiation_response( struct smbd_response *response, int packet_length) { … } static void smbd_post_send_credits(struct work_struct *work) { … } /* Called from softirq, when recv is done */ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) { … } static struct rdma_cm_id *smbd_create_id( struct smbd_connection *info, struct sockaddr *dstaddr, int port) { … } /* * Test if FRWR (Fast Registration Work Requests) is supported on the device * This implementation requries FRWR on RDMA read/write * return value: true if it is supported */ static bool frwr_is_supported(struct ib_device_attr *attrs) { … } static int smbd_ia_open( struct smbd_connection *info, struct sockaddr *dstaddr, int port) { … } /* * Send a negotiation request message to the peer * The negotiation procedure is in [MS-SMBD] 3.1.5.2 and 3.1.5.3 * After negotiation, the transport is connected and ready for * carrying upper layer SMB payload */ static int smbd_post_send_negotiate_req(struct smbd_connection *info) { … } /* * Extend the credits to remote peer * This implements [MS-SMBD] 3.1.5.9 * The idea is that we should extend credits to remote peer as quickly as * it's allowed, to maintain data flow. We allocate as much receive * buffer as possible, and extend the receive credits to remote peer * return value: the new credtis being granted. */ static int manage_credits_prior_sending(struct smbd_connection *info) { … } /* * Check if we need to send a KEEP_ALIVE message * The idle connection timer triggers a KEEP_ALIVE message when expires * SMB_DIRECT_RESPONSE_REQUESTED is set in the message flag to have peer send * back a response. * return value: * 1 if SMB_DIRECT_RESPONSE_REQUESTED needs to be set * 0: otherwise */ static int manage_keep_alive_before_sending(struct smbd_connection *info) { … } /* Post the send request */ static int smbd_post_send(struct smbd_connection *info, struct smbd_request *request) { … } static int smbd_post_send_iter(struct smbd_connection *info, struct iov_iter *iter, int *_remaining_data_length) { … } /* * Send an empty message * Empty message is used to extend credits to peer to for keep live * while there is no upper layer payload to send at the time */ static int smbd_post_send_empty(struct smbd_connection *info) { … } /* * Post a receive request to the transport * The remote peer can only send data when a receive request is posted * The interaction is controlled by send/receive credit system */ static int smbd_post_recv( struct smbd_connection *info, struct smbd_response *response) { … } /* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */ static int smbd_negotiate(struct smbd_connection *info) { … } static void put_empty_packet( struct smbd_connection *info, struct smbd_response *response) { … } /* * Implement Connection.FragmentReassemblyBuffer defined in [MS-SMBD] 3.1.1.1 * This is a queue for reassembling upper layer payload and present to upper * layer. All the inncoming payload go to the reassembly queue, regardless of * if reassembly is required. The uuper layer code reads from the queue for all * incoming payloads. * Put a received packet to the reassembly queue * response: the packet received * data_length: the size of payload in this packet */ static void enqueue_reassembly( struct smbd_connection *info, struct smbd_response *response, int data_length) { … } /* * Get the first entry at the front of reassembly queue * Caller is responsible for locking * return value: the first entry if any, NULL if queue is empty */ static struct smbd_response *_get_first_reassembly(struct smbd_connection *info) { … } static struct smbd_response *get_empty_queue_buffer( struct smbd_connection *info) { … } /* * Get a receive buffer * For each remote send, we need to post a receive. The receive buffers are * pre-allocated in advance. * return value: the receive buffer, NULL if none is available */ static struct smbd_response *get_receive_buffer(struct smbd_connection *info) { … } /* * Return a receive buffer * Upon returning of a receive buffer, we can post new receive and extend * more receive credits to remote peer. This is done immediately after a * receive buffer is returned. */ static void put_receive_buffer( struct smbd_connection *info, struct smbd_response *response) { … } /* Preallocate all receive buffer on transport establishment */ static int allocate_receive_buffers(struct smbd_connection *info, int num_buf) { … } static void destroy_receive_buffers(struct smbd_connection *info) { … } /* Implement idle connection timer [MS-SMBD] 3.1.6.2 */ static void idle_connection_timer(struct work_struct *work) { … } /* * Destroy the transport and related RDMA and memory resources * Need to go through all the pending counters and make sure on one is using * the transport while it is destroyed */ void smbd_destroy(struct TCP_Server_Info *server) { … } /* * Reconnect this SMBD connection, called from upper layer * return value: 0 on success, or actual error code */ int smbd_reconnect(struct TCP_Server_Info *server) { … } static void destroy_caches_and_workqueue(struct smbd_connection *info) { … } #define MAX_NAME_LEN … static int allocate_caches_and_workqueue(struct smbd_connection *info) { … } /* Create a SMBD connection, called by upper layer */ static struct smbd_connection *_smbd_get_connection( struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port) { … } struct smbd_connection *smbd_get_connection( struct TCP_Server_Info *server, struct sockaddr *dstaddr) { … } /* * Receive data from receive reassembly queue * All the incoming data packets are placed in reassembly queue * buf: the buffer to read data into * size: the length of data to read * return value: actual data read * Note: this implementation copies the data from reassebmly queue to receive * buffers used by upper layer. This is not the optimal code path. A better way * to do it is to not have upper layer allocate its receive buffers but rather * borrow the buffer from reassembly queue, and return it after data is * consumed. But this will require more changes to upper layer code, and also * need to consider packet boundaries while they still being reassembled. */ static int smbd_recv_buf(struct smbd_connection *info, char *buf, unsigned int size) { … } /* * Receive a page from receive reassembly queue * page: the page to read data into * to_read: the length of data to read * return value: actual data read */ static int smbd_recv_page(struct smbd_connection *info, struct page *page, unsigned int page_offset, unsigned int to_read) { … } /* * Receive data from transport * msg: a msghdr point to the buffer, can be ITER_KVEC or ITER_BVEC * return: total bytes read, or 0. SMB Direct will not do partial read. */ int smbd_recv(struct smbd_connection *info, struct msghdr *msg) { … } /* * Send data to transport * Each rqst is transported as a SMBDirect payload * rqst: the data to write * return value: 0 if successfully write, otherwise error code */ int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst_array) { … } static void register_mr_done(struct ib_cq *cq, struct ib_wc *wc) { … } /* * The work queue function that recovers MRs * We need to call ib_dereg_mr() and ib_alloc_mr() before this MR can be used * again. Both calls are slow, so finish them in a workqueue. This will not * block I/O path. * There is one workqueue that recovers MRs, there is no need to lock as the * I/O requests calling smbd_register_mr will never update the links in the * mr_list. */ static void smbd_mr_recovery_work(struct work_struct *work) { … } static void destroy_mr_list(struct smbd_connection *info) { … } /* * Allocate MRs used for RDMA read/write * The number of MRs will not exceed hardware capability in responder_resources * All MRs are kept in mr_list. The MR can be recovered after it's used * Recovery is done in smbd_mr_recovery_work. The content of list entry changes * as MRs are used and recovered for I/O, but the list links will not change */ static int allocate_mr_list(struct smbd_connection *info) { … } /* * Get a MR from mr_list. This function waits until there is at least one * MR available in the list. It may access the list while the * smbd_mr_recovery_work is recovering the MR list. This doesn't need a lock * as they never modify the same places. However, there may be several CPUs * issueing I/O trying to get MR at the same time, mr_list_lock is used to * protect this situation. */ static struct smbd_mr *get_mr(struct smbd_connection *info) { … } /* * Transcribe the pages from an iterator into an MR scatterlist. */ static int smbd_iter_to_mr(struct smbd_connection *info, struct iov_iter *iter, struct sg_table *sgt, unsigned int max_sg) { … } /* * Register memory for RDMA read/write * iter: the buffer to register memory with * writing: true if this is a RDMA write (SMB read), false for RDMA read * need_invalidate: true if this MR needs to be locally invalidated after I/O * return value: the MR registered, NULL if failed. */ struct smbd_mr *smbd_register_mr(struct smbd_connection *info, struct iov_iter *iter, bool writing, bool need_invalidate) { … } static void local_inv_done(struct ib_cq *cq, struct ib_wc *wc) { … } /* * Deregister a MR after I/O is done * This function may wait if remote invalidation is not used * and we have to locally invalidate the buffer to prevent data is being * modified by remote peer after upper layer consumes it */ int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) { … } static bool smb_set_sge(struct smb_extract_to_rdma *rdma, struct page *lowest_page, size_t off, size_t len) { … } /* * Extract page fragments from a BVEC-class iterator and add them to an RDMA * element list. The pages are not pinned. */ static ssize_t smb_extract_bvec_to_rdma(struct iov_iter *iter, struct smb_extract_to_rdma *rdma, ssize_t maxsize) { … } /* * Extract fragments from a KVEC-class iterator and add them to an RDMA list. * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers. * The pages are not pinned. */ static ssize_t smb_extract_kvec_to_rdma(struct iov_iter *iter, struct smb_extract_to_rdma *rdma, ssize_t maxsize) { … } /* * Extract folio fragments from an XARRAY-class iterator and add them to an * RDMA list. The folios are not pinned. */ static ssize_t smb_extract_xarray_to_rdma(struct iov_iter *iter, struct smb_extract_to_rdma *rdma, ssize_t maxsize) { … } /* * Extract page fragments from up to the given amount of the source iterator * and build up an RDMA list that refers to all of those bits. The RDMA list * is appended to, up to the maximum number of elements set in the parameter * block. * * The extracted page fragments are not pinned or ref'd in any way; if an * IOVEC/UBUF-type iterator is to be used, it should be converted to a * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some * way. */ static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len, struct smb_extract_to_rdma *rdma) { … }
Codebase Browser
linux