// SPDX-License-Identifier: GPL-2.0 /* * Shared Memory Communications over RDMA (SMC-R) and RoCE * * Link Layer Control (LLC) * * Copyright IBM Corp. 2016 * * Author(s): Klaus Wacker <[email protected]> * Ursula Braun <[email protected]> */ #include <net/tcp.h> #include <rdma/ib_verbs.h> #include "smc.h" #include "smc_core.h" #include "smc_clc.h" #include "smc_llc.h" #include "smc_pnet.h" #define SMC_LLC_DATA_LEN … struct smc_llc_hdr { … } __packed; /* format defined in * IBM Shared Memory Communications Version 2 * (https://www.ibm.com/support/pages/node/6326337) */ #define SMC_LLC_FLAG_NO_RMBE_EYEC … struct smc_llc_msg_confirm_link { … }; #define SMC_LLC_FLAG_ADD_LNK_REJ … #define SMC_LLC_REJ_RSN_NO_ALT_PATH … struct smc_llc_msg_add_link { … }; struct smc_llc_msg_add_link_cont_rt { … }; struct smc_llc_msg_add_link_v2_ext { … } __packed; /* format defined in * IBM Shared Memory Communications Version 2 * (https://www.ibm.com/support/pages/node/6326337) */ struct smc_llc_msg_req_add_link_v2 { … }; #define SMC_LLC_RKEYS_PER_CONT_MSG … struct smc_llc_msg_add_link_cont { … } __packed; /* format defined in RFC7609 */ #define SMC_LLC_FLAG_DEL_LINK_ALL … #define SMC_LLC_FLAG_DEL_LINK_ORDERLY … struct smc_llc_msg_del_link { … } __packed; /* format defined in RFC7609 */ struct smc_llc_msg_test_link { … }; struct smc_rmb_rtoken { … } __packed; /* format defined in RFC7609 */ #define SMC_LLC_RKEYS_PER_MSG … #define SMC_LLC_RKEYS_PER_MSG_V2 … struct smc_llc_msg_confirm_rkey { … }; #define SMC_LLC_DEL_RKEY_MAX … #define SMC_LLC_FLAG_RKEY_RETRY … #define SMC_LLC_FLAG_RKEY_NEG … struct smc_llc_msg_delete_rkey { … }; struct smc_llc_msg_delete_rkey_v2 { … }; smc_llc_msg; #define SMC_LLC_FLAG_RESP … struct smc_llc_qentry { … }; static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) { … } void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) { … } static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, struct smc_llc_qentry *qentry) { … } static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, struct smc_llc_qentry *qentry) { … } /* try to start a new llc flow, initiated by an incoming llc msg */ static bool smc_llc_flow_start(struct smc_llc_flow *flow, struct smc_llc_qentry *qentry) { … } /* start a new local llc flow, wait till current flow finished */ int smc_llc_flow_initiate(struct smc_link_group *lgr, enum smc_llc_flowtype type) { … } /* finish the current llc flow */ void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) { … } /* lnk is optional and used for early wakeup when link goes down, useful in * cases where we wait for a response on the link after we sent a request */ struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, struct smc_link *lnk, int time_out, u8 exp_msg) { … } /********************************** send *************************************/ struct smc_llc_tx_pend { … }; /* handler for send/transmission completion of an LLC msg */ static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, struct smc_link *link, enum ib_wc_status wc_status) { … } /** * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits * @link: Pointer to SMC link used for sending LLC control message. * @wr_buf: Out variable returning pointer to work request payload buffer. * @pend: Out variable returning pointer to private pending WR tracking. * It's the context the transmit complete handler will get. * * Reserves and pre-fills an entry for a pending work request send/tx. * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. * Can sleep due to smc_get_ctrl_buf (if not in softirq context). * * Return: 0 on success, otherwise an error value. */ static int smc_llc_add_pending_send(struct smc_link *link, struct smc_wr_buf **wr_buf, struct smc_wr_tx_pend_priv **pend) { … } static int smc_llc_add_pending_send_v2(struct smc_link *link, struct smc_wr_v2_buf **wr_buf, struct smc_wr_tx_pend_priv **pend) { … } static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr, struct smc_link_group *lgr, size_t len) { … } /* high-level API to send LLC confirm link */ int smc_llc_send_confirm_link(struct smc_link *link, enum smc_llc_reqresp reqresp) { … } /* send LLC confirm rkey request */ static int smc_llc_send_confirm_rkey(struct smc_link *send_link, struct smc_buf_desc *rmb_desc) { … } /* send LLC delete rkey request */ static int smc_llc_send_delete_rkey(struct smc_link *link, struct smc_buf_desc *rmb_desc) { … } /* return first buffer from any of the next buf lists */ static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, int *buf_lst) { … } /* return next rmb from buffer lists */ static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, int *buf_lst, struct smc_buf_desc *buf_pos) { … } static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, int *buf_lst) { … } static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext, struct smc_link *link, struct smc_link *link_new) { … } /* send ADD LINK request or response */ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], struct smc_link *link_new, enum smc_llc_reqresp reqresp) { … } /* send DELETE LINK request or response */ int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, enum smc_llc_reqresp reqresp, bool orderly, u32 reason) { … } /* send LLC test link request */ static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) { … } /* schedule an llc send on link, may wait for buffers */ static int smc_llc_send_message(struct smc_link *link, void *llcbuf) { … } /* schedule an llc send on link, may wait for buffers, * and wait for send completion notification. * @return 0 on success */ static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) { … } /********************************* receive ***********************************/ static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, enum smc_lgr_type lgr_new_t) { … } /* send one add_link_continue msg */ static int smc_llc_add_link_cont(struct smc_link *link, struct smc_link *link_new, u8 *num_rkeys_todo, int *buf_lst, struct smc_buf_desc **buf_pos) { … } static int smc_llc_cli_rkey_exchange(struct smc_link *link, struct smc_link *link_new) { … } /* prepare and send an add link reject response */ static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) { … } static int smc_llc_cli_conf_link(struct smc_link *link, struct smc_init_info *ini, struct smc_link *link_new, enum smc_lgr_type lgr_new_t) { … } static void smc_llc_save_add_link_rkeys(struct smc_link *link, struct smc_link *link_new) { … } static void smc_llc_save_add_link_info(struct smc_link *link, struct smc_llc_msg_add_link *add_llc) { … } /* as an SMC client, process an add link request */ int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) { … } static void smc_llc_send_request_add_link(struct smc_link *link) { … } /* as an SMC client, invite server to start the add_link processing */ static void smc_llc_cli_add_link_invite(struct smc_link *link, struct smc_llc_qentry *qentry) { … } static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) { … } static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) { … } static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) { … } static int smc_llc_active_link_count(struct smc_link_group *lgr) { … } /* find the asymmetric link when 3 links are established */ static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) { … } static void smc_llc_delete_asym_link(struct smc_link_group *lgr) { … } static int smc_llc_srv_rkey_exchange(struct smc_link *link, struct smc_link *link_new) { … } static int smc_llc_srv_conf_link(struct smc_link *link, struct smc_link *link_new, enum smc_lgr_type lgr_new_t) { … } static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry) { … } int smc_llc_srv_add_link(struct smc_link *link, struct smc_llc_qentry *req_qentry) { … } static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) { … } /* enqueue a local add_link req to trigger a new add_link flow */ void smc_llc_add_link_local(struct smc_link *link) { … } /* worker to process an add link message */ static void smc_llc_add_link_work(struct work_struct *work) { … } /* enqueue a local del_link msg to trigger a new del_link flow, * called only for role SMC_SERV */ void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) { … } static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) { … } /* try to send a DELETE LINK ALL request on any active link, * waiting for send completion */ void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) { … } static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) { … } static void smc_llc_delete_link_work(struct work_struct *work) { … } /* process a confirm_rkey request from peer, remote flow */ static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) { … } /* process a delete_rkey request from peer, remote flow */ static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) { … } static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) { … } /* flush the llc event queue */ static void smc_llc_event_flush(struct smc_link_group *lgr) { … } static void smc_llc_event_handler(struct smc_llc_qentry *qentry) { … } /* worker to process llc messages on the event queue */ static void smc_llc_event_work(struct work_struct *work) { … } /* process llc responses in tasklet context */ static void smc_llc_rx_response(struct smc_link *link, struct smc_llc_qentry *qentry) { … } static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) { … } /* copy received msg and add it to the event queue */ static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) { … } /***************************** worker, utils *********************************/ static void smc_llc_testlink_work(struct work_struct *work) { … } void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) { … } /* called after lgr was removed from lgr_list */ void smc_llc_lgr_clear(struct smc_link_group *lgr) { … } int smc_llc_link_init(struct smc_link *link) { … } void smc_llc_link_active(struct smc_link *link) { … } /* called in worker context */ void smc_llc_link_clear(struct smc_link *link, bool log) { … } /* register a new rtoken at the remote peer (for all links) */ int smc_llc_do_confirm_rkey(struct smc_link *send_link, struct smc_buf_desc *rmb_desc) { … } /* unregister an rtoken at the remote peer */ int smc_llc_do_delete_rkey(struct smc_link_group *lgr, struct smc_buf_desc *rmb_desc) { … } void smc_llc_link_set_uid(struct smc_link *link) { … } /* save peers link user id, used for debug purposes */ void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) { … } /* evaluate confirm link request or response */ int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, enum smc_llc_reqresp type) { … } /***************************** init, exit, misc ******************************/ static struct smc_wr_rx_handler smc_llc_rx_handlers[] = …; int __init smc_llc_init(void) { … }
Codebase Browser
linux