// SPDX-License-Identifier: GPL-2.0-only /* * Shared Memory Communications over RDMA (SMC-R) and RoCE * * AF_SMC protocol family socket handler keeping the AF_INET sock address type * applies to SOCK_STREAM sockets only * offers an alternative communication option for TCP-protocol sockets * applicable with RoCE-cards only * * Initial restrictions: * - support for alternate links postponed * * Copyright IBM Corp. 2016, 2018 * * Author(s): Ursula Braun <[email protected]> * based on prototype from Frank Blaschka */ #define KMSG_COMPONENT … #define pr_fmt(fmt) … #include <linux/module.h> #include <linux/socket.h> #include <linux/workqueue.h> #include <linux/in.h> #include <linux/sched/signal.h> #include <linux/if_vlan.h> #include <linux/rcupdate_wait.h> #include <linux/ctype.h> #include <linux/splice.h> #include <net/sock.h> #include <net/tcp.h> #include <net/smc.h> #include <asm/ioctls.h> #include <net/net_namespace.h> #include <net/netns/generic.h> #include "smc_netns.h" #include "smc.h" #include "smc_clc.h" #include "smc_llc.h" #include "smc_cdc.h" #include "smc_core.h" #include "smc_ib.h" #include "smc_ism.h" #include "smc_pnet.h" #include "smc_netlink.h" #include "smc_tx.h" #include "smc_rx.h" #include "smc_close.h" #include "smc_stats.h" #include "smc_tracepoint.h" #include "smc_sysctl.h" #include "smc_loopback.h" #include "smc_inet.h" static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group * creation on server */ static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group * creation on client */ static struct workqueue_struct *smc_tcp_ls_wq; /* wq for tcp listen work */ struct workqueue_struct *smc_hs_wq; /* wq for handshake work */ struct workqueue_struct *smc_close_wq; /* wq for close work */ static void smc_tcp_listen_work(struct work_struct *); static void smc_connect_work(struct work_struct *); int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb) { … } int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info) { … } int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info) { … } static void smc_set_keepalive(struct sock *sk, int val) { … } static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, bool *own_req) { … } static bool smc_hs_congested(const struct sock *sk) { … } struct smc_hashinfo smc_v4_hashinfo = …; struct smc_hashinfo smc_v6_hashinfo = …; int smc_hash_sk(struct sock *sk) { … } void smc_unhash_sk(struct sock *sk) { … } /* This will be called before user really release sock_lock. So do the * work which we didn't do because of user hold the sock_lock in the * BH context */ void smc_release_cb(struct sock *sk) { … } struct proto smc_proto = …; EXPORT_SYMBOL_GPL(…); struct proto smc_proto6 = …; EXPORT_SYMBOL_GPL(…); static void smc_fback_restore_callbacks(struct smc_sock *smc) { … } static void smc_restore_fallback_changes(struct smc_sock *smc) { … } static int __smc_release(struct smc_sock *smc) { … } int smc_release(struct socket *sock) { … } static void smc_destruct(struct sock *sk) { … } void smc_sk_init(struct net *net, struct sock *sk, int protocol) { … } static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, int protocol) { … } int smc_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { … } /* copy only relevant settings and flags of SOL_SOCKET level from smc to * clc socket (since smc is not called for these options from net/core) */ #define SK_FLAGS_SMC_TO_CLC … /* if set, use value set by setsockopt() - else use IPv4 or SMC sysctl value */ static void smc_adjust_sock_bufsizes(struct sock *nsk, struct sock *osk, unsigned long mask) { … } static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk, unsigned long mask) { … } static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) { … } #define SK_FLAGS_CLC_TO_SMC … /* copy only settings and flags relevant for smc from clc to smc socket */ static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) { … } /* register the new vzalloced sndbuf on all links */ static int smcr_lgr_reg_sndbufs(struct smc_link *link, struct smc_buf_desc *snd_desc) { … } /* register the new rmb on all links */ static int smcr_lgr_reg_rmbs(struct smc_link *link, struct smc_buf_desc *rmb_desc) { … } static int smcr_clnt_conf_first_link(struct smc_sock *smc) { … } static bool smc_isascii(char *hostname) { … } static void smc_conn_save_peer_info_fce(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { … } static void smcr_conn_save_peer_info(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { … } static void smcd_conn_save_peer_info(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { … } static void smc_conn_save_peer_info(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { … } static void smc_link_save_peer_info(struct smc_link *link, struct smc_clc_msg_accept_confirm *clc, struct smc_init_info *ini) { … } static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc, struct smc_stats_fback *fback_arr) { … } static void smc_stat_fallback(struct smc_sock *smc) { … } /* must be called under rcu read lock */ static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key) { … } static int smc_fback_mark_woken(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key) { … } static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk, void (*clcsock_callback)(struct sock *sk)) { … } static void smc_fback_state_change(struct sock *clcsk) { … } static void smc_fback_data_ready(struct sock *clcsk) { … } static void smc_fback_write_space(struct sock *clcsk) { … } static void smc_fback_error_report(struct sock *clcsk) { … } static void smc_fback_replace_callbacks(struct smc_sock *smc) { … } static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code) { … } /* fall back during connect */ static int smc_connect_fallback(struct smc_sock *smc, int reason_code) { … } /* decline and fall back during connect */ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, u8 version) { … } static void smc_conn_abort(struct smc_sock *smc, int local_first) { … } /* check if there is a rdma device available for this connection. */ /* called for connect and listen */ static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini) { … } /* check if there is an ISM device available for this connection. */ /* called for connect and listen */ static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) { … } /* is chid unique for the ism devices that are already determined? */ static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini, int cnt) { … } /* determine possible V2 ISM devices (either without PNETID or with PNETID plus * PNETID matching net_device) */ static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, struct smc_init_info *ini) { … } /* Check for VLAN ID and register it on ISM device just for CLC handshake */ static int smc_connect_ism_vlan_setup(struct smc_sock *smc, struct smc_init_info *ini) { … } static int smc_find_proposal_devices(struct smc_sock *smc, struct smc_init_info *ini) { … } /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is * used, the VLAN ID will be registered again during the connection setup. */ static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, struct smc_init_info *ini) { … } #define SMC_CLC_MAX_ACCEPT_LEN … /* CLC handshake during connect */ static int smc_connect_clc(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *aclc, struct smc_init_info *ini) { … } void smc_fill_gid_list(struct smc_link_group *lgr, struct smc_gidlist *gidlist, struct smc_ib_device *known_dev, u8 *known_gid) { … } static int smc_connect_rdma_v2_prepare(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *aclc, struct smc_init_info *ini) { … } /* setup for RDMA connection of client */ static int smc_connect_rdma(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *aclc, struct smc_init_info *ini) { … } /* The server has chosen one of the proposed ISM devices for the communication. * Determine from the CHID of the received CLC ACCEPT the ISM device chosen. */ static int smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm *aclc, struct smc_init_info *ini) { … } /* setup for ISM connection of client */ static int smc_connect_ism(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *aclc, struct smc_init_info *ini) { … } /* check if received accept type and version matches a proposed one */ static int smc_connect_check_aclc(struct smc_init_info *ini, struct smc_clc_msg_accept_confirm *aclc) { … } /* perform steps before actually connecting */ static int __smc_connect(struct smc_sock *smc) { … } static void smc_connect_work(struct work_struct *work) { … } int smc_connect(struct socket *sock, struct sockaddr *addr, int alen, int flags) { … } static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc) { … } /* add a just created sock to the accept queue of the listen sock as * candidate for a following socket accept call from user space */ static void smc_accept_enqueue(struct sock *parent, struct sock *sk) { … } /* remove a socket from the accept queue of its parental listening socket */ static void smc_accept_unlink(struct sock *sk) { … } /* remove a sock from the accept queue to bind it to a new socket created * for a socket accept call from user space */ struct sock *smc_accept_dequeue(struct sock *parent, struct socket *new_sock) { … } /* clean up for a created but never accepted sock */ void smc_close_non_accepted(struct sock *sk) { … } static int smcr_serv_conf_first_link(struct smc_sock *smc) { … } /* listen worker: finish */ static void smc_listen_out(struct smc_sock *new_smc) { … } /* listen worker: finish in state connected */ static void smc_listen_out_connected(struct smc_sock *new_smc) { … } /* listen worker: finish in error state */ static void smc_listen_out_err(struct smc_sock *new_smc) { … } /* listen worker: decline and fall back if possible */ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, int local_first, u8 version) { … } /* listen worker: version checking */ static int smc_listen_v2_check(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } /* listen worker: check prefixes */ static int smc_listen_prfx_check(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc) { … } /* listen worker: initialize connection and buffers */ static int smc_listen_rdma_init(struct smc_sock *new_smc, struct smc_init_info *ini) { … } /* listen worker: initialize connection and buffers for SMC-D */ static int smc_listen_ism_init(struct smc_sock *new_smc, struct smc_init_info *ini) { … } static bool smc_is_already_selected(struct smcd_dev *smcd, struct smc_init_info *ini, int matches) { … } /* check for ISM devices matching proposed ISM devices */ static void smc_check_ism_v2_match(struct smc_init_info *ini, u16 proposed_chid, struct smcd_gid *proposed_gid, unsigned int *matches) { … } static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini) { … } static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } /* listen worker: register buffers */ static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first) { … } static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } /* determine the local device matching to proposal */ static int smc_listen_find_device(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { … } /* listen worker: finish RDMA setup */ static int smc_listen_rdma_finish(struct smc_sock *new_smc, struct smc_clc_msg_accept_confirm *cclc, bool local_first, struct smc_init_info *ini) { … } /* setup for connection of server */ static void smc_listen_work(struct work_struct *work) { … } static void smc_tcp_listen_work(struct work_struct *work) { … } static void smc_clcsock_data_ready(struct sock *listen_clcsock) { … } int smc_listen(struct socket *sock, int backlog) { … } int smc_accept(struct socket *sock, struct socket *new_sock, struct proto_accept_arg *arg) { … } int smc_getname(struct socket *sock, struct sockaddr *addr, int peer) { … } int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { … } int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { … } static __poll_t smc_accept_poll(struct sock *parent) { … } __poll_t smc_poll(struct file *file, struct socket *sock, poll_table *wait) { … } int smc_shutdown(struct socket *sock, int how) { … } static int __smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { … } static int __smc_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { … } int smc_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { … } int smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { … } int smc_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) { … } /* Map the affected portions of the rmbe into an spd, note the number of bytes * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor * updates till whenever a respective page has been fully processed. * Note that subsequent recv() calls have to wait till all splice() processing * completed. */ ssize_t smc_splice_read(struct socket *sock, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { … } /* must look like tcp */ static const struct proto_ops smc_sock_ops = …; int smc_create_clcsk(struct net *net, struct sock *sk, int family) { … } static int __smc_create(struct net *net, struct socket *sock, int protocol, int kern, struct socket *clcsock) { … } static int smc_create(struct net *net, struct socket *sock, int protocol, int kern) { … } static const struct net_proto_family smc_sock_family_ops = …; static int smc_ulp_init(struct sock *sk) { … } static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk, const gfp_t priority) { … } static struct tcp_ulp_ops smc_ulp_ops __read_mostly = …; unsigned int smc_net_id; static __net_init int smc_net_init(struct net *net) { … } static void __net_exit smc_net_exit(struct net *net) { … } static __net_init int smc_net_stat_init(struct net *net) { … } static void __net_exit smc_net_stat_exit(struct net *net) { … } static struct pernet_operations smc_net_ops = …; static struct pernet_operations smc_net_stat_ops = …; static int __init smc_init(void) { … } static void __exit smc_exit(void) { … } module_init(…) …; module_exit(smc_exit); MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_ALIAS_NETPROTO(…); MODULE_ALIAS_TCP_ULP(…) …; /* 256 for IPPROTO_SMC and 1 for SOCK_STREAM */ MODULE_ALIAS_NET_PF_PROTO_TYPE(…); #if IS_ENABLED(CONFIG_IPV6) MODULE_ALIAS_NET_PF_PROTO_TYPE(…); #endif /* CONFIG_IPV6 */ MODULE_ALIAS_GENL_FAMILY(…);
Codebase Browser
linux