// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2004-2007 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. */ #include <linux/completion.h> #include <linux/dma-mapping.h> #include <linux/device.h> #include <linux/module.h> #include <linux/err.h> #include <linux/idr.h> #include <linux/interrupt.h> #include <linux/random.h> #include <linux/rbtree.h> #include <linux/spinlock.h> #include <linux/slab.h> #include <linux/sysfs.h> #include <linux/workqueue.h> #include <linux/kdev_t.h> #include <linux/etherdevice.h> #include <rdma/ib_cache.h> #include <rdma/ib_cm.h> #include <rdma/ib_sysfs.h> #include "cm_msgs.h" #include "core_priv.h" #include "cm_trace.h" MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; #define CM_DESTROY_ID_WAIT_TIMEOUT … static const char * const ibcm_rej_reason_strs[] = …; const char *__attribute_const__ ibcm_reject_msg(int reason) { … } EXPORT_SYMBOL(…); struct cm_id_private; struct cm_work; static int cm_add_one(struct ib_device *device); static void cm_remove_one(struct ib_device *device, void *client_data); static void cm_process_work(struct cm_id_private *cm_id_priv, struct cm_work *work); static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, struct ib_cm_sidr_rep_param *param); static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len); static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, void *private_data, u8 private_data_len); static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len); static struct ib_client cm_client = …; static struct ib_cm { … } cm; /* Counter indexes ordered by attribute ID */ enum { … }; enum { … }; struct cm_counter_attribute { … }; struct cm_port { … }; struct cm_device { … }; struct cm_av { … }; struct cm_work { … }; struct cm_timewait_info { … }; struct cm_id_private { … }; static void cm_dev_release(struct kref *kref) { … } static void cm_device_put(struct cm_device *cm_dev) { … } static void cm_work_handler(struct work_struct *work); static inline void cm_deref_id(struct cm_id_private *cm_id_priv) { … } static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv) { … } static void cm_free_msg(struct ib_mad_send_buf *msg) { … } static struct ib_mad_send_buf * cm_alloc_priv_msg(struct cm_id_private *cm_id_priv) { … } static void cm_free_priv_msg(struct ib_mad_send_buf *msg) { … } static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc) { … } static int cm_create_response_msg_ah(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc, struct ib_mad_send_buf *msg) { … } static int cm_alloc_response_msg(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc, struct ib_mad_send_buf **msg) { … } static void cm_free_response_msg(struct ib_mad_send_buf *msg) { … } static void *cm_copy_private_data(const void *private_data, u8 private_data_len) { … } static void cm_set_private_data(struct cm_id_private *cm_id_priv, void *private_data, u8 private_data_len) { … } static void cm_set_av_port(struct cm_av *av, struct cm_port *port) { … } static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc, struct rdma_ah_attr *ah_attr, struct cm_av *av) { … } static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, struct ib_grh *grh, struct cm_av *av) { … } static struct cm_port * get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr) { … } static int cm_init_av_by_path(struct sa_path_rec *path, const struct ib_gid_attr *sgid_attr, struct cm_av *av) { … } /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */ static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src) { … } static void cm_destroy_av(struct cm_av *av) { … } static u32 cm_local_id(__be32 local_id) { … } static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id) { … } /* * Trivial helpers to strip endian annotation and compare; the * endianness doesn't actually matter since we just need a stable * order for the RB tree. */ static int be32_lt(__be32 a, __be32 b) { … } static int be32_gt(__be32 a, __be32 b) { … } static int be64_lt(__be64 a, __be64 b) { … } static int be64_gt(__be64 a, __be64 b) { … } /* * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv * if the new ID was inserted, NULL if it could not be inserted due to a * collision, or the existing cm_id_priv ready for shared usage. */ static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, ib_cm_handler shared_handler) { … } static struct cm_id_private *cm_find_listen(struct ib_device *device, __be64 service_id) { … } static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info *timewait_info) { … } static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid, __be32 remote_id) { … } static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info *timewait_info) { … } static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private *cm_id_priv) { … } static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device, ib_cm_handler cm_handler, void *context) { … } /* * Make the ID visible to the MAD handlers and other threads that use the * xarray. */ static void cm_finalize_id(struct cm_id_private *cm_id_priv) { … } struct ib_cm_id *ib_create_cm_id(struct ib_device *device, ib_cm_handler cm_handler, void *context) { … } EXPORT_SYMBOL(…); static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv) { … } static void cm_free_work(struct cm_work *work) { … } static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv, struct cm_work *work) __releases(&cm_id_priv->lock) { … } static inline int cm_convert_to_ms(int iba_time) { … } /* * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time * Because of how ack_timeout is stored, adding one doubles the timeout. * To avoid large timeouts, select the max(ack_delay, life_time + 1), and * increment it (round up) only if the other is within 50%. */ static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time) { … } static void cm_remove_remote(struct cm_id_private *cm_id_priv) { … } static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id) { … } static void cm_enter_timewait(struct cm_id_private *cm_id_priv) { … } static void cm_reset_to_idle(struct cm_id_private *cm_id_priv) { … } static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id, enum ib_cm_state old_state) { … } static void cm_destroy_id(struct ib_cm_id *cm_id, int err) { … } void ib_destroy_cm_id(struct ib_cm_id *cm_id) { … } EXPORT_SYMBOL(…); static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id) { … } /** * ib_cm_listen - Initiates listening on the specified service ID for * connection and service ID resolution requests. * @cm_id: Connection identifier associated with the listen request. * @service_id: Service identifier matched against incoming connection * and service ID resolution requests. The service ID should be specified * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will * assign a service ID to the caller. */ int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id) { … } EXPORT_SYMBOL(…); /** * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on * the given service ID. * * If there's an existing ID listening on that same device and service ID, * return it. * * @device: Device associated with the cm_id. All related communication will * be associated with the specified device. * @cm_handler: Callback invoked to notify the user of CM events. * @service_id: Service identifier matched against incoming connection * and service ID resolution requests. The service ID should be specified * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will * assign a service ID to the caller. * * Callers should call ib_destroy_cm_id when done with the listener ID. */ struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, ib_cm_handler cm_handler, __be64 service_id) { … } EXPORT_SYMBOL(…); static __be64 cm_form_tid(struct cm_id_private *cm_id_priv) { … } static void cm_format_mad_hdr(struct ib_mad_hdr *hdr, __be16 attr_id, __be64 tid) { … } static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id, __be64 tid, u32 attr_mod) { … } static void cm_format_req(struct cm_req_msg *req_msg, struct cm_id_private *cm_id_priv, struct ib_cm_req_param *param) { … } static int cm_validate_req_param(struct ib_cm_req_param *param) { … } int ib_send_cm_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param) { … } EXPORT_SYMBOL(…); static int cm_issue_rej(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc, enum ib_cm_rej_reason reason, enum cm_msg_response msg_rejected, void *ari, u8 ari_length) { … } static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) { … } static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num, struct sa_path_rec *path, union ib_gid *gid) { … } static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, struct sa_path_rec *alt_path, struct ib_wc *wc) { … } static void cm_format_paths_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, struct sa_path_rec *alt_path, struct ib_wc *wc) { … } static u16 cm_get_bth_pkey(struct cm_work *work) { … } /** * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID * ULPs (such as IPoIB) do not understand OPA GIDs and will * reject them as the local_gid will not match the sgid. Therefore, * change the pathrec's SGID to an IB SGID. * * @work: Work completion * @path: Path record */ static void cm_opa_to_ib_sgid(struct cm_work *work, struct sa_path_rec *path) { … } static void cm_format_req_event(struct cm_work *work, struct cm_id_private *cm_id_priv, struct ib_cm_id *listen_id) { … } static void cm_process_work(struct cm_id_private *cm_id_priv, struct cm_work *work) { … } static void cm_format_mra(struct cm_mra_msg *mra_msg, struct cm_id_private *cm_id_priv, enum cm_msg_response msg_mraed, u8 service_timeout, const void *private_data, u8 private_data_len) { … } static void cm_format_rej(struct cm_rej_msg *rej_msg, struct cm_id_private *cm_id_priv, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len, enum ib_cm_state state) { … } static void cm_dup_req_handler(struct cm_work *work, struct cm_id_private *cm_id_priv) { … } static struct cm_id_private *cm_match_req(struct cm_work *work, struct cm_id_private *cm_id_priv) { … } /* * Work-around for inter-subnet connections. If the LIDs are permissive, * we need to override the LID/SL data in the REQ with the LID information * in the work completion. */ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) { … } static int cm_req_handler(struct cm_work *work) { … } static void cm_format_rep(struct cm_rep_msg *rep_msg, struct cm_id_private *cm_id_priv, struct ib_cm_rep_param *param) { … } int ib_send_cm_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param) { … } EXPORT_SYMBOL(…); static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { … } int ib_send_cm_rtu(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { … } EXPORT_SYMBOL(…); static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) { … } static void cm_dup_rep_handler(struct cm_work *work) { … } static int cm_rep_handler(struct cm_work *work) { … } static int cm_establish_handler(struct cm_work *work) { … } static int cm_rtu_handler(struct cm_work *work) { … } static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { … } static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { … } int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { … } EXPORT_SYMBOL(…); static void cm_format_drep(struct cm_drep_msg *drep_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { … } static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, void *private_data, u8 private_data_len) { … } int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { … } EXPORT_SYMBOL(…); static int cm_issue_drep(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc) { … } static int cm_dreq_handler(struct cm_work *work) { … } static int cm_drep_handler(struct cm_work *work) { … } static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len) { … } int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len) { … } EXPORT_SYMBOL(…); static void cm_format_rej_event(struct cm_work *work) { … } static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) { … } static int cm_rej_handler(struct cm_work *work) { … } int ib_send_cm_mra(struct ib_cm_id *cm_id, u8 service_timeout, const void *private_data, u8 private_data_len) { … } EXPORT_SYMBOL(…); static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) { … } static int cm_mra_handler(struct cm_work *work) { … } static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, struct sa_path_rec *path) { … } static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, struct sa_path_rec *path, struct cm_lap_msg *lap_msg) { … } static int cm_lap_handler(struct cm_work *work) { … } static int cm_apr_handler(struct cm_work *work) { … } static int cm_timewait_handler(struct cm_work *work) { … } static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, struct cm_id_private *cm_id_priv, struct ib_cm_sidr_req_param *param) { … } int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, struct ib_cm_sidr_req_param *param) { … } EXPORT_SYMBOL(…); static void cm_format_sidr_req_event(struct cm_work *work, const struct cm_id_private *rx_cm_id, struct ib_cm_id *listen_id) { … } static int cm_sidr_req_handler(struct cm_work *work) { … } static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, struct cm_id_private *cm_id_priv, struct ib_cm_sidr_rep_param *param) { … } static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, struct ib_cm_sidr_rep_param *param) { … } int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, struct ib_cm_sidr_rep_param *param) { … } EXPORT_SYMBOL(…); static void cm_format_sidr_rep_event(struct cm_work *work, const struct cm_id_private *cm_id_priv) { … } static int cm_sidr_rep_handler(struct cm_work *work) { … } static void cm_process_send_error(struct cm_id_private *cm_id_priv, struct ib_mad_send_buf *msg, enum ib_cm_state state, enum ib_wc_status wc_status) { … } static void cm_send_handler(struct ib_mad_agent *mad_agent, struct ib_mad_send_wc *mad_send_wc) { … } static void cm_work_handler(struct work_struct *_work) { … } static int cm_establish(struct ib_cm_id *cm_id) { … } static int cm_migrate(struct ib_cm_id *cm_id) { … } int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event) { … } EXPORT_SYMBOL(…); static void cm_recv_handler(struct ib_mad_agent *mad_agent, struct ib_mad_send_buf *send_buf, struct ib_mad_recv_wc *mad_recv_wc) { … } static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { … } static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { … } static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { … } int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { … } EXPORT_SYMBOL(…); static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num, struct ib_port_attribute *attr, char *buf) { … } #define CM_COUNTER_ATTR(_name, _group, _index) … #define CM_COUNTER_GROUP(_group, _name) … CM_COUNTER_GROUP(…) CM_COUNTER_GROUP(…) CM_COUNTER_GROUP(…) CM_COUNTER_GROUP(…) static const struct attribute_group *cm_counter_groups[] = …; static int cm_add_one(struct ib_device *ib_device) { … } static void cm_remove_one(struct ib_device *ib_device, void *client_data) { … } static int __init ib_cm_init(void) { … } static void __exit ib_cm_cleanup(void) { … } module_init(…) …; module_exit(ib_cm_cleanup);
Codebase Browser
linux