// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. * Copyright (c) 2020, Intel Corporation. All rights reserved. */ #include <linux/debugfs.h> #include <linux/highmem.h> #include <linux/module.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/pci.h> #include <linux/dma-mapping.h> #include <linux/slab.h> #include <linux/bitmap.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include <linux/delay.h> #include <rdma/ib_user_verbs.h> #include <rdma/ib_addr.h> #include <rdma/ib_cache.h> #include <linux/mlx5/port.h> #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> #include <linux/mlx5/eswitch.h> #include <linux/mlx5/driver.h> #include <linux/list.h> #include <rdma/ib_smi.h> #include <rdma/ib_umem_odp.h> #include <rdma/lag.h> #include <linux/in.h> #include <linux/etherdevice.h> #include "mlx5_ib.h" #include "ib_rep.h" #include "cmd.h" #include "devx.h" #include "dm.h" #include "fs.h" #include "srq.h" #include "qp.h" #include "wr.h" #include "restrack.h" #include "counters.h" #include "umr.h" #include <rdma/uverbs_std_types.h> #include <rdma/uverbs_ioctl.h> #include <rdma/mlx5_user_ioctl_verbs.h> #include <rdma/mlx5_user_ioctl_cmds.h> #include "macsec.h" #include "data_direct.h" #define UVERBS_MODULE_NAME … #include <rdma/uverbs_named_ioctl.h> MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; struct mlx5_ib_event_work { … }; enum { … }; static struct workqueue_struct *mlx5_ib_event_wq; static LIST_HEAD(mlx5_ib_unaffiliated_port_list); static LIST_HEAD(mlx5_ib_dev_list); /* * This mutex should be held when accessing either of the above lists */ static DEFINE_MUTEX(mlx5_ib_multiport_mutex); struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi) { … } static enum rdma_link_layer mlx5_port_type_cap_to_rdma_ll(int port_type_cap) { … } static enum rdma_link_layer mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num) { … } static int get_port_state(struct ib_device *ibdev, u32 port_num, enum ib_port_state *state) { … } static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev, struct net_device *ndev, struct net_device *upper, u32 *port_num) { … } static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev, struct net_device *ndev, struct net_device *upper, struct net_device *ib_ndev) { … } static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev) { … } static int mlx5_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { … } struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 ib_port_num, u32 *native_port_num) { … } void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num) { … } static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u16 *active_speed, u8 *active_width) { … } static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed, u8 *active_width) { … } static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed, u8 *active_width, bool ext) { … } static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, struct ib_port_attr *props) { … } int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr) { … } static int mlx5_ib_add_gid(const struct ib_gid_attr *attr, __always_unused void **context) { … } static int mlx5_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context) { … } __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev, const struct ib_gid_attr *attr) { … } static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev) { … } enum { … }; static int mlx5_get_vport_access_method(struct ib_device *ibdev) { … } static void get_atomic_caps(struct mlx5_ib_dev *dev, u8 atomic_size_qp, struct ib_device_attr *props) { … } static void get_atomic_caps_qp(struct mlx5_ib_dev *dev, struct ib_device_attr *props) { … } static int mlx5_query_system_image_guid(struct ib_device *ibdev, __be64 *sys_image_guid) { … } static int mlx5_query_max_pkeys(struct ib_device *ibdev, u16 *max_pkeys) { … } static int mlx5_query_vendor_id(struct ib_device *ibdev, u32 *vendor_id) { … } static int mlx5_query_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid) { … } struct mlx5_reg_node_desc { … }; static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc) { … } static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev, struct mlx5_ib_query_device_resp *resp) { … } static int mlx5_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw) { … } static void translate_active_width(struct ib_device *ibdev, u16 active_width, u8 *ib_width) { … } static int mlx5_mtu_to_ib_mtu(int mtu) { … } enum ib_max_vl_num { … }; enum mlx5_vl_hw_cap { … }; static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap, u8 *max_vl_num) { … } static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { … } int mlx5_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { … } static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { … } static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { … } static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) { … } static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask, u32 value) { … } static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props) { … } static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps) { … } static u16 calc_dynamic_bfregs(int uars_per_sys_page) { … } static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k, struct mlx5_ib_alloc_ucontext_req_v2 *req, struct mlx5_bfreg_info *bfregi) { … } static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context) { … } static void deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context) { … } int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) { … } void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) { … } static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn, u16 uid) { … } static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, u16 uid) { … } static int set_ucontext_resp(struct ib_ucontext *uctx, struct mlx5_ib_alloc_ucontext_resp *resp) { … } static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { … } static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext, struct uverbs_attr_bundle *attrs) { … } static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) { … } static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int uar_idx) { … } static u64 uar_index2paddress(struct mlx5_ib_dev *dev, int uar_idx) { … } static int get_command(unsigned long offset) { … } static int get_arg(unsigned long offset) { … } static int get_index(unsigned long offset) { … } /* Index resides in an extra byte to enable larger values than 255 */ static int get_extended_index(unsigned long offset) { … } static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext) { … } static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd) { … } static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev, struct vm_area_struct *vma, struct mlx5_ib_ucontext *context) { … } static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) { … } static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd, struct vm_area_struct *vma, struct mlx5_ib_ucontext *context) { … } static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma) { … } static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev, struct vm_area_struct *vma, struct ib_ucontext *ucontext) { … } static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry) { … } static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) { … } static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { … } static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { … } static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { … } static int init_node_data(struct mlx5_ib_dev *dev) { … } static ssize_t fw_pages_show(struct device *device, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(fw_pages); static ssize_t reg_pages_show(struct device *device, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(reg_pages); static ssize_t hca_type_show(struct device *device, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(hca_type); static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(hw_rev); static ssize_t board_id_show(struct device *device, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(board_id); static struct attribute *mlx5_class_attributes[] = …; static const struct attribute_group mlx5_attr_group = …; static void pkey_change_handler(struct work_struct *work) { … } static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev) { … } static void delay_drop_handler(struct work_struct *work) { … } static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, struct ib_event *ibev) { … } static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, struct ib_event *ibev) { … } static void mlx5_ib_handle_event(struct work_struct *_work) { … } static int mlx5_ib_event(struct notifier_block *nb, unsigned long event, void *param) { … } static int mlx5_ib_event_slave_port(struct notifier_block *nb, unsigned long event, void *param) { … } static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane) { … } static int set_has_smi_cap(struct mlx5_ib_dev *dev) { … } static void get_ext_port_caps(struct mlx5_ib_dev *dev) { … } static u8 mlx5_get_umr_fence(u8 umr_fence_cap) { … } int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev) { … } int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev) { … } static u32 get_core_cap_flags(struct ib_device *ibdev, struct mlx5_hca_vport_context *rep) { … } static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { … } static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { … } static void get_dev_fw_str(struct ib_device *ibdev, char *str) { … } static int lag_event(struct notifier_block *nb, unsigned long event, void *data) { … } static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev) { … } static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev) { … } static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev) { … } static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev) { … } static void mlx5_netdev_notifier_register(struct mlx5_roce *roce, struct net_device *netdev) { … } static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce) { … } static int mlx5e_mdev_notifier_event(struct notifier_block *nb, unsigned long event, void *data) { … } static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num) { … } static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num) { … } static int mlx5_enable_eth(struct mlx5_ib_dev *dev) { … } static void mlx5_disable_eth(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params) { … } static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { … } static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { … } static const struct file_operations fops_delay_drop_timeout = …; static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, struct mlx5_ib_multiport_info *mpi) { … } static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev, struct mlx5_ib_multiport_info *mpi) { … } static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev) { … } static int mmap_obj_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { … } static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c, struct mlx5_user_mmap_entry *entry, size_t length) { … } static struct mlx5_user_mmap_entry * alloc_var_entry(struct mlx5_ib_ucontext *c) { … } static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)( struct uverbs_attr_bundle *attrs) { … } DECLARE_UVERBS_NAMED_METHOD(…); DECLARE_UVERBS_NAMED_METHOD_DESTROY(…); DECLARE_UVERBS_NAMED_OBJECT(…); static bool var_is_supported(struct ib_device *device) { … } static struct mlx5_user_mmap_entry * alloc_uar_entry(struct mlx5_ib_ucontext *c, enum mlx5_ib_uapi_uar_alloc_type alloc_type) { … } static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)( struct uverbs_attr_bundle *attrs) { … } DECLARE_UVERBS_NAMED_METHOD(…); DECLARE_UVERBS_NAMED_METHOD_DESTROY(…); DECLARE_UVERBS_NAMED_OBJECT(…); ADD_UVERBS_ATTRIBUTES_SIMPLE( mlx5_ib_query_context, UVERBS_OBJECT_DEVICE, UVERBS_METHOD_QUERY_CONTEXT, UVERBS_ATTR_PTR_OUT( MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX, UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp, dump_fill_mkey), UA_MANDATORY)); ADD_UVERBS_ATTRIBUTES_SIMPLE( mlx5_ib_reg_dmabuf_mr, UVERBS_OBJECT_MR, UVERBS_METHOD_REG_DMABUF_MR, UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS, enum mlx5_ib_uapi_reg_dmabuf_flags, UA_OPTIONAL)); static const struct uapi_definition mlx5_ib_defs[] = …; static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) { … } static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent, enum rdma_nl_dev_type type, const char *name); static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev); static const struct ib_device_ops mlx5_ib_dev_ops = …; static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = …; static const struct ib_device_ops mlx5_ib_dev_sriov_ops = …; static const struct ib_device_ops mlx5_ib_dev_mw_ops = …; static const struct ib_device_ops mlx5_ib_dev_xrc_ops = …; static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) { … } static const struct ib_device_ops mlx5_ib_dev_port_ops = …; static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev) { … } static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = …; static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev) { … } static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = …; static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev) { … } static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev) { … } static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) { … } void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, struct mlx5_data_direct_dev *dev) { … } void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev) { … } void __mlx5_ib_remove(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile, int stage) { … } int __mlx5_ib_add(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile) { … } static const struct mlx5_ib_profile pf_profile = …; const struct mlx5_ib_profile raw_eth_profile = …; static const struct mlx5_ib_profile plane_profile = …; static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent, enum rdma_nl_dev_type type, const char *name) { … } static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev) { … } static int mlx5r_mp_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { … } static void mlx5r_mp_remove(struct auxiliary_device *adev) { … } static int mlx5r_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { … } static void mlx5r_remove(struct auxiliary_device *adev) { … } static const struct auxiliary_device_id mlx5r_mp_id_table[] = …; static const struct auxiliary_device_id mlx5r_id_table[] = …; MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table); MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table); static struct auxiliary_driver mlx5r_mp_driver = …; static struct auxiliary_driver mlx5r_driver = …; static int __init mlx5_ib_init(void) { … } static void __exit mlx5_ib_cleanup(void) { … } module_init(…) …; module_exit(mlx5_ib_cleanup);
Codebase Browser
linux