// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2015 - 2020 Intel Corporation. */ #include <rdma/ib_mad.h> #include <rdma/ib_user_verbs.h> #include <linux/io.h> #include <linux/module.h> #include <linux/utsname.h> #include <linux/rculist.h> #include <linux/mm.h> #include <linux/vmalloc.h> #include <rdma/opa_addr.h> #include <linux/nospec.h> #include "hfi.h" #include "common.h" #include "device.h" #include "trace.h" #include "qp.h" #include "verbs_txreq.h" #include "debugfs.h" #include "vnic.h" #include "fault.h" #include "affinity.h" #include "ipoib.h" static unsigned int hfi1_lkey_table_size = …; module_param_named(lkey_table_size, hfi1_lkey_table_size, uint, S_IRUGO); MODULE_PARM_DESC(…) …; static unsigned int hfi1_max_pds = …; module_param_named(max_pds, hfi1_max_pds, uint, S_IRUGO); MODULE_PARM_DESC(…) …; static unsigned int hfi1_max_ahs = …; module_param_named(max_ahs, hfi1_max_ahs, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_cqes = …; module_param_named(max_cqes, hfi1_max_cqes, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_cqs = …; module_param_named(max_cqs, hfi1_max_cqs, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_qp_wrs = …; module_param_named(max_qp_wrs, hfi1_max_qp_wrs, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_qps = …; module_param_named(max_qps, hfi1_max_qps, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_sges = …; module_param_named(max_sges, hfi1_max_sges, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_mcast_grps = …; module_param_named(max_mcast_grps, hfi1_max_mcast_grps, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_mcast_qp_attached = …; module_param_named(max_mcast_qp_attached, hfi1_max_mcast_qp_attached, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_srqs = …; module_param_named(max_srqs, hfi1_max_srqs, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_srq_sges = …; module_param_named(max_srq_sges, hfi1_max_srq_sges, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned int hfi1_max_srq_wrs = …; module_param_named(max_srq_wrs, hfi1_max_srq_wrs, uint, S_IRUGO); MODULE_PARM_DESC(…) …; unsigned short piothreshold = …; module_param(piothreshold, ushort, S_IRUGO); MODULE_PARM_DESC(…) …; static unsigned int sge_copy_mode; module_param(sge_copy_mode, uint, S_IRUGO); MODULE_PARM_DESC(…) …; static void verbs_sdma_complete( struct sdma_txreq *cookie, int status); static int pio_wait(struct rvt_qp *qp, struct send_context *sc, struct hfi1_pkt_state *ps, u32 flag); /* Length of buffer to create verbs txreq cache name */ #define TXREQ_NAME_LEN … static uint wss_threshold = …; module_param(wss_threshold, uint, S_IRUGO); MODULE_PARM_DESC(…) …; static uint wss_clean_period = …; module_param(wss_clean_period, uint, S_IRUGO); MODULE_PARM_DESC(…) …; /* * Translate ib_wr_opcode into ib_wc_opcode. */ const enum ib_wc_opcode ib_hfi1_wc_opcode[] = …; /* * Length of header by opcode, 0 --> not supported */ const u8 hdr_len_by_opcode[256] = …; static const opcode_handler opcode_handler_tbl[256] = …; #define OPMASK … static const u32 pio_opmask[BIT(3)] = …; /* * System image GUID. */ __be64 ib_hfi1_sys_image_guid; /* * Make sure the QP is ready and able to accept the given opcode. */ static inline opcode_handler qp_ok(struct hfi1_packet *packet) { … } static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc) { … } static opcode_handler tid_qp_ok(int opcode, struct hfi1_packet *packet) { … } void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet) { … } void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet) { … } static int hfi1_do_pkey_check(struct hfi1_packet *packet) { … } static inline void hfi1_handle_packet(struct hfi1_packet *packet, bool is_mcast) { … } /** * hfi1_ib_rcv - process an incoming packet * @packet: data packet information * * This is called to process an incoming packet at interrupt level. */ void hfi1_ib_rcv(struct hfi1_packet *packet) { … } void hfi1_16B_rcv(struct hfi1_packet *packet) { … } /* * This is called from a timer to check for QPs * which need kernel memory in order to send a packet. */ static void mem_timer(struct timer_list *t) { … } /* * This is called with progress side lock held. */ /* New API */ static void verbs_sdma_complete( struct sdma_txreq *cookie, int status) { … } void hfi1_wait_kmem(struct rvt_qp *qp) { … } static int wait_kmem(struct hfi1_ibdev *dev, struct rvt_qp *qp, struct hfi1_pkt_state *ps) { … } /* * This routine calls txadds for each sg entry. * * Add failures will revert the sge cursor */ static noinline int build_verbs_ulp_payload( struct sdma_engine *sde, u32 length, struct verbs_txreq *tx) { … } /** * update_tx_opstats - record stats by opcode * @qp: the qp * @ps: transmit packet state * @plen: the plen in dwords * * This is a routine to record the tx opstats after a * packet has been presented to the egress mechanism. */ static void update_tx_opstats(struct rvt_qp *qp, struct hfi1_pkt_state *ps, u32 plen) { … } /* * Build the number of DMA descriptors needed to send length bytes of data. * * NOTE: DMA mapping is held in the tx until completed in the ring or * the tx desc is freed without having been submitted to the ring * * This routine ensures all the helper routine calls succeed. */ /* New API */ static int build_verbs_tx_desc( struct sdma_engine *sde, u32 length, struct verbs_txreq *tx, struct hfi1_ahg_info *ahg_info, u64 pbc) { … } static u64 update_hcrc(u8 opcode, u64 pbc) { … } int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, u64 pbc) { … } /* * If we are now in the error state, return zero to flush the * send work request. */ static int pio_wait(struct rvt_qp *qp, struct send_context *sc, struct hfi1_pkt_state *ps, u32 flag) { … } static void verbs_pio_complete(void *arg, int code) { … } int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, u64 pbc) { … } /* * egress_pkey_matches_entry - return 1 if the pkey matches ent (ent * being an entry from the partition key table), return 0 * otherwise. Use the matching criteria for egress partition keys * specified in the OPAv1 spec., section 9.1l.7. */ static inline int egress_pkey_matches_entry(u16 pkey, u16 ent) { … } /** * egress_pkey_check - check P_KEY of a packet * @ppd: Physical IB port data * @slid: SLID for packet * @pkey: PKEY for header * @sc5: SC for packet * @s_pkey_index: It will be used for look up optimization for kernel contexts * only. If it is negative value, then it means user contexts is calling this * function. * * It checks if hdr's pkey is valid. * * Return: 0 on success, otherwise, 1 */ int egress_pkey_check(struct hfi1_pportdata *ppd, u32 slid, u16 pkey, u8 sc5, int8_t s_pkey_index) { … } /* * get_send_routine - choose an egress routine * * Choose an egress routine based on QP type * and size */ static inline send_routine get_send_routine(struct rvt_qp *qp, struct hfi1_pkt_state *ps) { … } /** * hfi1_verbs_send - send a packet * @qp: the QP to send on * @ps: the state of the packet to send * * Return zero if packet is sent or queued OK. * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise. */ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps) { … } /** * hfi1_fill_device_attr - Fill in rvt dev info device attributes. * @dd: the device data structure */ static void hfi1_fill_device_attr(struct hfi1_devdata *dd) { … } static inline u16 opa_speed_to_ib(u16 in) { … } /* * Convert a single OPA link width (no multiple flags) to an IB value. * A zero OPA link width means link down, which means the IB width value * is a don't care. */ static inline u16 opa_width_to_ib(u16 in) { … } static int query_port(struct rvt_dev_info *rdi, u32 port_num, struct ib_port_attr *props) { … } static int modify_device(struct ib_device *device, int device_modify_mask, struct ib_device_modify *device_modify) { … } static int shut_down_port(struct rvt_dev_info *rdi, u32 port_num) { … } static int hfi1_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, int guid_index, __be64 *guid) { … } /* * convert ah port,sl to sc */ u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah) { … } static int hfi1_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr) { … } static void hfi1_notify_new_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr, struct rvt_ah *ah) { … } /** * hfi1_get_npkeys - return the size of the PKEY table for context 0 * @dd: the hfi1_ib device */ unsigned hfi1_get_npkeys(struct hfi1_devdata *dd) { … } static void init_ibport(struct hfi1_pportdata *ppd) { … } static void hfi1_get_dev_fw_str(struct ib_device *ibdev, char *str) { … } static const char * const driver_cntr_names[] = …; static struct rdma_stat_desc *dev_cntr_descs; static struct rdma_stat_desc *port_cntr_descs; int num_driver_cntrs = …; static int num_dev_cntrs; static int num_port_cntrs; /* * Convert a list of names separated by '\n' into an array of NULL terminated * strings. Optionally some entries can be reserved in the array to hold extra * external strings. */ static int init_cntr_names(const char *names_in, const size_t names_len, int num_extra_names, int *num_cntrs, struct rdma_stat_desc **cntr_descs) { … } static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev) { … } static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num) { … } static u64 hfi1_sps_ints(void) { … } static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u32 port, int index) { … } static const struct ib_device_ops hfi1_dev_ops = …; /** * hfi1_register_ib_device - register our device with the infiniband core * @dd: the device data structure * Return 0 if successful, errno if unsuccessful. */ int hfi1_register_ib_device(struct hfi1_devdata *dd) { … } void hfi1_unregister_ib_device(struct hfi1_devdata *dd) { … } void hfi1_cnp_rcv(struct hfi1_packet *packet) { … }
Codebase Browser
linux