// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2015 - 2020 Intel Corporation. * Copyright(c) 2021 Cornelis Networks. */ /* * This file contains all of the code that is specific to the HFI chip */ #include <linux/pci.h> #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/module.h> #include "hfi.h" #include "trace.h" #include "mad.h" #include "pio.h" #include "sdma.h" #include "eprom.h" #include "efivar.h" #include "platform.h" #include "aspm.h" #include "affinity.h" #include "debugfs.h" #include "fault.h" #include "netdev.h" uint num_vls = …; module_param(num_vls, uint, S_IRUGO); MODULE_PARM_DESC(…) …; /* * Default time to aggregate two 10K packets from the idle state * (timer not running). The timer starts at the end of the first packet, * so only the time for one 10K packet and header plus a bit extra is needed. * 10 * 1024 + 64 header byte = 10304 byte * 10304 byte / 12.5 GB/s = 824.32ns */ uint rcv_intr_timeout = …; /* 16 is for coalescing interrupt */ module_param(rcv_intr_timeout, uint, S_IRUGO); MODULE_PARM_DESC(…) …; uint rcv_intr_count = …; /* same as qib */ module_param(rcv_intr_count, uint, S_IRUGO); MODULE_PARM_DESC(…) …; ushort link_crc_mask = …; module_param(link_crc_mask, ushort, S_IRUGO); MODULE_PARM_DESC(…) …; uint loopback; module_param_named(loopback, loopback, uint, S_IRUGO); MODULE_PARM_DESC(…) …; /* Other driver tunables */ uint rcv_intr_dynamic = …; /* enable dynamic mode for rcv int mitigation*/ static ushort crc_14b_sideband = …; static uint use_flr = …; uint quick_linkup; /* skip LNI */ struct flag_table { … }; /* str must be a string constant */ #define FLAG_ENTRY(str, extra, flag) … #define FLAG_ENTRY0(str, flag) … /* Send Error Consequences */ #define SEC_WRITE_DROPPED … #define SEC_PACKET_DROPPED … #define SEC_SC_HALTED … #define SEC_SPC_FREEZE … #define DEFAULT_KRCVQS … #define MIN_KERNEL_KCTXTS … #define FIRST_KERNEL_KCTXT … /* * RSM instance allocation * 0 - User Fecn Handling * 1 - Vnic * 2 - AIP * 3 - Verbs */ #define RSM_INS_FECN … #define RSM_INS_VNIC … #define RSM_INS_AIP … #define RSM_INS_VERBS … /* Bit offset into the GUID which carries HFI id information */ #define GUID_HFI_INDEX_SHIFT … /* extract the emulation revision */ #define emulator_rev(dd) … /* parallel and serial emulation versions are 3 and 4 respectively */ #define is_emulator_p(dd) … #define is_emulator_s(dd) … /* RSM fields for Verbs */ /* packet type */ #define IB_PACKET_TYPE … #define QW_SHIFT … /* QPN[7..1] */ #define QPN_WIDTH … /* LRH.BTH: QW 0, OFFSET 48 - for match */ #define LRH_BTH_QW … #define LRH_BTH_BIT_OFFSET … #define LRH_BTH_OFFSET(off) … #define LRH_BTH_MATCH_OFFSET … #define LRH_BTH_SELECT #define LRH_BTH_MASK … #define LRH_BTH_VALUE … /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */ #define LRH_SC_QW … #define LRH_SC_BIT_OFFSET … #define LRH_SC_OFFSET(off) … #define LRH_SC_MATCH_OFFSET … #define LRH_SC_MASK … #define LRH_SC_VALUE … /* SC[n..0] QW 0, OFFSET 60 - for select */ #define LRH_SC_SELECT_OFFSET … /* QPN[m+n:1] QW 1, OFFSET 1 */ #define QPN_SELECT_OFFSET … /* RSM fields for AIP */ /* LRH.BTH above is reused for this rule */ /* BTH.DESTQP: QW 1, OFFSET 16 for match */ #define BTH_DESTQP_QW … #define BTH_DESTQP_BIT_OFFSET … #define BTH_DESTQP_OFFSET(off) … #define BTH_DESTQP_MATCH_OFFSET … #define BTH_DESTQP_MASK … #define BTH_DESTQP_VALUE … /* DETH.SQPN: QW 1 Offset 56 for select */ /* We use 8 most significant Soure QPN bits as entropy fpr AIP */ #define DETH_AIP_SQPN_QW … #define DETH_AIP_SQPN_BIT_OFFSET … #define DETH_AIP_SQPN_OFFSET(off) … #define DETH_AIP_SQPN_SELECT_OFFSET … /* RSM fields for Vnic */ /* L2_TYPE: QW 0, OFFSET 61 - for match */ #define L2_TYPE_QW … #define L2_TYPE_BIT_OFFSET … #define L2_TYPE_OFFSET(off) … #define L2_TYPE_MATCH_OFFSET … #define L2_TYPE_MASK … #define L2_16B_VALUE … /* L4_TYPE QW 1, OFFSET 0 - for match */ #define L4_TYPE_QW … #define L4_TYPE_BIT_OFFSET … #define L4_TYPE_OFFSET(off) … #define L4_TYPE_MATCH_OFFSET … #define L4_16B_TYPE_MASK … #define L4_16B_ETH_VALUE … /* 16B VESWID - for select */ #define L4_16B_HDR_VESWID_OFFSET … /* 16B ENTROPY - for select */ #define L2_16B_ENTROPY_OFFSET … /* defines to build power on SC2VL table */ #define SC2VL_VAL( \ num, \ sc0, sc0val, \ sc1, sc1val, \ sc2, sc2val, \ sc3, sc3val, \ sc4, sc4val, \ sc5, sc5val, \ sc6, sc6val, \ sc7, sc7val) … #define DC_SC_VL_VAL( \ range, \ e0, e0val, \ e1, e1val, \ e2, e2val, \ e3, e3val, \ e4, e4val, \ e5, e5val, \ e6, e6val, \ e7, e7val, \ e8, e8val, \ e9, e9val, \ e10, e10val, \ e11, e11val, \ e12, e12val, \ e13, e13val, \ e14, e14val, \ e15, e15val) … /* all CceStatus sub-block freeze bits */ #define ALL_FROZE … /* all CceStatus sub-block TXE pause bits */ #define ALL_TXE_PAUSE … /* all CceStatus sub-block RXE pause bits */ #define ALL_RXE_PAUSE … #define CNTR_MAX … #define CNTR_32BIT_MAX … /* * CCE Error flags. */ static const struct flag_table cce_err_status_flags[] = …; /* * Misc Error flags */ #define MES(text) … static const struct flag_table misc_err_status_flags[] = …; /* * TXE PIO Error flags and consequences */ static const struct flag_table pio_err_status_flags[] = …; /* TXE PIO errors that cause an SPC freeze */ #define ALL_PIO_FREEZE_ERR … /* * TXE SDMA Error flags */ static const struct flag_table sdma_err_status_flags[] = …; /* TXE SDMA errors that cause an SPC freeze */ #define ALL_SDMA_FREEZE_ERR … /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */ #define PORT_DISCARD_EGRESS_ERRS … /* * TXE Egress Error flags */ #define SEES(text) … static const struct flag_table egress_err_status_flags[] = …; /* * TXE Egress Error Info flags */ #define SEEI(text) … static const struct flag_table egress_err_info_flags[] = …; /* TXE Egress errors that cause an SPC freeze */ #define ALL_TXE_EGRESS_FREEZE_ERR … /* * TXE Send error flags */ #define SES(name) … static const struct flag_table send_err_status_flags[] = …; /* * TXE Send Context Error flags and consequences */ static const struct flag_table sc_err_status_flags[] = …; /* * RXE Receive Error flags */ #define RXES(name) … static const struct flag_table rxe_err_status_flags[] = …; /* RXE errors that will trigger an SPC freeze */ #define ALL_RXE_FREEZE_ERR … #define RXE_FREEZE_ABORT_MASK … /* * DCC Error Flags */ #define DCCE(name) … static const struct flag_table dcc_err_flags[] = …; /* * LCB error flags */ #define LCBE(name) … static const struct flag_table lcb_err_flags[] = …; /* * DC8051 Error Flags */ #define D8E(name) … static const struct flag_table dc8051_err_flags[] = …; /* * DC8051 Information Error flags * * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field. */ static const struct flag_table dc8051_info_err_flags[] = …; /* * DC8051 Information Host Information flags * * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field. */ static const struct flag_table dc8051_info_host_msg_flags[] = …; static u32 encoded_size(u32 size); static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate); static int set_physical_link_state(struct hfi1_devdata *dd, u64 state); static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, u8 *continuous); static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, u8 *vcu, u16 *vl15buf, u8 *crc_sizes); static void read_vc_remote_link_width(struct hfi1_devdata *dd, u8 *remote_tx_rate, u16 *link_widths); static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, u8 *flag_bits, u16 *link_widths); static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, u8 *device_rev); static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx); static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, u8 *tx_polarity_inversion, u8 *rx_polarity_inversion, u8 *max_rate); static void handle_sdma_eng_err(struct hfi1_devdata *dd, unsigned int context, u64 err_status); static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg); static void handle_dcc_err(struct hfi1_devdata *dd, unsigned int context, u64 err_status); static void handle_lcb_err(struct hfi1_devdata *dd, unsigned int context, u64 err_status); static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); static void set_partition_keys(struct hfi1_pportdata *ppd); static const char *link_state_name(u32 state); static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state); static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, u64 *out_data); static int read_idle_sma(struct hfi1_devdata *dd, u64 *data); static int thermal_init(struct hfi1_devdata *dd); static void update_statusp(struct hfi1_pportdata *ppd, u32 state); static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, int msecs); static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, int msecs); static void log_state_transition(struct hfi1_pportdata *ppd, u32 state); static void log_physical_state(struct hfi1_pportdata *ppd, u32 state); static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, int msecs); static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, int msecs); static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc); static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); static void handle_temp_err(struct hfi1_devdata *dd); static void dc_shutdown(struct hfi1_devdata *dd); static void dc_start(struct hfi1_devdata *dd); static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, unsigned int *np); static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width); /* * Error interrupt table entry. This is used as input to the interrupt * "clear down" routine used for all second tier error interrupt register. * Second tier interrupt registers have a single bit representing them * in the top-level CceIntStatus. */ struct err_reg_info { … }; #define NUM_MISC_ERRS … #define NUM_DC_ERRS … #define NUM_VARIOUS … /* * Helpers for building HFI and DC error interrupt table entries. Different * helpers are needed because of inconsistent register names. */ #define EE(reg, handler, desc) … #define DC_EE1(reg, handler, desc) … #define DC_EE2(reg, handler, desc) … /* * Table of the "misc" grouping of error interrupts. Each entry refers to * another register containing more information. */ static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = …; /* * Index into the Various section of the interrupt sources * corresponding to the Critical Temperature interrupt. */ #define TCRIT_INT_SOURCE … /* * SDMA error interrupt entry - refers to another register containing more * information. */ static const struct err_reg_info sdma_eng_err = …; static const struct err_reg_info various_err[NUM_VARIOUS] = …; /* * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG * register can not be derived from the MTU value because 10K is not * a power of 2. Therefore, we need a constant. Everything else can * be calculated. */ #define DCC_CFG_PORT_MTU_CAP_10240 … /* * Table of the DC grouping of error interrupts. Each entry refers to * another register containing more information. */ static const struct err_reg_info dc_errs[NUM_DC_ERRS] = …; struct cntr_entry { … }; #define C_RCV_HDR_OVF_FIRST … #define C_RCV_HDR_OVF_LAST … #define CNTR_ELEM(name, csr, offset, flags, accessor) … /* 32bit RXE */ #define RXE32_PORT_CNTR_ELEM(name, counter, flags) … #define RXE32_DEV_CNTR_ELEM(name, counter, flags) … /* 64bit RXE */ #define RXE64_PORT_CNTR_ELEM(name, counter, flags) … #define RXE64_DEV_CNTR_ELEM(name, counter, flags) … #define OVR_LBL(ctx) … #define OVR_ELM(ctx) … /* 32bit TXE */ #define TXE32_PORT_CNTR_ELEM(name, counter, flags) … /* 64bit TXE */ #define TXE64_PORT_CNTR_ELEM(name, counter, flags) … #define TX64_DEV_CNTR_ELEM(name, counter, flags) … /* CCE */ #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) … #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) … /* DC */ #define DC_PERF_CNTR(name, counter, flags) … #define DC_PERF_CNTR_LCB(name, counter, flags) … /* ibp counters */ #define SW_IBP_CNTR(name, cntr) … /** * hfi1_addr_from_offset - return addr for readq/writeq * @dd: the dd device * @offset: the offset of the CSR within bar0 * * This routine selects the appropriate base address * based on the indicated offset. */ static inline void __iomem *hfi1_addr_from_offset( const struct hfi1_devdata *dd, u32 offset) { … } /** * read_csr - read CSR at the indicated offset * @dd: the dd device * @offset: the offset of the CSR within bar0 * * Return: the value read or all FF's if there * is no mapping */ u64 read_csr(const struct hfi1_devdata *dd, u32 offset) { … } /** * write_csr - write CSR at the indicated offset * @dd: the dd device * @offset: the offset of the CSR within bar0 * @value: value to write */ void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value) { … } /** * get_csr_addr - return te iomem address for offset * @dd: the dd device * @offset: the offset of the CSR within bar0 * * Return: The iomem address to use in subsequent * writeq/readq operations. */ void __iomem *get_csr_addr( const struct hfi1_devdata *dd, u32 offset) { … } static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr, int mode, u64 value) { … } /* Dev Access */ static u64 dev_access_u32_csr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sde_err_cnt(const struct cntr_entry *entry, void *context, int idx, int mode, u64 data) { … } static u64 access_sde_int_cnt(const struct cntr_entry *entry, void *context, int idx, int mode, u64 data) { … } static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry, void *context, int idx, int mode, u64 data) { … } static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry, void *context, int idx, int mode, u64 data) { … } static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* Port Access */ static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 port_access_u64_csr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* Software defined */ static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode, u64 data) { … } static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_xmit_discards(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_xmit_constraint_errs(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rcv_constraint_errs(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } u64 get_all_cpu_total(u64 __percpu *cntr) { … } static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val, u64 __percpu *cntr, int vl, int mode, u64 data) { … } static u64 access_sw_cpu_intr(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_pio_wait(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_pio_drain(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_vtx_wait(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_kmem_wait(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sw_send_schedule(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* Software counters for the error status bits within MISC_ERR_STATUS */ static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_efuse_read_bad_addr_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_csr_write_bad_addr_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counter for the aggregate of * individual CceErrStatus counters */ static u64 access_sw_cce_err_status_aggregated_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within CceErrStatus */ static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_rxdma_conv_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_rcpl_async_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_la_triggered_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_transmit_back_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_transmit_front_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_n_post_dat_q_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cli2_async_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_cli0_async_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_trgt_async_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within RcvErrStatus */ static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_data_fifo_rd_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_data_fifo_rd_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_hq_intr_csr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_csr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_rcv_array_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_rcv_array_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_des_part2_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_des_part1_unc_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_lookup_des_part1_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_next_free_buf_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_next_free_buf_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_fl_initdone_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_block_list_read_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_block_list_read_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_qp_map_table_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_qp_map_table_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendPioErrStatus */ static u64 access_pio_pec_sop_head_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_pcc_sop_head_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_last_returned_cnt_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_current_free_cnt_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_ppmc_bqc_mem_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_block_qw_count_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_qw_valid_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_credit_ret_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_v1_len_mem_bank1_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_v1_len_mem_bank0_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_v1_len_mem_bank1_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_v1_len_mem_bank0_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_sm_pkt_reset_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_pkt_evict_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_sbrdctl_crrel_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendDmaErrStatus */ static u64 access_sdma_pcie_req_tracking_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_pcie_req_tracking_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendEgressErrStatus */ static u64 access_tx_read_pio_memory_csr_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_read_sdma_memory_csr_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_read_sdma_memory_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_read_sdma_memory_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_credit_return_partiy_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma15_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma14_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma13_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma12_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma11_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma10_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma9_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma8_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma7_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma6_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma5_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma4_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma3_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma2_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma1_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma0_disallowed_packet_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_sdma_launch_intf_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_pio_launch_intf_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_incorrect_link_state_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_pkt_integrity_mem_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_tx_pkt_integrity_mem_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendErrStatus */ static u64 access_send_csr_write_bad_addr_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendCtxtErrStatus */ static u64 access_pio_write_out_of_bounds_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_write_crosses_boundary_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } /* * Software counters corresponding to each of the * error status bits within SendDmaEngErrStatus */ static u64 access_sdma_header_request_fifo_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_storage_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_packet_tracking_cor_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_request_fifo_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_storage_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_packet_tracking_unc_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_packet_desc_overflow_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_tail_out_of_bounds_err_cnt( const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { … } #define def_access_sw_cpu(cntr) … def_access_sw_cpu(rc_acks); def_access_sw_cpu(rc_qacks); def_access_sw_cpu(rc_delayed_comp); #define def_access_ibp_counter(cntr) … def_access_ibp_counter(loop_pkts); def_access_ibp_counter(rc_resends); def_access_ibp_counter(rnr_naks); def_access_ibp_counter(other_naks); def_access_ibp_counter(rc_timeouts); def_access_ibp_counter(pkt_drops); def_access_ibp_counter(dmawait); def_access_ibp_counter(rc_seqnak); def_access_ibp_counter(rc_dupreq); def_access_ibp_counter(rdma_seq); def_access_ibp_counter(unaligned); def_access_ibp_counter(seq_naks); def_access_ibp_counter(rc_crwaits); static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = …; static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = …; /* ======================================================================== */ /* return true if this is chip revision revision a */ int is_ax(struct hfi1_devdata *dd) { … } /* return true if this is chip revision revision b */ int is_bx(struct hfi1_devdata *dd) { … } /* return true is kernel urg disabled for rcd */ bool is_urg_masked(struct hfi1_ctxtdata *rcd) { … } /* * Append string s to buffer buf. Arguments curp and len are the current * position and remaining length, respectively. * * return 0 on success, 1 on out of room */ static int append_str(char *buf, char **curp, int *lenp, const char *s) { … } /* * Using the given flag table, print a comma separated string into * the buffer. End in '*' if the buffer is too short. */ static char *flag_string(char *buf, int buf_len, u64 flags, const struct flag_table *table, int table_size) { … } /* first 8 CCE error interrupt source names */ static const char * const cce_misc_names[] = …; /* * Return the miscellaneous error interrupt name. */ static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the SDMA engine error interrupt name. */ static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the send context error interrupt name. */ static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source) { … } static const char * const various_names[] = …; /* * Return the various interrupt name. */ static char *is_various_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the DC interrupt name. */ static char *is_dc_name(char *buf, size_t bsize, unsigned int source) { … } static const char * const sdma_int_names[] = …; /* * Return the SDMA engine interrupt name. */ static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the receive available interrupt name. */ static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the receive urgent interrupt name. */ static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the send credit interrupt name. */ static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source) { … } /* * Return the reserved interrupt name. */ static char *is_reserved_name(char *buf, size_t bsize, unsigned int source) { … } static char *cce_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *rxe_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *misc_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *pio_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *sdma_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *egress_err_status_string(char *buf, int buf_len, u64 flags) { … } static char *egress_err_info_string(char *buf, int buf_len, u64 flags) { … } static char *send_err_status_string(char *buf, int buf_len, u64 flags) { … } static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } /* * Check counters for receive errors that do not have an interrupt * associated with them. */ #define RCVERR_CHECK_TIME … static void update_rcverr_timer(struct timer_list *t) { … } static int init_rcverr(struct hfi1_devdata *dd) { … } static void free_rcverr(struct hfi1_devdata *dd) { … } static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static inline void __count_port_discards(struct hfi1_pportdata *ppd) { … } static void count_port_inactive(struct hfi1_devdata *dd) { … } /* * We have had a "disallowed packet" error during egress. Determine the * integrity check which failed, and update relevant error counter, etc. * * Note that the SEND_EGRESS_ERR_INFO register has only a single * bit of state per integrity check, and so we can miss the reason for an * egress error if more than one packet fails the same integrity check * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO. */ static void handle_send_egress_err_info(struct hfi1_devdata *dd, int vl) { … } /* * Input value is a bit position within the SEND_EGRESS_ERR_STATUS * register. Does it represent a 'port inactive' error? */ static inline int port_inactive_err(u64 posn) { … } /* * Input value is a bit position within the SEND_EGRESS_ERR_STATUS * register. Does it represent a 'disallowed packet' error? */ static inline int disallowed_pkt_err(int posn) { … } /* * Input value is a bit position of one of the SDMA engine disallowed * packet errors. Return which engine. Use of this must be guarded by * disallowed_pkt_err(). */ static inline int disallowed_pkt_engine(int posn) { … } /* * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot * be done. */ static int engine_to_vl(struct hfi1_devdata *dd, int engine) { … } /* * Translate the send context (sofware index) into a VL. Return -1 if the * translation cannot be done. */ static int sc_to_vl(struct hfi1_devdata *dd, int sw_index) { … } static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } /* * The maximum number of times the error clear down will loop before * blocking a repeating error. This value is arbitrary. */ #define MAX_CLEAR_COUNT … /* * Clear and handle an error register. All error interrupts are funneled * through here to have a central location to correctly handle single- * or multi-shot errors. * * For non per-context registers, call this routine with a context value * of 0 so the per-context offset is zero. * * If the handler loops too many times, assume that something is wrong * and can't be fixed, so mask the error bits. */ static void interrupt_clear_down(struct hfi1_devdata *dd, u32 context, const struct err_reg_info *eri) { … } /* * CCE block "misc" interrupt. Source is < 16. */ static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source) { … } static char *send_context_err_status_string(char *buf, int buf_len, u64 flags) { … } /* * Send context error interrupt. Source (hw_context) is < 160. * * All send context errors cause the send context to halt. The normal * clear-down mechanism cannot be used because we cannot clear the * error bits until several other long-running items are done first. * This is OK because with the context halted, nothing else is going * to happen on it anyway. */ static void is_sendctxt_err_int(struct hfi1_devdata *dd, unsigned int hw_context) { … } static void handle_sdma_eng_err(struct hfi1_devdata *dd, unsigned int source, u64 status) { … } /* * CCE block SDMA error interrupt. Source is < 16. */ static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source) { … } /* * CCE block "various" interrupt. Source is < 8. */ static void is_various_int(struct hfi1_devdata *dd, unsigned int source) { … } static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg) { … } static int request_host_lcb_access(struct hfi1_devdata *dd) { … } static int request_8051_lcb_access(struct hfi1_devdata *dd) { … } /* * Set the LCB selector - allow host access. The DCC selector always * points to the host. */ static inline void set_host_lcb_access(struct hfi1_devdata *dd) { … } /* * Clear the LCB selector - allow 8051 access. The DCC selector always * points to the host. */ static inline void set_8051_lcb_access(struct hfi1_devdata *dd) { … } /* * Acquire LCB access from the 8051. If the host already has access, * just increment a counter. Otherwise, inform the 8051 that the * host is taking access. * * Returns: * 0 on success * -EBUSY if the 8051 has control and cannot be disturbed * -errno if unable to acquire access from the 8051 */ int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok) { … } /* * Release LCB access by decrementing the use count. If the count is moving * from 1 to 0, inform 8051 that it has control back. * * Returns: * 0 on success * -errno if unable to release access to the 8051 */ int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok) { … } /* * Initialize LCB access variables and state. Called during driver load, * after most of the initialization is finished. * * The DC default is LCB access on for the host. The driver defaults to * leaving access to the 8051. Assign access now - this constrains the call * to this routine to be after all LCB set-up is done. In particular, after * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts() */ static void init_lcb_access(struct hfi1_devdata *dd) { … } /* * Write a response back to a 8051 request. */ static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data) { … } /* * Handle host requests from the 8051. */ static void handle_8051_request(struct hfi1_pportdata *ppd) { … } /* * Set up allocation unit vaulue. */ void set_up_vau(struct hfi1_devdata *dd, u8 vau) { … } /* * Set up initial VL15 credits of the remote. Assumes the rest of * the CM credit registers are zero from a previous global or credit reset. * Shared limit for VL15 will always be 0. */ void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf) { … } /* * Zero all credit details from the previous connection and * reset the CM manager's internal counters. */ void reset_link_credits(struct hfi1_devdata *dd) { … } /* convert a vCU to a CU */ static u32 vcu_to_cu(u8 vcu) { … } /* convert a CU to a vCU */ static u8 cu_to_vcu(u32 cu) { … } /* convert a vAU to an AU */ static u32 vau_to_au(u8 vau) { … } static void set_linkup_defaults(struct hfi1_pportdata *ppd) { … } /* * Graceful LCB shutdown. This leaves the LCB FIFOs in reset. */ static void lcb_shutdown(struct hfi1_devdata *dd, int abort) { … } /* * This routine should be called after the link has been transitioned to * OFFLINE (OFFLINE state has the side effect of putting the SerDes into * reset). * * The expectation is that the caller of this routine would have taken * care of properly transitioning the link into the correct state. * NOTE: the caller needs to acquire the dd->dc8051_lock lock * before calling this function. */ static void _dc_shutdown(struct hfi1_devdata *dd) { … } static void dc_shutdown(struct hfi1_devdata *dd) { … } /* * Calling this after the DC has been brought out of reset should not * do any damage. * NOTE: the caller needs to acquire the dd->dc8051_lock lock * before calling this function. */ static void _dc_start(struct hfi1_devdata *dd) { … } static void dc_start(struct hfi1_devdata *dd) { … } /* * These LCB adjustments are for the Aurora SerDes core in the FPGA. */ static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd) { … } /* * Handle a SMA idle message * * This is a work-queue function outside of the interrupt. */ void handle_sma_message(struct work_struct *work) { … } static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear) { … } static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add) { … } static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear) { … } /* * Called from all interrupt handlers to start handling an SPC freeze. */ void start_freeze_handling(struct hfi1_pportdata *ppd, int flags) { … } /* * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen, * depending on the "freeze" parameter. * * No need to return an error if it times out, our only option * is to proceed anyway. */ static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze) { … } /* * Do all freeze handling for the RXE block. */ static void rxe_freeze(struct hfi1_devdata *dd) { … } /* * Unfreeze handling for the RXE block - kernel contexts only. * This will also enable the port. User contexts will do unfreeze * handling on a per-context basis as they call into the driver. * */ static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) { … } /* * Non-interrupt SPC freeze handling. * * This is a work-queue function outside of the triggering interrupt. */ void handle_freeze(struct work_struct *work) { … } /** * update_xmit_counters - update PortXmitWait/PortVlXmitWait * counters. * @ppd: info of physical Hfi port * @link_width: new link width after link up or downgrade * * Update the PortXmitWait and PortVlXmitWait counters after * a link up or downgrade event to reflect a link width change. */ static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width) { … } /* * Handle a link up interrupt from the 8051. * * This is a work-queue function outside of the interrupt. */ void handle_link_up(struct work_struct *work) { … } /* * Several pieces of LNI information were cached for SMA in ppd. * Reset these on link down */ static void reset_neighbor_info(struct hfi1_pportdata *ppd) { … } static const char * const link_down_reason_strs[] = …; /* return the neighbor link down reason string */ static const char *link_down_reason_str(u8 reason) { … } /* * Handle a link down interrupt from the 8051. * * This is a work-queue function outside of the interrupt. */ void handle_link_down(struct work_struct *work) { … } void handle_link_bounce(struct work_struct *work) { … } /* * Mask conversion: Capability exchange to Port LTP. The capability * exchange has an implicit 16b CRC that is mandatory. */ static int cap_to_port_ltp(int cap) { … } /* * Convert an OPA Port LTP mask to capability mask */ int port_ltp_to_cap(int port_ltp) { … } /* * Convert a single DC LCB CRC mode to an OPA Port LTP mask. */ static int lcb_to_port_ltp(int lcb_crc) { … } static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) { … } /* * Convert the given link width to the OPA link width bitmask. */ static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) { … } /* * Do a population count on the bottom nibble. */ static const u8 bit_counts[16] = …; static inline u8 nibble_to_count(u8 nibble) { … } /* * Read the active lane information from the 8051 registers and return * their widths. * * Active lane information is found in these 8051 registers: * enable_lane_tx * enable_lane_rx */ static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, u16 *rx_width) { … } /* * Read verify_cap_local_fm_link_width[1] to obtain the link widths. * Valid after the end of VerifyCap and during LinkUp. Does not change * after link up. I.e. look elsewhere for downgrade information. * * Bits are: * + bits [7:4] contain the number of active transmitters * + bits [3:0] contain the number of active receivers * These are numbers 1 through 4 and can be different values if the * link is asymmetric. * * verify_cap_local_fm_link_width[0] retains its original value. */ static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width, u16 *rx_width) { … } /* * Set ppd->link_width_active and ppd->link_width_downgrade_active using * hardware information when the link first comes up. * * The link width is not available until after VerifyCap.AllFramesReceived * (the trigger for handle_verify_cap), so this is outside that routine * and should be called when the 8051 signals linkup. */ void get_linkup_link_widths(struct hfi1_pportdata *ppd) { … } /* * Handle a verify capabilities interrupt from the 8051. * * This is a work-queue function outside of the interrupt. */ void handle_verify_cap(struct work_struct *work) { … } /** * apply_link_downgrade_policy - Apply the link width downgrade enabled * policy against the current active link widths. * @ppd: info of physical Hfi port * @refresh_widths: True indicates link downgrade event * @return: True indicates a successful link downgrade. False indicates * link downgrade event failed and the link will bounce back to * default link width. * * Called when the enabled policy changes or the active link widths * change. * Refresh_widths indicates that a link downgrade occurred. The * link_downgraded variable is set by refresh_widths and * determines the success/failure of the policy application. */ bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, bool refresh_widths) { … } /* * Handle a link downgrade interrupt from the 8051. * * This is a work-queue function outside of the interrupt. */ void handle_link_downgrade(struct work_struct *work) { … } static char *dcc_err_string(char *buf, int buf_len, u64 flags) { … } static char *lcb_err_string(char *buf, int buf_len, u64 flags) { … } static char *dc8051_err_string(char *buf, int buf_len, u64 flags) { … } static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags) { … } static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags) { … } static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static const char * const fm_config_txt[] = …; static const char * const port_rcv_txt[] = …; #define OPA_LDR_FMCONFIG_OFFSET … #define OPA_LDR_PORTRCV_OFFSET … static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg) { … } /* * CCE block DC interrupt. Source is < 8. */ static void is_dc_int(struct hfi1_devdata *dd, unsigned int source) { … } /* * TX block send credit interrupt. Source is < 160. */ static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source) { … } /* * TX block SDMA interrupt. Source is < 48. * * SDMA interrupts are grouped by type: * * 0 - N-1 = SDma * N - 2N-1 = SDmaProgress * 2N - 3N-1 = SDmaIdle */ static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source) { … } /** * is_rcv_avail_int() - User receive context available IRQ handler * @dd: valid dd * @source: logical IRQ source (offset from IS_RCVAVAIL_START) * * RX block receive available interrupt. Source is < 160. * * This is the general interrupt handler for user (PSM) receive contexts, * and can only be used for non-threaded IRQs. */ static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source) { … } /** * is_rcv_urgent_int() - User receive context urgent IRQ handler * @dd: valid dd * @source: logical IRQ source (offset from IS_RCVURGENT_START) * * RX block receive urgent interrupt. Source is < 160. * * NOTE: kernel receive contexts specifically do NOT enable this IRQ. */ static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source) { … } /* * Reserved range interrupt. Should not be called in normal operation. */ static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source) { … } static const struct is_table is_table[] = …; /* * Interrupt source interrupt - called when the given source has an interrupt. * Source is a bit index into an array of 64-bit integers. */ static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) { … } /** * general_interrupt - General interrupt handler * @irq: MSIx IRQ vector * @data: hfi1 devdata * * This is able to correctly handle all non-threaded interrupts. Receive * context DATA IRQs are threaded and are not supported by this handler. * */ irqreturn_t general_interrupt(int irq, void *data) { … } irqreturn_t sdma_interrupt(int irq, void *data) { … } /* * Clear the receive interrupt. Use a read of the interrupt clear CSR * to insure that the write completed. This does NOT guarantee that * queued DMA writes to memory from the chip are pushed. */ static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd) { … } /* force the receive interrupt */ void force_recv_intr(struct hfi1_ctxtdata *rcd) { … } /* * Return non-zero if a packet is present. * * This routine is called when rechecking for packets after the RcvAvail * interrupt has been cleared down. First, do a quick check of memory for * a packet present. If not found, use an expensive CSR read of the context * tail to determine the actual tail. The CSR read is necessary because there * is no method to push pending DMAs to memory other than an interrupt and we * are trying to determine if we need to force an interrupt. */ static inline int check_packet_present(struct hfi1_ctxtdata *rcd) { … } /* * Common code for receive contexts interrupt handlers. * Update traces, increment kernel IRQ counter and * setup ASPM when needed. */ static void receive_interrupt_common(struct hfi1_ctxtdata *rcd) { … } /* * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt * when there are packets present in the queue. When calling * with interrupts enabled please use hfi1_rcd_eoi_intr. * * @rcd: valid receive context */ static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) { … } /** * hfi1_rcd_eoi_intr() - End of Interrupt processing action * * @rcd: Ptr to hfi1_ctxtdata of receive context * * Hold IRQs so we can safely clear the interrupt and * recheck for a packet that may have arrived after the previous * check and the interrupt clear. If a packet arrived, force another * interrupt. This routine can be called at the end of receive packet * processing in interrupt service routines, interrupt service thread * and softirqs */ static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) { … } /** * hfi1_netdev_rx_napi - napi poll function to move eoi inline * @napi: pointer to napi object * @budget: netdev budget */ int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget) { … } /* Receive packet napi handler for netdevs VNIC and AIP */ irqreturn_t receive_context_interrupt_napi(int irq, void *data) { … } /* * Receive packet IRQ handler. This routine expects to be on its own IRQ. * This routine will try to handle packets immediately (latency), but if * it finds too many, it will invoke the thread handler (bandwitdh). The * chip receive interrupt is *not* cleared down until this or the thread (if * invoked) is finished. The intent is to avoid extra interrupts while we * are processing packets anyway. */ irqreturn_t receive_context_interrupt(int irq, void *data) { … } /* * Receive packet thread handler. This expects to be invoked with the * receive interrupt still blocked. */ irqreturn_t receive_context_thread(int irq, void *data) { … } /* ========================================================================= */ u32 read_physical_state(struct hfi1_devdata *dd) { … } u32 read_logical_state(struct hfi1_devdata *dd) { … } static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate) { … } /* * Use the 8051 to read a LCB CSR. */ static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data) { … } /* * Provide a cache for some of the LCB registers in case the LCB is * unavailable. * (The LCB is unavailable in certain link states, for example.) */ struct lcb_datum { … }; static struct lcb_datum lcb_cache[] = …; static void update_lcb_cache(struct hfi1_devdata *dd) { … } static int read_lcb_cache(u32 off, u64 *val) { … } /* * Read an LCB CSR. Access may not be in host control, so check. * Return 0 on success, -EBUSY on failure. */ int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data) { … } /* * Use the 8051 to write a LCB CSR. */ static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data) { … } /* * Write an LCB CSR. Access may not be in host control, so check. * Return 0 on success, -EBUSY on failure. */ int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) { … } /* * Returns: * < 0 = Linux error, not able to get access * > 0 = 8051 command RETURN_CODE */ static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, u64 *out_data) { … } static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) { … } int load_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, u32 config_data) { … } /* * Read the 8051 firmware "registers". Use the RAM directly. Always * set the result, even on error. * Return 0 on success, -errno on failure */ int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, u32 *result) { … } static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management, u8 continuous) { … } static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu, u16 vl15buf, u8 crc_sizes) { … } static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, u8 *flag_bits, u16 *link_widths) { … } static int write_vc_local_link_mode(struct hfi1_devdata *dd, u8 misc_bits, u8 flag_bits, u16 link_widths) { … } static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id, u8 device_rev) { … } static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, u8 *device_rev) { … } int write_host_interface_version(struct hfi1_devdata *dd, u8 version) { … } void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, u8 *ver_patch) { … } static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, u8 *continuous) { … } static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, u8 *vcu, u16 *vl15buf, u8 *crc_sizes) { … } static void read_vc_remote_link_width(struct hfi1_devdata *dd, u8 *remote_tx_rate, u16 *link_widths) { … } static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx) { … } static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls) { … } static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs) { … } void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality) { … } static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc) { … } static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr) { … } static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, u8 *tx_polarity_inversion, u8 *rx_polarity_inversion, u8 *max_rate) { … } static int write_tx_settings(struct hfi1_devdata *dd, u8 enable_lane_tx, u8 tx_polarity_inversion, u8 rx_polarity_inversion, u8 max_rate) { … } /* * Read an idle LCB message. * * Returns 0 on success, -EINVAL on error */ static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out) { … } /* * Read an idle SMA message. To be done in response to a notification from * the 8051. * * Returns 0 on success, -EINVAL on error */ static int read_idle_sma(struct hfi1_devdata *dd, u64 *data) { … } /* * Send an idle LCB message. * * Returns 0 on success, -EINVAL on error */ static int send_idle_message(struct hfi1_devdata *dd, u64 data) { … } /* * Send an idle SMA message. * * Returns 0 on success, -EINVAL on error */ int send_idle_sma(struct hfi1_devdata *dd, u64 message) { … } /* * Initialize the LCB then do a quick link up. This may or may not be * in loopback. * * return 0 on success, -errno on error */ static int do_quick_linkup(struct hfi1_devdata *dd) { … } /* * Do all special steps to set up loopback. */ static int init_loopback(struct hfi1_devdata *dd) { … } /* * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits * used in the Verify Capability link width attribute. */ static u16 opa_to_vc_link_widths(u16 opa_widths) { … } /* * Set link attributes before moving to polling. */ static int set_local_link_attributes(struct hfi1_pportdata *ppd) { … } /* * Call this to start the link. * Do not do anything if the link is disabled. * Returns 0 if link is disabled, moved to polling, or the driver is not ready. */ int start_link(struct hfi1_pportdata *ppd) { … } static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) { … } static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) { … } int reset_qsfp(struct hfi1_pportdata *ppd) { … } static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd, u8 *qsfp_interrupt_status) { … } /* This routine will only be scheduled if the QSFP module present is asserted */ void qsfp_event(struct work_struct *work) { … } void init_qsfp_int(struct hfi1_devdata *dd) { … } /* * Do a one-time initialize of the LCB block. */ static void init_lcb(struct hfi1_devdata *dd) { … } /* * Perform a test read on the QSFP. Return 0 on success, -ERRNO * on error. */ static int test_qsfp_read(struct hfi1_pportdata *ppd) { … } /* * Values for QSFP retry. * * Give up after 10s (20 x 500ms). The overall timeout was empirically * arrived at from experience on a large cluster. */ #define MAX_QSFP_RETRIES … #define QSFP_RETRY_WAIT … /* * Try a QSFP read. If it fails, schedule a retry for later. * Called on first link activation after driver load. */ static void try_start_link(struct hfi1_pportdata *ppd) { … } /* * Workqueue function to start the link after a delay. */ void handle_start_link(struct work_struct *work) { … } int bringup_serdes(struct hfi1_pportdata *ppd) { … } void hfi1_quiet_serdes(struct hfi1_pportdata *ppd) { … } static inline int init_cpu_counters(struct hfi1_devdata *dd) { … } /* * index is the index into the receive array */ void hfi1_put_tid(struct hfi1_devdata *dd, u32 index, u32 type, unsigned long pa, u16 order) { … } void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) { … } static const char * const ib_cfg_name_strings[] = …; static const char *ib_cfg_name(int which) { … } int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) { … } /* * The largest MAD packet size. */ #define MAX_MAD_PACKET … /* * Return the maximum header bytes that can go on the _wire_ * for this device. This count includes the ICRC which is * not part of the packet held in memory but it is appended * by the HW. * This is dependent on the device's receive header entry size. * HFI allows this to be set per-receive context, but the * driver presently enforces a global value. */ u32 lrh_max_header_bytes(struct hfi1_devdata *dd) { … } /* * Set Send Length * @ppd: per port data * * Set the MTU by limiting how many DWs may be sent. The SendLenCheck* * registers compare against LRH.PktLen, so use the max bytes included * in the LRH. * * This routine changes all VL values except VL15, which it maintains at * the same value. */ static void set_send_length(struct hfi1_pportdata *ppd) { … } static void set_lidlmc(struct hfi1_pportdata *ppd) { … } static const char *state_completed_string(u32 completed) { … } static const char all_lanes_dead_timeout_expired[] = …; static const char tx_out_of_policy[] = …; static const char no_state_complete[] = …; static const char * const state_complete_reasons[] = …; static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd, u32 code) { … } /* describe the given last state complete frame */ static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame, const char *prefix) { … } /* * Read the last state complete frames and explain them. This routine * expects to be called if the link went down during link negotiation * and initialization (LNI). That is, anywhere between polling and link up. */ static void check_lni_states(struct hfi1_pportdata *ppd) { … } /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */ static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms) { … } /* called when the logical link state is not down as it should be */ static void force_logical_link_state_down(struct hfi1_pportdata *ppd) { … } /* * Helper for set_link_state(). Do not call except from that routine. * Expects ppd->hls_mutex to be held. * * @rem_reason value to be sent to the neighbor * * LinkDownReasons only set if transition succeeds. */ static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) { … } /* return the link state name */ static const char *link_state_name(u32 state) { … } /* return the link state reason name */ static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state) { … } /* * driver_pstate - convert the driver's notion of a port's * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*). * Return -1 (converted to a u32) to indicate error. */ u32 driver_pstate(struct hfi1_pportdata *ppd) { … } /* * driver_lstate - convert the driver's notion of a port's * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1 * (converted to a u32) to indicate error. */ u32 driver_lstate(struct hfi1_pportdata *ppd) { … } void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, u8 neigh_reason, u8 rem_reason) { … } /** * data_vls_operational() - Verify if data VL BCT credits and MTU * are both set. * @ppd: pointer to hfi1_pportdata structure * * Return: true - Ok, false -otherwise. */ static inline bool data_vls_operational(struct hfi1_pportdata *ppd) { … } /* * Change the physical and/or logical link state. * * Do not call this routine while inside an interrupt. It contains * calls to routines that can take multiple seconds to finish. * * Returns 0 on success, -errno on failure. */ int set_link_state(struct hfi1_pportdata *ppd, u32 state) { … } int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val) { … } /* begin functions related to vl arbitration table caching */ static void init_vl_arb_caches(struct hfi1_pportdata *ppd) { … } /* * vl_arb_lock_cache * * All other vl_arb_* functions should be called only after locking * the cache. */ static inline struct vl_arb_cache * vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx) { … } static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx) { … } static void vl_arb_get_cache(struct vl_arb_cache *cache, struct ib_vl_weight_elem *vl) { … } static void vl_arb_set_cache(struct vl_arb_cache *cache, struct ib_vl_weight_elem *vl) { … } static int vl_arb_match_cache(struct vl_arb_cache *cache, struct ib_vl_weight_elem *vl) { … } /* end functions related to vl arbitration table caching */ static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target, u32 size, struct ib_vl_weight_elem *vl) { … } /* * Read one credit merge VL register. */ static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr, struct vl_limit *vll) { … } /* * Read the current credit merge limits. */ static int get_buffer_control(struct hfi1_devdata *dd, struct buffer_control *bc, u16 *overall_limit) { … } static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) { … } static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems, struct ib_vl_weight_elem *vl) { … } static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) { … } static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what, u16 limit) { … } /* change only the shared limit portion of SendCmGLobalCredit */ static void set_global_shared(struct hfi1_devdata *dd, u16 limit) { … } /* change only the total credit limit portion of SendCmGLobalCredit */ static void set_global_limit(struct hfi1_devdata *dd, u16 limit) { … } /* set the given per-VL shared limit */ static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit) { … } /* set the given per-VL dedicated limit */ static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit) { … } /* spin until the given per-VL status mask bits clear */ static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask, const char *which) { … } /* * The number of credits on the VLs may be changed while everything * is "live", but the following algorithm must be followed due to * how the hardware is actually implemented. In particular, * Return_Credit_Status[] is the only correct status check. * * if (reducing Global_Shared_Credit_Limit or any shared limit changing) * set Global_Shared_Credit_Limit = 0 * use_all_vl = 1 * mask0 = all VLs that are changing either dedicated or shared limits * set Shared_Limit[mask0] = 0 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0 * if (changing any dedicated limit) * mask1 = all VLs that are lowering dedicated limits * lower Dedicated_Limit[mask1] * spin until Return_Credit_Status[mask1] == 0 * raise Dedicated_Limits * raise Shared_Limits * raise Global_Shared_Credit_Limit * * lower = if the new limit is lower, set the limit to the new value * raise = if the new limit is higher than the current value (may be changed * earlier in the algorithm), set the new limit to the new value */ int set_buffer_control(struct hfi1_pportdata *ppd, struct buffer_control *new_bc) { … } /* * Read the given fabric manager table. Return the size of the * table (in bytes) on success, and a negative error code on * failure. */ int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t) { … } /* * Write the given fabric manager table. */ int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t) { … } /* * Disable all data VLs. * * Return 0 if disabled, non-zero if the VLs cannot be disabled. */ static int disable_data_vls(struct hfi1_devdata *dd) { … } /* * open_fill_data_vls() - the counterpart to stop_drain_data_vls(). * Just re-enables all data VLs (the "fill" part happens * automatically - the name was chosen for symmetry with * stop_drain_data_vls()). * * Return 0 if successful, non-zero if the VLs cannot be enabled. */ int open_fill_data_vls(struct hfi1_devdata *dd) { … } /* * drain_data_vls() - assumes that disable_data_vls() has been called, * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA * engines to drop to 0. */ static void drain_data_vls(struct hfi1_devdata *dd) { … } /* * stop_drain_data_vls() - disable, then drain all per-VL fifos. * * Use open_fill_data_vls() to resume using data VLs. This pair is * meant to be used like this: * * stop_drain_data_vls(dd); * // do things with per-VL resources * open_fill_data_vls(dd); */ int stop_drain_data_vls(struct hfi1_devdata *dd) { … } /* * Convert a nanosecond time to a cclock count. No matter how slow * the cclock, a non-zero ns will always have a non-zero result. */ u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns) { … } /* * Convert a cclock count to nanoseconds. Not matter how slow * the cclock, a non-zero cclocks will always have a non-zero result. */ u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks) { … } /* * Dynamically adjust the receive interrupt timeout for a context based on * incoming packet rate. * * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero. */ static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts) { … } void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, u32 intr_adjust, u32 npkts) { … } u32 hdrqempty(struct hfi1_ctxtdata *rcd) { … } /* * Context Control and Receive Array encoding for buffer size: * 0x0 invalid * 0x1 4 KB * 0x2 8 KB * 0x3 16 KB * 0x4 32 KB * 0x5 64 KB * 0x6 128 KB * 0x7 256 KB * 0x8 512 KB (Receive Array only) * 0x9 1 MB (Receive Array only) * 0xa 2 MB (Receive Array only) * * 0xB-0xF - reserved (Receive Array only) * * * This routine assumes that the value has already been sanity checked. */ static u32 encoded_size(u32 size) { … } /** * encode_rcv_header_entry_size - return chip specific encoding for size * @size: size in dwords * * Convert a receive header entry size that to the encoding used in the CSR. * * Return a zero if the given size is invalid, otherwise the encoding. */ u8 encode_rcv_header_entry_size(u8 size) { … } /** * hfi1_validate_rcvhdrcnt - validate hdrcnt * @dd: the device data * @thecnt: the header count */ int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) { … } /** * set_hdrq_regs - set header queue registers for context * @dd: the device data * @ctxt: the context * @entsize: the dword entry size * @hdrcnt: the number of header entries */ void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt) { … } void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, struct hfi1_ctxtdata *rcd) { … } u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp) { … } /* * Used by sysfs to create files for hfi stats to read */ u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp) { … } static void free_cntrs(struct hfi1_devdata *dd) { … } static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, u64 *psval, void *context, int vl) { … } static u64 write_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, u64 *psval, void *context, int vl, u64 data) { … } u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl) { … } u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data) { … } u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl) { … } u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data) { … } static void do_update_synth_timer(struct work_struct *work) { … } static void update_synth_timer(struct timer_list *t) { … } #define C_MAX_NAME … static int init_cntrs(struct hfi1_devdata *dd) { … } static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) { … } u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) { … } /* return the OPA port logical state name */ const char *opa_lstate_name(u32 lstate) { … } /* return the OPA port physical state name */ const char *opa_pstate_name(u32 pstate) { … } /** * update_statusp - Update userspace status flag * @ppd: Port data structure * @state: port state information * * Actual port status is determined by the host_link_state value * in the ppd. * * host_link_state MUST be updated before updating the user space * statusp. */ static void update_statusp(struct hfi1_pportdata *ppd, u32 state) { … } /** * wait_logical_linkstate - wait for an IB link state change to occur * @ppd: port device * @state: the state to wait for * @msecs: the number of milliseconds to wait * * Wait up to msecs milliseconds for IB link state change to occur. * For now, take the easy polling route. * Returns 0 if state reached, otherwise -ETIMEDOUT. */ static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, int msecs) { … } static void log_state_transition(struct hfi1_pportdata *ppd, u32 state) { … } /* * Read the physical hardware link state and check if it matches host * drivers anticipated state. */ static void log_physical_state(struct hfi1_pportdata *ppd, u32 state) { … } /* * wait_physical_linkstate - wait for an physical link state change to occur * @ppd: port device * @state: the state to wait for * @msecs: the number of milliseconds to wait * * Wait up to msecs milliseconds for physical link state change to occur. * Returns 0 if state reached, otherwise -ETIMEDOUT. */ static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, int msecs) { … } /* * wait_phys_link_offline_quiet_substates - wait for any offline substate * @ppd: port device * @msecs: the number of milliseconds to wait * * Wait up to msecs milliseconds for any offline physical link * state change to occur. * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. */ static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, int msecs) { … } /* * wait_phys_link_out_of_offline - wait for any out of offline state * @ppd: port device * @msecs: the number of milliseconds to wait * * Wait up to msecs milliseconds for any out of offline physical link * state change to occur. * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. */ static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, int msecs) { … } #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) … #define SET_STATIC_RATE_CONTROL_SMASK(r) … void hfi1_init_ctxt(struct send_context *sc) { … } int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp) { … } /* ========================================================================= */ /** * read_mod_write() - Calculate the IRQ register index and set/clear the bits * @dd: valid devdata * @src: IRQ source to determine register index from * @bits: the bits to set or clear * @set: true == set the bits, false == clear the bits * */ static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits, bool set) { … } /** * set_intr_bits() - Enable/disable a range (one or more) IRQ sources * @dd: valid devdata * @first: first IRQ source to set/clear * @last: last IRQ source (inclusive) to set/clear * @set: true == set the bits, false == clear the bits * * If first == last, set the exact source. */ int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set) { … } /* * Clear all interrupt sources on the chip. */ void clear_all_interrupts(struct hfi1_devdata *dd) { … } /* * Remap the interrupt source from the general handler to the given MSI-X * interrupt. */ void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr) { … } void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr) { … } /* * Set the general handler to accept all interrupts, remap all * chip interrupts back to MSI-X 0. */ void reset_interrupts(struct hfi1_devdata *dd) { … } /** * set_up_interrupts() - Initialize the IRQ resources and state * @dd: valid devdata * */ static int set_up_interrupts(struct hfi1_devdata *dd) { … } /* * Set up context values in dd. Sets: * * num_rcv_contexts - number of contexts being used * n_krcv_queues - number of kernel contexts * first_dyn_alloc_ctxt - first dynamically allocated context * in array of contexts * freectxts - number of free user contexts * num_send_contexts - number of PIO send contexts being used * num_netdev_contexts - number of contexts reserved for netdev */ static int set_up_context_variables(struct hfi1_devdata *dd) { … } /* * Set the device/port partition key table. The MAD code * will ensure that, at least, the partial management * partition key is present in the table. */ static void set_partition_keys(struct hfi1_pportdata *ppd) { … } /* * These CSRs and memories are uninitialized on reset and must be * written before reading to set the ECC/parity bits. * * NOTE: All user context CSRs that are not mmaped write-only * (e.g. the TID flows) must be initialized even if the driver never * reads them. */ static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd) { … } /* * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus. */ static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits, u64 ctrl_bits) { … } /* set CCE CSRs to chip reset defaults */ static void reset_cce_csrs(struct hfi1_devdata *dd) { … } /* set MISC CSRs to chip reset defaults */ static void reset_misc_csrs(struct hfi1_devdata *dd) { … } /* set TXE CSRs to chip reset defaults */ static void reset_txe_csrs(struct hfi1_devdata *dd) { … } /* * Expect on entry: * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0 */ static void init_rbufs(struct hfi1_devdata *dd) { … } /* set RXE CSRs to chip reset defaults */ static void reset_rxe_csrs(struct hfi1_devdata *dd) { … } /* * Set sc2vl tables. * * They power on to zeros, so to avoid send context errors * they need to be set: * * SC 0-7 -> VL 0-7 (respectively) * SC 15 -> VL 15 * otherwise * -> VL 0 */ static void init_sc2vl_tables(struct hfi1_devdata *dd) { … } /* * Read chip sizes and then reset parts to sane, disabled, values. We cannot * depend on the chip going through a power-on reset - a driver may be loaded * and unloaded many times. * * Do not write any CSR values to the chip in this routine - there may be * a reset following the (possible) FLR in this routine. * */ static int init_chip(struct hfi1_devdata *dd) { … } static void init_early_variables(struct hfi1_devdata *dd) { … } static void init_kdeth_qp(struct hfi1_devdata *dd) { … } /** * hfi1_get_qp_map - get qp map * @dd: device data * @idx: index to read */ u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx) { … } /** * init_qpmap_table - init qp map * @dd: device data * @first_ctxt: first context * @last_ctxt: first context * * This return sets the qpn mapping table that * is indexed by qpn[8:1]. * * The routine will round robin the 256 settings * from first_ctxt to last_ctxt. * * The first/last looks ahead to having specialized * receive contexts for mgmt and bypass. Normal * verbs traffic will assumed to be on a range * of receive contexts. */ static void init_qpmap_table(struct hfi1_devdata *dd, u32 first_ctxt, u32 last_ctxt) { … } struct rsm_map_table { … }; struct rsm_rule_data { … }; /* * Return an initialized RMT map table for users to fill in. OK if it * returns NULL, indicating no table. */ static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd) { … } /* * Write the final RMT map table to the chip and free the table. OK if * table is NULL. */ static void complete_rsm_map_table(struct hfi1_devdata *dd, struct rsm_map_table *rmt) { … } /* Is a receive side mapping rule */ static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) { … } /* * Add a receive side mapping rule. */ static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index, struct rsm_rule_data *rrd) { … } /* * Clear a receive side mapping rule. */ static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) { … } /* return the number of RSM map table entries that will be used for QOS */ static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, unsigned int *np) { … } /** * init_qos - init RX qos * @dd: device data * @rmt: RSM map table * * This routine initializes Rule 0 and the RSM map table to implement * quality of service (qos). * * If all of the limit tests succeed, qos is applied based on the array * interpretation of krcvqs where entry 0 is VL0. * * The number of vl bits (n) and the number of qpn bits (m) are computed to * feed both the RSM map table and the single rule. */ static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) { … } static void init_fecn_handling(struct hfi1_devdata *dd, struct rsm_map_table *rmt) { … } static inline bool hfi1_is_rmt_full(int start, int spare) { … } static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd) { … } static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd, int rule, struct rsm_rule_data *rrd) { … } void hfi1_init_aip_rsm(struct hfi1_devdata *dd) { … } /* Initialize RSM for VNIC */ void hfi1_init_vnic_rsm(struct hfi1_devdata *dd) { … } void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd) { … } void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd) { … } static int init_rxe(struct hfi1_devdata *dd) { … } static void init_other(struct hfi1_devdata *dd) { … } /* * Fill out the given AU table using the given CU. A CU is defined in terms * AUs. The table is a an encoding: given the index, how many AUs does that * represent? * * NOTE: Assumes that the register layout is the same for the * local and remote tables. */ static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu, u32 csr0to3, u32 csr4to7) { … } static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu) { … } void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu) { … } static void init_txe(struct hfi1_devdata *dd) { … } int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, u16 jkey) { … } int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) { … } int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, u16 pkey) { … } int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt) { … } /* * Start doing the clean up the chip. Our clean up happens in multiple * stages and this is just the first. */ void hfi1_start_cleanup(struct hfi1_devdata *dd) { … } #define HFI_BASE_GUID(dev) … /* * Information can be shared between the two HFIs on the same ASIC * in the same OS. This function finds the peer device and sets * up a shared structure. */ static int init_asic_data(struct hfi1_devdata *dd) { … } /* * Set dd->boardname. Use a generic name if a name is not returned from * EFI variable space. * * Return 0 on success, -ENOMEM if space could not be allocated. */ static int obtain_boardname(struct hfi1_devdata *dd) { … } /* * Check the interrupt registers to make sure that they are mapped correctly. * It is intended to help user identify any mismapping by VMM when the driver * is running in a VM. This function should only be called before interrupt * is set up properly. * * Return 0 on success, -EINVAL on failure. */ static int check_int_registers(struct hfi1_devdata *dd) { … } /** * hfi1_init_dd() - Initialize most of the dd structure. * @dd: the dd device * * This is global, and is called directly at init to set up the * chip-specific function pointers for later use. */ int hfi1_init_dd(struct hfi1_devdata *dd) { … } static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate, u32 dw_len) { … } /** * create_pbc - build a pbc for transmission * @ppd: info of physical Hfi port * @flags: special case flags or-ed in built pbc * @srate_mbs: static rate * @vl: vl * @dw_len: dword length (header words + data words + pbc words) * * Create a PBC with the given flags, rate, VL, and length. * * NOTE: The PBC created will not insert any HCRC - all callers but one are * for verbs, which does not use this PSM feature. The lone other caller * is for the diagnostic interface which calls this if the user does not * supply their own PBC. */ u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, u32 dw_len) { … } #define SBUS_THERMAL … #define SBUS_THERM_MONITOR_MODE … #define THERM_FAILURE(dev, ret, reason) … /* * Initialize the thermal sensor. * * After initialization, enable polling of thermal sensor through * SBus interface. In order for this to work, the SBus Master * firmware has to be loaded due to the fact that the HW polling * logic uses SBus interrupts, which are not supported with * default firmware. Otherwise, no data will be returned through * the ASIC_STS_THERM CSR. */ static int thermal_init(struct hfi1_devdata *dd) { … } static void handle_temp_err(struct hfi1_devdata *dd) { … }
Codebase Browser
linux