// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) /* QLogic qed NIC Driver * Copyright (c) 2015-2017 QLogic Corporation * Copyright (c) 2019-2020 Marvell International Ltd. */ #include <linux/types.h> #include <asm/byteorder.h> #include <linux/io.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/errno.h> #include <linux/kernel.h> #include <linux/mutex.h> #include <linux/pci.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/vmalloc.h> #include <linux/etherdevice.h> #include <linux/qed/qed_chain.h> #include <linux/qed/qed_if.h> #include "qed.h" #include "qed_cxt.h" #include "qed_dcbx.h" #include "qed_dev_api.h" #include "qed_fcoe.h" #include "qed_hsi.h" #include "qed_iro_hsi.h" #include "qed_hw.h" #include "qed_init_ops.h" #include "qed_int.h" #include "qed_iscsi.h" #include "qed_ll2.h" #include "qed_mcp.h" #include "qed_ooo.h" #include "qed_reg_addr.h" #include "qed_sp.h" #include "qed_sriov.h" #include "qed_vf.h" #include "qed_rdma.h" #include "qed_nvmetcp.h" static DEFINE_SPINLOCK(qm_lock); /******************** Doorbell Recovery *******************/ /* The doorbell recovery mechanism consists of a list of entries which represent * doorbelling entities (l2 queues, roce sq/rq/cqs, the slowpath spq, etc). Each * entity needs to register with the mechanism and provide the parameters * describing it's doorbell, including a location where last used doorbell data * can be found. The doorbell execute function will traverse the list and * doorbell all of the registered entries. */ struct qed_db_recovery_entry { … }; /* Display a single doorbell recovery entry */ static void qed_db_recovery_dp_entry(struct qed_hwfn *p_hwfn, struct qed_db_recovery_entry *db_entry, char *action) { … } /* Doorbell address sanity (address within doorbell bar range) */ static bool qed_db_rec_sanity(struct qed_dev *cdev, void __iomem *db_addr, enum qed_db_rec_width db_width, void *db_data) { … } /* Find hwfn according to the doorbell address */ static struct qed_hwfn *qed_db_rec_find_hwfn(struct qed_dev *cdev, void __iomem *db_addr) { … } /* Add a new entry to the doorbell recovery mechanism */ int qed_db_recovery_add(struct qed_dev *cdev, void __iomem *db_addr, void *db_data, enum qed_db_rec_width db_width, enum qed_db_rec_space db_space) { … } /* Remove an entry from the doorbell recovery mechanism */ int qed_db_recovery_del(struct qed_dev *cdev, void __iomem *db_addr, void *db_data) { … } /* Initialize the doorbell recovery mechanism */ static int qed_db_recovery_setup(struct qed_hwfn *p_hwfn) { … } /* Destroy the doorbell recovery mechanism */ static void qed_db_recovery_teardown(struct qed_hwfn *p_hwfn) { … } /* Print the content of the doorbell recovery mechanism */ void qed_db_recovery_dp(struct qed_hwfn *p_hwfn) { … } /* Ring the doorbell of a single doorbell recovery entry */ static void qed_db_recovery_ring(struct qed_hwfn *p_hwfn, struct qed_db_recovery_entry *db_entry) { … } /* Traverse the doorbell recovery entry list and ring all the doorbells */ void qed_db_recovery_execute(struct qed_hwfn *p_hwfn) { … } /******************** Doorbell Recovery end ****************/ /********************************** NIG LLH ***********************************/ enum qed_llh_filter_type { … }; struct qed_llh_mac_filter { … }; struct qed_llh_protocol_filter { … }; qed_llh_filter; struct qed_llh_filter_info { … }; struct qed_llh_info { … }; static void qed_llh_free(struct qed_dev *cdev) { … } static int qed_llh_alloc(struct qed_dev *cdev) { … } static int qed_llh_shadow_sanity(struct qed_dev *cdev, u8 ppfid, u8 filter_idx, const char *action) { … } #define QED_LLH_INVALID_FILTER_IDX … static int qed_llh_shadow_search_filter(struct qed_dev *cdev, u8 ppfid, union qed_llh_filter *p_filter, u8 *p_filter_idx) { … } static int qed_llh_shadow_get_free_idx(struct qed_dev *cdev, u8 ppfid, u8 *p_filter_idx) { … } static int __qed_llh_shadow_add_filter(struct qed_dev *cdev, u8 ppfid, u8 filter_idx, enum qed_llh_filter_type type, union qed_llh_filter *p_filter, u32 *p_ref_cnt) { … } static int qed_llh_shadow_add_filter(struct qed_dev *cdev, u8 ppfid, enum qed_llh_filter_type type, union qed_llh_filter *p_filter, u8 *p_filter_idx, u32 *p_ref_cnt) { … } static int __qed_llh_shadow_remove_filter(struct qed_dev *cdev, u8 ppfid, u8 filter_idx, u32 *p_ref_cnt) { … } static int qed_llh_shadow_remove_filter(struct qed_dev *cdev, u8 ppfid, union qed_llh_filter *p_filter, u8 *p_filter_idx, u32 *p_ref_cnt) { … } static int qed_llh_abs_ppfid(struct qed_dev *cdev, u8 ppfid, u8 *p_abs_ppfid) { … } static int qed_llh_set_engine_affin(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_llh_hw_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } u8 qed_llh_get_num_ppfid(struct qed_dev *cdev) { … } #define NIG_REG_PPF_TO_ENGINE_SEL_ROCE_MASK … #define NIG_REG_PPF_TO_ENGINE_SEL_ROCE_SHIFT … #define NIG_REG_PPF_TO_ENGINE_SEL_NON_ROCE_MASK … #define NIG_REG_PPF_TO_ENGINE_SEL_NON_ROCE_SHIFT … int qed_llh_set_ppfid_affinity(struct qed_dev *cdev, u8 ppfid, enum qed_eng eng) { … } int qed_llh_set_roce_affinity(struct qed_dev *cdev, enum qed_eng eng) { … } struct qed_llh_filter_details { … }; static int qed_llh_access_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u8 abs_ppfid, u8 filter_idx, struct qed_llh_filter_details *p_details) { … } static int qed_llh_add_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u8 abs_ppfid, u8 filter_idx, u8 filter_prot_type, u32 high, u32 low) { … } static int qed_llh_remove_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u8 abs_ppfid, u8 filter_idx) { … } int qed_llh_add_mac_filter(struct qed_dev *cdev, u8 ppfid, const u8 mac_addr[ETH_ALEN]) { … } static int qed_llh_protocol_filter_stringify(struct qed_dev *cdev, enum qed_llh_prot_filter_type_t type, u16 source_port_or_eth_type, u16 dest_port, u8 *str, size_t str_len) { … } static int qed_llh_protocol_filter_to_hilo(struct qed_dev *cdev, enum qed_llh_prot_filter_type_t type, u16 source_port_or_eth_type, u16 dest_port, u32 *p_high, u32 *p_low) { … } int qed_llh_add_protocol_filter(struct qed_dev *cdev, u8 ppfid, enum qed_llh_prot_filter_type_t type, u16 source_port_or_eth_type, u16 dest_port) { … } void qed_llh_remove_mac_filter(struct qed_dev *cdev, u8 ppfid, u8 mac_addr[ETH_ALEN]) { … } void qed_llh_remove_protocol_filter(struct qed_dev *cdev, u8 ppfid, enum qed_llh_prot_filter_type_t type, u16 source_port_or_eth_type, u16 dest_port) { … } /******************************* NIG LLH - End ********************************/ #define QED_MIN_DPIS … #define QED_MIN_PWM_REGION … static u32 qed_hw_bar_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, enum BAR_ID bar_id) { … } void qed_init_dp(struct qed_dev *cdev, u32 dp_module, u8 dp_level) { … } void qed_init_struct(struct qed_dev *cdev) { … } static void qed_qm_info_free(struct qed_hwfn *p_hwfn) { … } static void qed_dbg_user_data_free(struct qed_hwfn *p_hwfn) { … } void qed_resc_free(struct qed_dev *cdev) { … } /******************** QM initialization *******************/ #define ACTIVE_TCS_BMAP … #define ACTIVE_TCS_BMAP_4PORT_K2 … /* determines the physical queue flags for a given PF. */ static u32 qed_get_pq_flags(struct qed_hwfn *p_hwfn) { … } /* Getters for resource amounts necessary for qm initialization */ static u8 qed_init_qm_get_num_tcs(struct qed_hwfn *p_hwfn) { … } static u16 qed_init_qm_get_num_vfs(struct qed_hwfn *p_hwfn) { … } static u8 qed_init_qm_get_num_mtc_tcs(struct qed_hwfn *p_hwfn) { … } #define NUM_DEFAULT_RLS … static u16 qed_init_qm_get_num_pf_rls(struct qed_hwfn *p_hwfn) { … } static u16 qed_init_qm_get_num_vports(struct qed_hwfn *p_hwfn) { … } /* calc amount of PQs according to the requested flags */ static u16 qed_init_qm_get_num_pqs(struct qed_hwfn *p_hwfn) { … } /* initialize the top level QM params */ static void qed_init_qm_params(struct qed_hwfn *p_hwfn) { … } /* initialize qm vport params */ static void qed_init_qm_vport_params(struct qed_hwfn *p_hwfn) { … } /* initialize qm port params */ static void qed_init_qm_port_params(struct qed_hwfn *p_hwfn) { … } /* Reset the params which must be reset for qm init. QM init may be called as * a result of flows other than driver load (e.g. dcbx renegotiation). Other * params may be affected by the init but would simply recalculate to the same * values. The allocations made for QM init, ports, vports, pqs and vfqs are not * affected as these amounts stay the same. */ static void qed_init_qm_reset_params(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_advance_vport(struct qed_hwfn *p_hwfn) { … } /* initialize a single pq and manage qm_info resources accounting. * The pq_init_flags param determines whether the PQ is rate limited * (for VF or PF) and whether a new vport is allocated to the pq or not * (i.e. vport will be shared). */ /* flags for pq init */ #define PQ_INIT_SHARE_VPORT … #define PQ_INIT_PF_RL … #define PQ_INIT_VF_RL … /* defines for pq init */ #define PQ_INIT_DEFAULT_WRR_GROUP … #define PQ_INIT_DEFAULT_TC … void qed_hw_info_set_offload_tc(struct qed_hw_info *p_info, u8 tc) { … } static bool qed_is_offload_tc_set(struct qed_hwfn *p_hwfn) { … } static u32 qed_get_offload_tc(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_pq(struct qed_hwfn *p_hwfn, struct qed_qm_info *qm_info, u8 tc, u32 pq_init_flags) { … } /* get pq index according to PQ_FLAGS */ static u16 *qed_init_qm_get_idx_from_flags(struct qed_hwfn *p_hwfn, unsigned long pq_flags) { … } /* save pq index in qm info */ static void qed_init_qm_set_idx(struct qed_hwfn *p_hwfn, u32 pq_flags, u16 pq_val) { … } /* get tx pq index, with the PQ TX base already set (ready for context init) */ u16 qed_get_cm_pq_idx(struct qed_hwfn *p_hwfn, u32 pq_flags) { … } u16 qed_get_cm_pq_idx_mcos(struct qed_hwfn *p_hwfn, u8 tc) { … } u16 qed_get_cm_pq_idx_vf(struct qed_hwfn *p_hwfn, u16 vf) { … } u16 qed_get_cm_pq_idx_ofld_mtc(struct qed_hwfn *p_hwfn, u8 tc) { … } u16 qed_get_cm_pq_idx_llt_mtc(struct qed_hwfn *p_hwfn, u8 tc) { … } /* Functions for creating specific types of pqs */ static void qed_init_qm_lb_pq(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_ooo_pq(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_pure_ack_pq(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_mtc_pqs(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_offload_pq(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_low_latency_pq(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_mcos_pqs(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_vf_pqs(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_rl_pqs(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_pq_params(struct qed_hwfn *p_hwfn) { … } /* compare values of getters against resources amounts */ static int qed_init_qm_sanity(struct qed_hwfn *p_hwfn) { … } static void qed_dp_init_qm_params(struct qed_hwfn *p_hwfn) { … } static void qed_init_qm_info(struct qed_hwfn *p_hwfn) { … } /* This function reconfigures the QM pf on the fly. * For this purpose we: * 1. reconfigure the QM database * 2. set new values to runtime array * 3. send an sdm_qm_cmd through the rbc interface to stop the QM * 4. activate init tool in QM_PF stage * 5. send an sdm_qm_cmd through rbc interface to release the QM */ int qed_qm_reconf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_alloc_qm_data(struct qed_hwfn *p_hwfn) { … } int qed_resc_alloc(struct qed_dev *cdev) { … } static int qed_fw_err_handler(struct qed_hwfn *p_hwfn, u8 opcode, u16 echo, union event_ring_data *data, u8 fw_return_code) { … } static int qed_common_eqe_event(struct qed_hwfn *p_hwfn, u8 opcode, __le16 echo, union event_ring_data *data, u8 fw_return_code) { … } void qed_resc_setup(struct qed_dev *cdev) { … } #define FINAL_CLEANUP_POLL_CNT … #define FINAL_CLEANUP_POLL_TIME … int qed_final_cleanup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 id, bool is_vf) { … } static int qed_calc_hw_mode(struct qed_hwfn *p_hwfn) { … } /* Init run time data for all PFs on an engine. */ static void qed_init_cau_rt_data(struct qed_dev *cdev) { … } static void qed_init_cache_line_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_init_common(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, int hw_mode) { … } static int qed_hw_init_dpi_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 pwm_region_size, u32 n_cpus) { … } enum QED_ROCE_EDPM_MODE { … }; bool qed_edpm_enabled(struct qed_hwfn *p_hwfn) { … } static int qed_hw_init_pf_doorbell_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_init_port(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, int hw_mode) { … } static int qed_hw_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, struct qed_tunnel_info *p_tunn, int hw_mode, bool b_hw_start, enum qed_int_mode int_mode, bool allow_npar_tx_switch) { … } int qed_pglueb_set_pfid_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool b_enable) { … } static void qed_reset_mb_shadow(struct qed_hwfn *p_hwfn, struct qed_ptt *p_main_ptt) { … } static void qed_fill_load_req_params(struct qed_load_req_params *p_load_req, struct qed_drv_load_params *p_drv_load) { … } static int qed_vf_start(struct qed_hwfn *p_hwfn, struct qed_hw_init_params *p_params) { … } static void qed_pglueb_clear_err(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } int qed_hw_init(struct qed_dev *cdev, struct qed_hw_init_params *p_params) { … } #define QED_HW_STOP_RETRY_LIMIT … static void qed_hw_timers_stop(struct qed_dev *cdev, struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } void qed_hw_timers_stop_all(struct qed_dev *cdev) { … } int qed_hw_stop(struct qed_dev *cdev) { … } int qed_hw_stop_fastpath(struct qed_dev *cdev) { … } int qed_hw_start_fastpath(struct qed_hwfn *p_hwfn) { … } /* Free hwfn memory and resources acquired in hw_hwfn_prepare */ static void qed_hw_hwfn_free(struct qed_hwfn *p_hwfn) { … } /* Setup bar access */ static void qed_hw_hwfn_prepare(struct qed_hwfn *p_hwfn) { … } static void get_function_id(struct qed_hwfn *p_hwfn) { … } static void qed_hw_set_feat(struct qed_hwfn *p_hwfn) { … } const char *qed_hw_get_resc_name(enum qed_resources res_id) { … } static int __qed_hw_set_soft_resc_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, enum qed_resources res_id, u32 resc_max_val, u32 *p_mcp_resp) { … } static u32 qed_hsi_def_val[][MAX_CHIP_IDS] = …; u32 qed_get_hsi_def_val(struct qed_dev *cdev, enum qed_hsi_def_type type) { … } static int qed_hw_set_soft_resc_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_get_dflt_resc(struct qed_hwfn *p_hwfn, enum qed_resources res_id, u32 *p_resc_num, u32 *p_resc_start) { … } static int __qed_hw_set_resc_info(struct qed_hwfn *p_hwfn, enum qed_resources res_id) { … } static int qed_hw_set_resc_info(struct qed_hwfn *p_hwfn) { … } static int qed_hw_get_ppfid_bitmap(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_get_resc(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static void qed_get_num_funcs(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static void qed_hw_info_port_num(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static void qed_get_eee_caps(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_get_hw_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, enum qed_pci_personality personality) { … } static int qed_get_dev_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } static int qed_hw_prepare_single(struct qed_hwfn *p_hwfn, void __iomem *p_regview, void __iomem *p_doorbells, u64 db_phys_addr, enum qed_pci_personality personality) { … } int qed_hw_prepare(struct qed_dev *cdev, int personality) { … } void qed_hw_remove(struct qed_dev *cdev) { … } int qed_fw_l2_queue(struct qed_hwfn *p_hwfn, u16 src_id, u16 *dst_id) { … } int qed_fw_vport(struct qed_hwfn *p_hwfn, u8 src_id, u8 *dst_id) { … } int qed_fw_rss_eng(struct qed_hwfn *p_hwfn, u8 src_id, u8 *dst_id) { … } static int qed_set_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 hw_addr, void *p_eth_qzone, size_t eth_qzone_size, u8 timeset) { … } int qed_set_queue_coalesce(u16 rx_coal, u16 tx_coal, void *p_handle) { … } int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 coalesce, struct qed_queue_cid *p_cid) { … } int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 coalesce, struct qed_queue_cid *p_cid) { … } /* Calculate final WFQ values for all vports and configure them. * After this configuration each vport will have * approx min rate = min_pf_rate * (vport_wfq / QED_WFQ_UNIT) */ static void qed_configure_wfq_for_all_vports(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 min_pf_rate) { … } static void qed_init_wfq_default_param(struct qed_hwfn *p_hwfn, u32 min_pf_rate) { … } static void qed_disable_wfq_for_all_vports(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 min_pf_rate) { … } /* This function performs several validations for WFQ * configuration and required min rate for a given vport * 1. req_rate must be greater than one percent of min_pf_rate. * 2. req_rate should not cause other vports [not configured for WFQ explicitly] * rates to get less than one percent of min_pf_rate. * 3. total_req_min_rate [all vports min rate sum] shouldn't exceed min_pf_rate. */ static int qed_init_wfq_param(struct qed_hwfn *p_hwfn, u16 vport_id, u32 req_rate, u32 min_pf_rate) { … } static int __qed_configure_vport_wfq(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 vp_id, u32 rate) { … } static int __qed_configure_vp_wfq_on_link_change(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 min_pf_rate) { … } /* Main API for qed clients to configure vport min rate. * vp_id - vport id in PF Range[0 - (total_num_vports_per_pf - 1)] * rate - Speed in Mbps needs to be assigned to a given vport. */ int qed_configure_vport_wfq(struct qed_dev *cdev, u16 vp_id, u32 rate) { … } /* API to configure WFQ from mcp link change */ void qed_configure_vp_wfq_on_link_change(struct qed_dev *cdev, struct qed_ptt *p_ptt, u32 min_pf_rate) { … } int __qed_configure_pf_max_bandwidth(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, struct qed_mcp_link_state *p_link, u8 max_bw) { … } /* Main API to configure PF max bandwidth where bw range is [1 - 100] */ int qed_configure_pf_max_bandwidth(struct qed_dev *cdev, u8 max_bw) { … } int __qed_configure_pf_min_bandwidth(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, struct qed_mcp_link_state *p_link, u8 min_bw) { … } /* Main API to configure PF min bandwidth where bw range is [1-100] */ int qed_configure_pf_min_bandwidth(struct qed_dev *cdev, u8 min_bw) { … } void qed_clean_wfq_db(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) { … } int qed_device_num_ports(struct qed_dev *cdev) { … } void qed_set_fw_mac_addr(__le16 *fw_msb, __le16 *fw_mid, __le16 *fw_lsb, u8 *mac) { … } static int qed_llh_shadow_remove_all_filters(struct qed_dev *cdev, u8 ppfid) { … } static void qed_llh_clear_ppfid_filters(struct qed_dev *cdev, u8 ppfid) { … } int qed_llh_add_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port) { … } void qed_llh_remove_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port) { … } int qed_llh_add_dst_tcp_port_filter(struct qed_dev *cdev, u16 dest_port) { … } void qed_llh_remove_dst_tcp_port_filter(struct qed_dev *cdev, u16 dest_port) { … } void qed_llh_clear_all_filters(struct qed_dev *cdev) { … }
Codebase Browser
linux