// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018 Intel Corporation */ #include <linux/module.h> #include <linux/types.h> #include <linux/if_vlan.h> #include <linux/tcp.h> #include <linux/udp.h> #include <linux/ip.h> #include <linux/pm_runtime.h> #include <net/pkt_sched.h> #include <linux/bpf_trace.h> #include <net/xdp_sock_drv.h> #include <linux/pci.h> #include <linux/mdio.h> #include <net/ipv6.h> #include "igc.h" #include "igc_hw.h" #include "igc_tsn.h" #include "igc_xdp.h" #define DRV_SUMMARY … #define DEFAULT_MSG_ENABLE … #define IGC_XDP_PASS … #define IGC_XDP_CONSUMED … #define IGC_XDP_TX … #define IGC_XDP_REDIRECT … static int debug = …; MODULE_DESCRIPTION(…); MODULE_LICENSE(…) …; module_param(debug, int, 0); MODULE_PARM_DESC(…) …; char igc_driver_name[] = …; static const char igc_driver_string[] = …; static const char igc_copyright[] = …; static const struct igc_info *igc_info_tbl[] = …; static const struct pci_device_id igc_pci_tbl[] = …; MODULE_DEVICE_TABLE(pci, igc_pci_tbl); enum latency_range { … }; void igc_reset(struct igc_adapter *adapter) { … } /** * igc_power_up_link - Power up the phy link * @adapter: address of board private structure */ static void igc_power_up_link(struct igc_adapter *adapter) { … } /** * igc_release_hw_control - release control of the h/w to f/w * @adapter: address of board private structure * * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means that the * driver is no longer loaded. */ static void igc_release_hw_control(struct igc_adapter *adapter) { … } /** * igc_get_hw_control - get control of the h/w from f/w * @adapter: address of board private structure * * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means that * the driver is loaded. */ static void igc_get_hw_control(struct igc_adapter *adapter) { … } static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) { … } /** * igc_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned */ static void igc_clean_tx_ring(struct igc_ring *tx_ring) { … } /** * igc_free_tx_resources - Free Tx Resources per Queue * @tx_ring: Tx descriptor ring for a specific queue * * Free all transmit software resources */ void igc_free_tx_resources(struct igc_ring *tx_ring) { … } /** * igc_free_all_tx_resources - Free Tx Resources for All Queues * @adapter: board private structure * * Free all transmit software resources */ static void igc_free_all_tx_resources(struct igc_adapter *adapter) { … } /** * igc_clean_all_tx_rings - Free Tx Buffers for all queues * @adapter: board private structure */ static void igc_clean_all_tx_rings(struct igc_adapter *adapter) { … } static void igc_disable_tx_ring_hw(struct igc_ring *ring) { … } /** * igc_disable_all_tx_rings_hw - Disable all transmit queue operation * @adapter: board private structure */ static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter) { … } /** * igc_setup_tx_resources - allocate Tx resources (Descriptors) * @tx_ring: tx descriptor ring (for a specific queue) to setup * * Return 0 on success, negative on failure */ int igc_setup_tx_resources(struct igc_ring *tx_ring) { … } /** * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues * @adapter: board private structure * * Return 0 on success, negative on failure */ static int igc_setup_all_tx_resources(struct igc_adapter *adapter) { … } static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) { … } static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) { … } /** * igc_clean_rx_ring - Free Rx Buffers per Queue * @ring: ring to free buffers from */ static void igc_clean_rx_ring(struct igc_ring *ring) { … } /** * igc_clean_all_rx_rings - Free Rx Buffers for all queues * @adapter: board private structure */ static void igc_clean_all_rx_rings(struct igc_adapter *adapter) { … } /** * igc_free_rx_resources - Free Rx Resources * @rx_ring: ring to clean the resources from * * Free all receive software resources */ void igc_free_rx_resources(struct igc_ring *rx_ring) { … } /** * igc_free_all_rx_resources - Free Rx Resources for All Queues * @adapter: board private structure * * Free all receive software resources */ static void igc_free_all_rx_resources(struct igc_adapter *adapter) { … } /** * igc_setup_rx_resources - allocate Rx resources (Descriptors) * @rx_ring: rx descriptor ring (for a specific queue) to setup * * Returns 0 on success, negative on failure */ int igc_setup_rx_resources(struct igc_ring *rx_ring) { … } /** * igc_setup_all_rx_resources - wrapper to allocate Rx resources * (Descriptors) for all queues * @adapter: board private structure * * Return 0 on success, negative on failure */ static int igc_setup_all_rx_resources(struct igc_adapter *adapter) { … } static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, struct igc_ring *ring) { … } /** * igc_configure_rx_ring - Configure a receive ring after Reset * @adapter: board private structure * @ring: receive ring to be configured * * Configure the Rx unit of the MAC after a reset. */ static void igc_configure_rx_ring(struct igc_adapter *adapter, struct igc_ring *ring) { … } /** * igc_configure_rx - Configure receive Unit after Reset * @adapter: board private structure * * Configure the Rx unit of the MAC after a reset. */ static void igc_configure_rx(struct igc_adapter *adapter) { … } /** * igc_configure_tx_ring - Configure transmit ring after Reset * @adapter: board private structure * @ring: tx ring to configure * * Configure a transmit ring after a reset. */ static void igc_configure_tx_ring(struct igc_adapter *adapter, struct igc_ring *ring) { … } /** * igc_configure_tx - Configure transmit Unit after Reset * @adapter: board private structure * * Configure the Tx unit of the MAC after a reset. */ static void igc_configure_tx(struct igc_adapter *adapter) { … } /** * igc_setup_mrqc - configure the multiple receive queue control registers * @adapter: Board private structure */ static void igc_setup_mrqc(struct igc_adapter *adapter) { … } /** * igc_setup_rctl - configure the receive control registers * @adapter: Board private structure */ static void igc_setup_rctl(struct igc_adapter *adapter) { … } /** * igc_setup_tctl - configure the transmit control registers * @adapter: Board private structure */ static void igc_setup_tctl(struct igc_adapter *adapter) { … } /** * igc_set_mac_filter_hw() - Set MAC address filter in hardware * @adapter: Pointer to adapter where the filter should be set * @index: Filter index * @type: MAC address filter type (source or destination) * @addr: MAC address * @queue: If non-negative, queue assignment feature is enabled and frames * matching the filter are enqueued onto 'queue'. Otherwise, queue * assignment is disabled. */ static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, enum igc_mac_filter_type type, const u8 *addr, int queue) { … } /** * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware * @adapter: Pointer to adapter where the filter should be cleared * @index: Filter index */ static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) { … } /* Set default MAC address for the PF in the first RAR entry */ static void igc_set_default_mac_filter(struct igc_adapter *adapter) { … } /** * igc_set_mac - Change the Ethernet Address of the NIC * @netdev: network interface device structure * @p: pointer to an address structure * * Returns 0 on success, negative on failure */ static int igc_set_mac(struct net_device *netdev, void *p) { … } /** * igc_write_mc_addr_list - write multicast addresses to MTA * @netdev: network interface device structure * * Writes multicast address list to the MTA hash table. * Returns: -ENOMEM on failure * 0 on no addresses written * X on writing X addresses to MTA **/ static int igc_write_mc_addr_list(struct net_device *netdev) { … } static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime, bool *first_flag, bool *insert_empty) { … } static int igc_init_empty_frame(struct igc_ring *ring, struct igc_tx_buffer *buffer, struct sk_buff *skb) { … } static int igc_init_tx_empty_descriptor(struct igc_ring *ring, struct sk_buff *skb, struct igc_tx_buffer *first) { … } #define IGC_EMPTY_FRAME_SIZE … static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, __le32 launch_time, bool first_flag, u32 vlan_macip_lens, u32 type_tucmd, u32 mss_l4len_idx) { … } static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first, __le32 launch_time, bool first_flag) { … } static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) { … } static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) { … } #define IGC_SET_FLAG(_input, _flag, _result) … static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { … } static void igc_tx_olinfo_status(struct igc_ring *tx_ring, union igc_adv_tx_desc *tx_desc, u32 tx_flags, unsigned int paylen) { … } static int igc_tx_map(struct igc_ring *tx_ring, struct igc_tx_buffer *first, const u8 hdr_len) { … } static int igc_tso(struct igc_ring *tx_ring, struct igc_tx_buffer *first, __le32 launch_time, bool first_flag, u8 *hdr_len) { … } static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags) { … } static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, struct igc_ring *tx_ring) { … } static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, struct sk_buff *skb) { … } static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { … } static void igc_rx_checksum(struct igc_ring *ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { … } /* Mapping HW RSS Type to enum pkt_hash_types */ static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = …; static inline void igc_rx_hash(struct igc_ring *ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { … } static void igc_rx_vlan(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { … } /** * igc_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: rx descriptor ring packet is being transacted on * @rx_desc: pointer to the EOP Rx descriptor * @skb: pointer to current skb being populated * * This function checks the ring, descriptor, and packet information in order * to populate the hash, checksum, VLAN, protocol, and other fields within the * skb. */ static void igc_process_skb_fields(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { … } static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) { … } static void igc_restore_vlan(struct igc_adapter *adapter) { … } static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, const unsigned int size, int *rx_buffer_pgcnt) { … } static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer, unsigned int truesize) { … } static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring, unsigned int size) { … } /** * igc_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on * @rx_buffer: buffer containing page to add * @skb: sk_buff to place the data into * @size: size of buffer to be added * * This function will add the data contained in rx_buffer->page to the skb. */ static void igc_add_rx_frag(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, struct sk_buff *skb, unsigned int size) { … } static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, struct xdp_buff *xdp) { … } static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, struct igc_xdp_buff *ctx) { … } /** * igc_reuse_rx_page - page flip buffer and store it back on the ring * @rx_ring: rx descriptor ring to store buffers on * @old_buff: donor buffer to have page reused * * Synchronizes page for reuse by the adapter */ static void igc_reuse_rx_page(struct igc_ring *rx_ring, struct igc_rx_buffer *old_buff) { … } static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer, int rx_buffer_pgcnt) { … } /** * igc_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the * sk_buff in the next buffer to be chained and return true indicating * that this is in fact a non-EOP buffer. */ static bool igc_is_non_eop(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc) { … } /** * igc_cleanup_headers - Correct corrupted or empty headers * @rx_ring: rx descriptor ring packet is being transacted on * @rx_desc: pointer to the EOP Rx descriptor * @skb: pointer to current skb being fixed * * Address the case where we are pulling data in on pages only * and as such no data is present in the skb header. * * In addition if skb is not at least 60 bytes we need to pad it so that * it is large enough to qualify as a valid Ethernet frame. * * Returns true if an error was encountered and skb was freed. */ static bool igc_cleanup_headers(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { … } static void igc_put_rx_buffer(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, int rx_buffer_pgcnt) { … } static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) { … } static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, struct igc_rx_buffer *bi) { … } /** * igc_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring * @cleaned_count: number of buffers to clean */ static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) { … } static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) { … } /* This function requires __netif_tx_lock is held by the caller. */ static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, struct xdp_frame *xdpf) { … } static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter, int cpu) { … } static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) { … } /* This function assumes rcu_read_lock() is held by the caller. */ static int __igc_xdp_run_prog(struct igc_adapter *adapter, struct bpf_prog *prog, struct xdp_buff *xdp) { … } static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter, struct xdp_buff *xdp) { … } /* This function assumes __netif_tx_lock is held by the caller. */ static void igc_flush_tx_descriptors(struct igc_ring *ring) { … } static void igc_finalize_xdp(struct igc_adapter *adapter, int status) { … } static void igc_update_rx_stats(struct igc_q_vector *q_vector, unsigned int packets, unsigned int bytes) { … } static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) { … } static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, struct xdp_buff *xdp) { … } static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, union igc_adv_rx_desc *desc, struct xdp_buff *xdp, ktime_t timestamp) { … } static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp) { … } static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) { … } static void igc_update_tx_stats(struct igc_q_vector *q_vector, unsigned int packets, unsigned int bytes) { … } static void igc_xsk_request_timestamp(void *_priv) { … } static u64 igc_xsk_fill_timestamp(void *_priv) { … } const struct xsk_tx_metadata_ops igc_xsk_tx_metadata_ops = …; static void igc_xdp_xmit_zc(struct igc_ring *ring) { … } /** * igc_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: pointer to q_vector containing needed info * @napi_budget: Used to determine if we are in netpoll * * returns true if ring is completely cleaned */ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) { … } static int igc_find_mac_filter(struct igc_adapter *adapter, enum igc_mac_filter_type type, const u8 *addr) { … } static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) { … } /** * igc_add_mac_filter() - Add MAC address filter * @adapter: Pointer to adapter where the filter should be added * @type: MAC address filter type (source or destination) * @addr: MAC address * @queue: If non-negative, queue assignment feature is enabled and frames * matching the filter are enqueued onto 'queue'. Otherwise, queue * assignment is disabled. * * Return: 0 in case of success, negative errno code otherwise. */ static int igc_add_mac_filter(struct igc_adapter *adapter, enum igc_mac_filter_type type, const u8 *addr, int queue) { … } /** * igc_del_mac_filter() - Delete MAC address filter * @adapter: Pointer to adapter where the filter should be deleted from * @type: MAC address filter type (source or destination) * @addr: MAC address */ static void igc_del_mac_filter(struct igc_adapter *adapter, enum igc_mac_filter_type type, const u8 *addr) { … } /** * igc_add_vlan_prio_filter() - Add VLAN priority filter * @adapter: Pointer to adapter where the filter should be added * @prio: VLAN priority value * @queue: Queue number which matching frames are assigned to * * Return: 0 in case of success, negative errno code otherwise. */ static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, int queue) { … } /** * igc_del_vlan_prio_filter() - Delete VLAN priority filter * @adapter: Pointer to adapter where the filter should be deleted from * @prio: VLAN priority value */ static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) { … } static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) { … } /** * igc_add_etype_filter() - Add ethertype filter * @adapter: Pointer to adapter where the filter should be added * @etype: Ethertype value * @queue: If non-negative, queue assignment feature is enabled and frames * matching the filter are enqueued onto 'queue'. Otherwise, queue * assignment is disabled. * * Return: 0 in case of success, negative errno code otherwise. */ static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, int queue) { … } static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) { … } /** * igc_del_etype_filter() - Delete ethertype filter * @adapter: Pointer to adapter where the filter should be deleted from * @etype: Ethertype value */ static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) { … } static int igc_flex_filter_select(struct igc_adapter *adapter, struct igc_flex_filter *input, u32 *fhft) { … } static int igc_write_flex_filter_ll(struct igc_adapter *adapter, struct igc_flex_filter *input) { … } static void igc_flex_filter_add_field(struct igc_flex_filter *flex, const void *src, unsigned int offset, size_t len, const void *mask) { … } static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter) { … } static bool igc_flex_filter_in_use(struct igc_adapter *adapter) { … } static int igc_add_flex_filter(struct igc_adapter *adapter, struct igc_nfc_rule *rule) { … } static void igc_del_flex_filter(struct igc_adapter *adapter, u16 reg_index) { … } static int igc_enable_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) { … } static void igc_disable_nfc_rule(struct igc_adapter *adapter, const struct igc_nfc_rule *rule) { … } /** * igc_get_nfc_rule() - Get NFC rule * @adapter: Pointer to adapter * @location: Rule location * * Context: Expects adapter->nfc_rule_lock to be held by caller. * * Return: Pointer to NFC rule at @location. If not found, NULL. */ struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, u32 location) { … } /** * igc_del_nfc_rule() - Delete NFC rule * @adapter: Pointer to adapter * @rule: Pointer to rule to be deleted * * Disable NFC rule in hardware and delete it from adapter. * * Context: Expects adapter->nfc_rule_lock to be held by caller. */ void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) { … } static void igc_flush_nfc_rules(struct igc_adapter *adapter) { … } /** * igc_add_nfc_rule() - Add NFC rule * @adapter: Pointer to adapter * @rule: Pointer to rule to be added * * Enable NFC rule in hardware and add it to adapter. * * Context: Expects adapter->nfc_rule_lock to be held by caller. * * Return: 0 on success, negative errno on failure. */ int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) { … } static void igc_restore_nfc_rules(struct igc_adapter *adapter) { … } static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) { … } static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) { … } /** * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set * @netdev: network interface device structure * * The set_rx_mode entry point is called whenever the unicast or multicast * address lists or the network interface flags are updated. This routine is * responsible for configuring the hardware for proper unicast, multicast, * promiscuous mode, and all-multi behavior. */ static void igc_set_rx_mode(struct net_device *netdev) { … } /** * igc_configure - configure the hardware for RX and TX * @adapter: private board structure */ static void igc_configure(struct igc_adapter *adapter) { … } /** * igc_write_ivar - configure ivar for given MSI-X vector * @hw: pointer to the HW structure * @msix_vector: vector number we are allocating to a given ring * @index: row index of IVAR register to write within IVAR table * @offset: column offset of in IVAR, should be multiple of 8 * * The IVAR table consists of 2 columns, * each containing an cause allocation for an Rx and Tx ring, and a * variable number of rows depending on the number of queues supported. */ static void igc_write_ivar(struct igc_hw *hw, int msix_vector, int index, int offset) { … } static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) { … } /** * igc_configure_msix - Configure MSI-X hardware * @adapter: Pointer to adapter structure * * igc_configure_msix sets up the hardware to properly * generate MSI-X interrupts. */ static void igc_configure_msix(struct igc_adapter *adapter) { … } /** * igc_irq_enable - Enable default interrupt generation settings * @adapter: board private structure */ static void igc_irq_enable(struct igc_adapter *adapter) { … } /** * igc_irq_disable - Mask off interrupt generation on the NIC * @adapter: board private structure */ static void igc_irq_disable(struct igc_adapter *adapter) { … } void igc_set_flag_queue_pairs(struct igc_adapter *adapter, const u32 max_rss_queues) { … } unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) { … } static void igc_init_queue_configuration(struct igc_adapter *adapter) { … } /** * igc_reset_q_vector - Reset config for interrupt vector * @adapter: board private structure to initialize * @v_idx: Index of vector to be reset * * If NAPI is enabled it will delete any references to the * NAPI struct. This is preparation for igc_free_q_vector. */ static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) { … } /** * igc_free_q_vector - Free memory allocated for specific interrupt vector * @adapter: board private structure to initialize * @v_idx: Index of vector to be freed * * This function frees the memory allocated to the q_vector. */ static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) { … } /** * igc_free_q_vectors - Free memory allocated for interrupt vectors * @adapter: board private structure to initialize * * This function frees the memory allocated to the q_vectors. In addition if * NAPI is enabled it will delete any references to the NAPI struct prior * to freeing the q_vector. */ static void igc_free_q_vectors(struct igc_adapter *adapter) { … } /** * igc_update_itr - update the dynamic ITR value based on statistics * @q_vector: pointer to q_vector * @ring_container: ring info to update the itr for * * Stores a new ITR value based on packets and byte * counts during the last interrupt. The advantage of per interrupt * computation is faster updates and more accurate ITR for the current * traffic pattern. Constants in this function were computed * based on theoretical maximum wire speed and thresholds were set based * on testing data as well as attempting to minimize response time * while increasing bulk throughput. * NOTE: These calculations are only valid when operating in a single- * queue environment. */ static void igc_update_itr(struct igc_q_vector *q_vector, struct igc_ring_container *ring_container) { … } static void igc_set_itr(struct igc_q_vector *q_vector) { … } static void igc_reset_interrupt_capability(struct igc_adapter *adapter) { … } /** * igc_set_interrupt_capability - set MSI or MSI-X if supported * @adapter: Pointer to adapter structure * @msix: boolean value for MSI-X capability * * Attempt to configure interrupts using the best available * capabilities of the hardware and kernel. */ static void igc_set_interrupt_capability(struct igc_adapter *adapter, bool msix) { … } /** * igc_update_ring_itr - update the dynamic ITR value based on packet size * @q_vector: pointer to q_vector * * Stores a new ITR value based on strictly on packet size. This * algorithm is less sophisticated than that used in igc_update_itr, * due to the difficulty of synchronizing statistics across multiple * receive rings. The divisors and thresholds used by this function * were determined based on theoretical maximum wire speed and testing * data, in order to minimize response time while increasing bulk * throughput. * NOTE: This function is called only when operating in a multiqueue * receive environment. */ static void igc_update_ring_itr(struct igc_q_vector *q_vector) { … } static void igc_ring_irq_enable(struct igc_q_vector *q_vector) { … } static void igc_add_ring(struct igc_ring *ring, struct igc_ring_container *head) { … } /** * igc_cache_ring_register - Descriptor ring to register mapping * @adapter: board private structure to initialize * * Once we know the feature-set enabled for the device, we'll cache * the register offset the descriptor ring is assigned to. */ static void igc_cache_ring_register(struct igc_adapter *adapter) { … } /** * igc_poll - NAPI Rx polling callback * @napi: napi polling structure * @budget: count of how many packets we should handle */ static int igc_poll(struct napi_struct *napi, int budget) { … } /** * igc_alloc_q_vector - Allocate memory for a single interrupt vector * @adapter: board private structure to initialize * @v_count: q_vectors allocated on adapter, used for ring interleaving * @v_idx: index of vector in adapter struct * @txr_count: total number of Tx rings to allocate * @txr_idx: index of first Tx ring to allocate * @rxr_count: total number of Rx rings to allocate * @rxr_idx: index of first Rx ring to allocate * * We allocate one q_vector. If allocation fails we return -ENOMEM. */ static int igc_alloc_q_vector(struct igc_adapter *adapter, unsigned int v_count, unsigned int v_idx, unsigned int txr_count, unsigned int txr_idx, unsigned int rxr_count, unsigned int rxr_idx) { … } /** * igc_alloc_q_vectors - Allocate memory for interrupt vectors * @adapter: board private structure to initialize * * We allocate one q_vector per queue interrupt. If allocation fails we * return -ENOMEM. */ static int igc_alloc_q_vectors(struct igc_adapter *adapter) { … } /** * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors * @adapter: Pointer to adapter structure * @msix: boolean for MSI-X capability * * This function initializes the interrupts and allocates all of the queues. */ static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) { … } /** * igc_sw_init - Initialize general software structures (struct igc_adapter) * @adapter: board private structure to initialize * * igc_sw_init initializes the Adapter private data structure. * Fields are initialized based on PCI device information and * OS network device settings (MTU size). */ static int igc_sw_init(struct igc_adapter *adapter) { … } /** * igc_up - Open the interface and prepare it to handle traffic * @adapter: board private structure */ void igc_up(struct igc_adapter *adapter) { … } /** * igc_update_stats - Update the board statistics counters * @adapter: board private structure */ void igc_update_stats(struct igc_adapter *adapter) { … } /** * igc_down - Close the interface * @adapter: board private structure */ void igc_down(struct igc_adapter *adapter) { … } void igc_reinit_locked(struct igc_adapter *adapter) { … } static void igc_reset_task(struct work_struct *work) { … } /** * igc_change_mtu - Change the Maximum Transfer Unit * @netdev: network interface device structure * @new_mtu: new value for maximum frame size * * Returns 0 on success, negative on failure */ static int igc_change_mtu(struct net_device *netdev, int new_mtu) { … } /** * igc_tx_timeout - Respond to a Tx Hang * @netdev: network interface device structure * @txqueue: queue number that timed out **/ static void igc_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) { … } /** * igc_get_stats64 - Get System Network Statistics * @netdev: network interface device structure * @stats: rtnl_link_stats64 pointer * * Returns the address of the device statistics structure. * The statistics are updated here and also from the timer callback. */ static void igc_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { … } static netdev_features_t igc_fix_features(struct net_device *netdev, netdev_features_t features) { … } static int igc_set_features(struct net_device *netdev, netdev_features_t features) { … } static netdev_features_t igc_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { … } static void igc_tsync_interrupt(struct igc_adapter *adapter) { … } /** * igc_msix_other - msix other interrupt handler * @irq: interrupt number * @data: pointer to a q_vector */ static irqreturn_t igc_msix_other(int irq, void *data) { … } static void igc_write_itr(struct igc_q_vector *q_vector) { … } static irqreturn_t igc_msix_ring(int irq, void *data) { … } /** * igc_request_msix - Initialize MSI-X interrupts * @adapter: Pointer to adapter structure * * igc_request_msix allocates MSI-X vectors and requests interrupts from the * kernel. */ static int igc_request_msix(struct igc_adapter *adapter) { … } /** * igc_clear_interrupt_scheme - reset the device to a state of no interrupts * @adapter: Pointer to adapter structure * * This function resets the device so that it has 0 rx queues, tx queues, and * MSI-X interrupts allocated. */ static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) { … } /* Need to wait a few seconds after link up to get diagnostic information from * the phy */ static void igc_update_phy_info(struct timer_list *t) { … } /** * igc_has_link - check shared code for link and determine up/down * @adapter: pointer to driver private info */ bool igc_has_link(struct igc_adapter *adapter) { … } /** * igc_watchdog - Timer Call-back * @t: timer for the watchdog */ static void igc_watchdog(struct timer_list *t) { … } static void igc_watchdog_task(struct work_struct *work) { … } /** * igc_intr_msi - Interrupt Handler * @irq: interrupt number * @data: pointer to a network interface device structure */ static irqreturn_t igc_intr_msi(int irq, void *data) { … } /** * igc_intr - Legacy Interrupt Handler * @irq: interrupt number * @data: pointer to a network interface device structure */ static irqreturn_t igc_intr(int irq, void *data) { … } static void igc_free_irq(struct igc_adapter *adapter) { … } /** * igc_request_irq - initialize interrupts * @adapter: Pointer to adapter structure * * Attempts to configure interrupts using the best available * capabilities of the hardware and kernel. */ static int igc_request_irq(struct igc_adapter *adapter) { … } /** * __igc_open - Called when a network interface is made active * @netdev: network interface device structure * @resuming: boolean indicating if the device is resuming * * Returns 0 on success, negative value on failure * * The open entry point is called when a network interface is made * active by the system (IFF_UP). At this point all resources needed * for transmit and receive operations are allocated, the interrupt * handler is registered with the OS, the watchdog timer is started, * and the stack is notified that the interface is ready. */ static int __igc_open(struct net_device *netdev, bool resuming) { … } int igc_open(struct net_device *netdev) { … } /** * __igc_close - Disables a network interface * @netdev: network interface device structure * @suspending: boolean indicating the device is suspending * * Returns 0, this is not allowed to fail * * The close entry point is called when an interface is de-activated * by the OS. The hardware is still under the driver's control, but * needs to be disabled. A global MAC reset is issued to stop the * hardware, and all transmit and receive resources are freed. */ static int __igc_close(struct net_device *netdev, bool suspending) { … } int igc_close(struct net_device *netdev) { … } /** * igc_ioctl - Access the hwtstamp interface * @netdev: network interface device structure * @ifr: interface request data * @cmd: ioctl command **/ static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) { … } static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, bool enable) { … } static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) { … } static bool validate_schedule(struct igc_adapter *adapter, const struct tc_taprio_qopt_offload *qopt) { … } static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, struct tc_etf_qopt_offload *qopt) { … } static int igc_qbv_clear_schedule(struct igc_adapter *adapter) { … } static int igc_tsn_clear_schedule(struct igc_adapter *adapter) { … } static void igc_taprio_stats(struct net_device *dev, struct tc_taprio_qopt_stats *stats) { … } static void igc_taprio_queue_stats(struct net_device *dev, struct tc_taprio_qopt_queue_stats *queue_stats) { … } static int igc_save_qbv_schedule(struct igc_adapter *adapter, struct tc_taprio_qopt_offload *qopt) { … } static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, struct tc_taprio_qopt_offload *qopt) { … } static int igc_save_cbs_params(struct igc_adapter *adapter, int queue, bool enable, int idleslope, int sendslope, int hicredit, int locredit) { … } static int igc_tsn_enable_cbs(struct igc_adapter *adapter, struct tc_cbs_qopt_offload *qopt) { … } static int igc_tc_query_caps(struct igc_adapter *adapter, struct tc_query_caps_base *base) { … } static void igc_save_mqprio_params(struct igc_adapter *adapter, u8 num_tc, u16 *offset) { … } static int igc_tsn_enable_mqprio(struct igc_adapter *adapter, struct tc_mqprio_qopt_offload *mqprio) { … } static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { … } static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) { … } static int igc_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags) { … } static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter, struct igc_q_vector *q_vector) { … } int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) { … } static ktime_t igc_get_tstamp(struct net_device *dev, const struct skb_shared_hwtstamps *hwtstamps, bool cycles) { … } static const struct net_device_ops igc_netdev_ops = …; /* PCIe configuration access */ void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) { … } void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) { … } s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) { … } s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) { … } u32 igc_rd32(struct igc_hw *hw, u32 reg) { … } /* Mapping HW RSS Type to enum xdp_rss_hash_type */ static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = …; static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, enum xdp_rss_hash_type *rss_type) { … } static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) { … } static const struct xdp_metadata_ops igc_xdp_metadata_ops = …; static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer) { … } /** * igc_probe - Device Initialization Routine * @pdev: PCI device information struct * @ent: entry in igc_pci_tbl * * Returns 0 on success, negative on failure * * igc_probe initializes an adapter identified by a pci_dev structure. * The OS initialization, configuring the adapter private structure, * and a hardware reset occur. */ static int igc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { … } /** * igc_remove - Device Removal Routine * @pdev: PCI device information struct * * igc_remove is called by the PCI subsystem to alert the driver * that it should release a PCI device. This could be caused by a * Hot-Plug event, or because the driver is going to be removed from * memory. */ static void igc_remove(struct pci_dev *pdev) { … } static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, bool runtime) { … } static int igc_runtime_suspend(struct device *dev) { … } static void igc_deliver_wake_packet(struct net_device *netdev) { … } static int igc_resume(struct device *dev) { … } static int igc_runtime_resume(struct device *dev) { … } static int igc_suspend(struct device *dev) { … } static int __maybe_unused igc_runtime_idle(struct device *dev) { … } static void igc_shutdown(struct pci_dev *pdev) { … } /** * igc_io_error_detected - called when PCI error is detected * @pdev: Pointer to PCI device * @state: The current PCI connection state * * This function is called after a PCI bus error affecting * this device has been detected. **/ static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { … } /** * igc_io_slot_reset - called after the PCI bus has been reset. * @pdev: Pointer to PCI device * * Restart the card from scratch, as if from a cold-boot. Implementation * resembles the first-half of the igc_resume routine. **/ static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) { … } /** * igc_io_resume - called when traffic can start to flow again. * @pdev: Pointer to PCI device * * This callback is called when the error recovery driver tells us that * its OK to resume normal operation. Implementation resembles the * second-half of the igc_resume routine. */ static void igc_io_resume(struct pci_dev *pdev) { … } static const struct pci_error_handlers igc_err_handler = …; static _DEFINE_DEV_PM_OPS(igc_pm_ops, igc_suspend, igc_resume, igc_runtime_suspend, igc_runtime_resume, igc_runtime_idle); static struct pci_driver igc_driver = …; /** * igc_reinit_queues - return error * @adapter: pointer to adapter structure */ int igc_reinit_queues(struct igc_adapter *adapter) { … } /** * igc_get_hw_dev - return device * @hw: pointer to hardware structure * * used by hardware layer to print debugging information */ struct net_device *igc_get_hw_dev(struct igc_hw *hw) { … } static void igc_disable_rx_ring_hw(struct igc_ring *ring) { … } void igc_disable_rx_ring(struct igc_ring *ring) { … } void igc_enable_rx_ring(struct igc_ring *ring) { … } void igc_disable_tx_ring(struct igc_ring *ring) { … } void igc_enable_tx_ring(struct igc_ring *ring) { … } /** * igc_init_module - Driver Registration Routine * * igc_init_module is the first routine called when the driver is * loaded. All it does is register with the PCI subsystem. */ static int __init igc_init_module(void) { … } module_init(…) …; /** * igc_exit_module - Driver Exit Cleanup Routine * * igc_exit_module is called just before the driver is removed * from memory. */ static void __exit igc_exit_module(void) { … } module_exit(igc_exit_module); /* igc_main.c */
Codebase Browser
linux