// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #define pr_fmt(fmt) … #ifdef CONFIG_RFS_ACCEL #include <linux/cpu_rmap.h> #endif /* CONFIG_RFS_ACCEL */ #include <linux/ethtool.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/numa.h> #include <linux/pci.h> #include <linux/utsname.h> #include <linux/version.h> #include <linux/vmalloc.h> #include <net/ip.h> #include "ena_netdev.h" #include "ena_pci_id_tbl.h" #include "ena_xdp.h" MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…); MODULE_LICENSE(…) …; /* Time in jiffies before concluding the transmitter is hung. */ #define TX_TIMEOUT … #define ENA_MAX_RINGS … #define DEFAULT_MSG_ENABLE … static struct ena_aenq_handlers aenq_handlers; static struct workqueue_struct *ena_wq; MODULE_DEVICE_TABLE(pci, ena_pci_tbl); static int ena_rss_init_default(struct ena_adapter *adapter); static void check_for_admin_com_state(struct ena_adapter *adapter); static int ena_destroy_device(struct ena_adapter *adapter, bool graceful); static int ena_restore_device(struct ena_adapter *adapter); static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue) { … } static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu) { … } static int ena_change_mtu(struct net_device *dev, int new_mtu) { … } int ena_xmit_common(struct ena_adapter *adapter, struct ena_ring *ring, struct ena_tx_buffer *tx_info, struct ena_com_tx_ctx *ena_tx_ctx, u16 next_to_use, u32 bytes) { … } static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter) { … } static void ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, u16 qid) { … } void ena_init_io_rings(struct ena_adapter *adapter, int first_index, int count) { … } /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors) * @adapter: network interface device structure * @qid: queue index * * Return 0 on success, negative on failure */ static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid) { … } /* ena_free_tx_resources - Free I/O Tx Resources per Queue * @adapter: network interface device structure * @qid: queue index * * Free all transmit software resources */ static void ena_free_tx_resources(struct ena_adapter *adapter, int qid) { … } int ena_setup_tx_resources_in_range(struct ena_adapter *adapter, int first_index, int count) { … } void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter, int first_index, int count) { … } /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues * @adapter: board private structure * * Free all transmit software resources */ void ena_free_all_io_tx_resources(struct ena_adapter *adapter) { … } /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors) * @adapter: network interface device structure * @qid: queue index * * Returns 0 on success, negative on failure */ static int ena_setup_rx_resources(struct ena_adapter *adapter, u32 qid) { … } /* ena_free_rx_resources - Free I/O Rx Resources * @adapter: network interface device structure * @qid: queue index * * Free all receive software resources */ static void ena_free_rx_resources(struct ena_adapter *adapter, u32 qid) { … } /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues * @adapter: board private structure * * Return 0 on success, negative on failure */ static int ena_setup_all_rx_resources(struct ena_adapter *adapter) { … } /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues * @adapter: board private structure * * Free all receive software resources */ static void ena_free_all_io_rx_resources(struct ena_adapter *adapter) { … } static struct page *ena_alloc_map_page(struct ena_ring *rx_ring, dma_addr_t *dma) { … } static int ena_alloc_rx_buffer(struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) { … } static void ena_unmap_rx_buff_attrs(struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info, unsigned long attrs) { … } static void ena_free_rx_page(struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) { … } static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num) { … } static void ena_free_rx_bufs(struct ena_adapter *adapter, u32 qid) { … } /* ena_refill_all_rx_bufs - allocate all queues Rx buffers * @adapter: board private structure */ static void ena_refill_all_rx_bufs(struct ena_adapter *adapter) { … } static void ena_free_all_rx_bufs(struct ena_adapter *adapter) { … } void ena_unmap_tx_buff(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info) { … } /* ena_free_tx_bufs - Free Tx Buffers per Queue * @tx_ring: TX ring for which buffers be freed */ static void ena_free_tx_bufs(struct ena_ring *tx_ring) { … } static void ena_free_all_tx_bufs(struct ena_adapter *adapter) { … } static void ena_destroy_all_tx_queues(struct ena_adapter *adapter) { … } static void ena_destroy_all_rx_queues(struct ena_adapter *adapter) { … } static void ena_destroy_all_io_queues(struct ena_adapter *adapter) { … } int handle_invalid_req_id(struct ena_ring *ring, u16 req_id, struct ena_tx_buffer *tx_info, bool is_xdp) { … } static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) { … } static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) { … } static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag, u16 len) { … } static bool ena_try_rx_buf_page_reuse(struct ena_rx_buffer *rx_info, u16 buf_len, u16 len, int pkt_offset) { … } static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, u32 descs, u16 *next_to_clean) { … } /* ena_rx_checksum - indicate in skb if hw indicated a good cksum * @adapter: structure containing adapter specific data * @ena_rx_ctx: received packet context/metadata * @skb: skb currently being received and modified */ static void ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, struct sk_buff *skb) { … } static void ena_set_rx_hash(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, struct sk_buff *skb) { … } static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp, u16 num_descs) { … } /* ena_clean_rx_irq - Cleanup RX irq * @rx_ring: RX ring to clean * @napi: napi handler * @budget: how many packets driver is allowed to clean * * Returns the number of cleaned buffers. */ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, u32 budget) { … } static void ena_dim_work(struct work_struct *w) { … } static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi) { … } void ena_unmask_interrupt(struct ena_ring *tx_ring, struct ena_ring *rx_ring) { … } void ena_update_ring_numa_node(struct ena_ring *tx_ring, struct ena_ring *rx_ring) { … } static int ena_io_poll(struct napi_struct *napi, int budget) { … } static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data) { … } /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx * @irq: interrupt number * @data: pointer to a network interface private napi device structure */ static irqreturn_t ena_intr_msix_io(int irq, void *data) { … } /* Reserve a single MSI-X vector for management (admin + aenq). * plus reserve one vector for each potential io queue. * the number of potential io queues is the minimum of what the device * supports and the number of vCPUs. */ static int ena_enable_msix(struct ena_adapter *adapter) { … } static void ena_setup_mgmnt_intr(struct ena_adapter *adapter) { … } static void ena_setup_io_intr(struct ena_adapter *adapter) { … } static int ena_request_mgmnt_irq(struct ena_adapter *adapter) { … } static int ena_request_io_irq(struct ena_adapter *adapter) { … } static void ena_free_mgmnt_irq(struct ena_adapter *adapter) { … } static void ena_free_io_irq(struct ena_adapter *adapter) { … } static void ena_disable_msix(struct ena_adapter *adapter) { … } static void ena_disable_io_intr_sync(struct ena_adapter *adapter) { … } static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index, int count) { … } static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index, int count) { … } static void ena_napi_disable_in_range(struct ena_adapter *adapter, int first_index, int count) { … } static void ena_napi_enable_in_range(struct ena_adapter *adapter, int first_index, int count) { … } /* Configure the Rx forwarding */ static int ena_rss_configure(struct ena_adapter *adapter) { … } static int ena_up_complete(struct ena_adapter *adapter) { … } static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid) { … } int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter, int first_index, int count) { … } static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid) { … } static int ena_create_all_io_rx_queues(struct ena_adapter *adapter) { … } static void set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, int new_rx_size) { … } /* This function allows queue allocation to backoff when the system is * low on memory. If there is not enough memory to allocate io queues * the driver will try to allocate smaller queues. * * The backoff algorithm is as follows: * 1. Try to allocate TX and RX and if successful. * 1.1. return success * * 2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same). * * 3. If TX or RX is smaller than 256 * 3.1. return failure. * 4. else * 4.1. go back to 1. */ static int create_queues_with_size_backoff(struct ena_adapter *adapter) { … } int ena_up(struct ena_adapter *adapter) { … } void ena_down(struct ena_adapter *adapter) { … } /* ena_open - Called when a network interface is made active * @netdev: network interface device structure * * Returns 0 on success, negative value on failure * * The open entry point is called when a network interface is made * active by the system (IFF_UP). At this point all resources needed * for transmit and receive operations are allocated, the interrupt * handler is registered with the OS, the watchdog timer is started, * and the stack is notified that the interface is ready. */ static int ena_open(struct net_device *netdev) { … } /* ena_close - Disables a network interface * @netdev: network interface device structure * * Returns 0, this is not allowed to fail * * The close entry point is called when an interface is de-activated * by the OS. The hardware is still under the drivers control, but * needs to be disabled. A global MAC reset is issued to stop the * hardware, and all transmit and receive resources are freed. */ static int ena_close(struct net_device *netdev) { … } int ena_update_queue_params(struct ena_adapter *adapter, u32 new_tx_size, u32 new_rx_size, u32 new_llq_header_len) { … } int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak) { … } int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count) { … } static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb, bool disable_meta_caching) { … } static int ena_check_and_linearize_skb(struct ena_ring *tx_ring, struct sk_buff *skb) { … } static int ena_tx_map_skb(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info, struct sk_buff *skb, void **push_hdr, u16 *header_len) { … } /* Called with netif_tx_lock. */ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) { … } static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev) { … } static void ena_config_debug_area(struct ena_adapter *adapter) { … } int ena_update_hw_stats(struct ena_adapter *adapter) { … } static void ena_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { … } static const struct net_device_ops ena_netdev_ops = …; static int ena_calc_io_queue_size(struct ena_adapter *adapter, struct ena_com_dev_get_features_ctx *get_feat_ctx) { … } static int ena_device_validate_params(struct ena_adapter *adapter, struct ena_com_dev_get_features_ctx *get_feat_ctx) { … } static void set_default_llq_configurations(struct ena_adapter *adapter, struct ena_llq_configurations *llq_config, struct ena_admin_feature_llq_desc *llq) { … } static int ena_set_queues_placement_policy(struct pci_dev *pdev, struct ena_com_dev *ena_dev, struct ena_admin_feature_llq_desc *llq, struct ena_llq_configurations *llq_default_configurations) { … } static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev, int bars) { … } static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev, struct ena_com_dev_get_features_ctx *get_feat_ctx, bool *wd_state) { … } static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) { … } static int ena_destroy_device(struct ena_adapter *adapter, bool graceful) { … } static int ena_restore_device(struct ena_adapter *adapter) { … } static void ena_fw_reset_device(struct work_struct *work) { … } static int check_for_rx_interrupt_queue(struct ena_adapter *adapter, struct ena_ring *rx_ring) { … } static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, struct ena_ring *tx_ring) { … } static void check_for_missing_completions(struct ena_adapter *adapter) { … } /* trigger napi schedule after 2 consecutive detections */ #define EMPTY_RX_REFILL … /* For the rare case where the device runs out of Rx descriptors and the * napi handler failed to refill new Rx descriptors (due to a lack of memory * for example). * This case will lead to a deadlock: * The device won't send interrupts since all the new Rx packets will be dropped * The napi handler won't allocate new Rx descriptors so the device will be * able to send new packets. * * This scenario can happen when the kernel's vm.min_free_kbytes is too small. * It is recommended to have at least 512MB, with a minimum of 128MB for * constrained environment). * * When such a situation is detected - Reschedule napi */ static void check_for_empty_rx_ring(struct ena_adapter *adapter) { … } /* Check for keep alive expiration */ static void check_for_missing_keep_alive(struct ena_adapter *adapter) { … } static void check_for_admin_com_state(struct ena_adapter *adapter) { … } static void ena_update_hints(struct ena_adapter *adapter, struct ena_admin_ena_hw_hints *hints) { … } static void ena_update_host_info(struct ena_admin_host_info *host_info, struct net_device *netdev) { … } static void ena_timer_service(struct timer_list *t) { … } static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev, struct ena_com_dev *ena_dev, struct ena_com_dev_get_features_ctx *get_feat_ctx) { … } static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat, struct net_device *netdev) { … } static void ena_set_conf_feat_params(struct ena_adapter *adapter, struct ena_com_dev_get_features_ctx *feat) { … } static int ena_rss_init_default(struct ena_adapter *adapter) { … } static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev) { … } /* ena_probe - Device Initialization Routine * @pdev: PCI device information struct * @ent: entry in ena_pci_tbl * * Returns 0 on success, negative on failure * * ena_probe initializes an adapter identified by a pci_dev structure. * The OS initialization, configuring of the adapter private structure, * and a hardware reset occur. */ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { … } /*****************************************************************************/ /* __ena_shutoff - Helper used in both PCI remove/shutdown routines * @pdev: PCI device information struct * @shutdown: Is it a shutdown operation? If false, means it is a removal * * __ena_shutoff is a helper routine that does the real work on shutdown and * removal paths; the difference between those paths is with regards to whether * dettach or unregister the netdevice. */ static void __ena_shutoff(struct pci_dev *pdev, bool shutdown) { … } /* ena_remove - Device Removal Routine * @pdev: PCI device information struct * * ena_remove is called by the PCI subsystem to alert the driver * that it should release a PCI device. */ static void ena_remove(struct pci_dev *pdev) { … } /* ena_shutdown - Device Shutdown Routine * @pdev: PCI device information struct * * ena_shutdown is called by the PCI subsystem to alert the driver that * a shutdown/reboot (or kexec) is happening and device must be disabled. */ static void ena_shutdown(struct pci_dev *pdev) { … } /* ena_suspend - PM suspend callback * @dev_d: Device information struct */ static int __maybe_unused ena_suspend(struct device *dev_d) { … } /* ena_resume - PM resume callback * @dev_d: Device information struct */ static int __maybe_unused ena_resume(struct device *dev_d) { … } static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume); static struct pci_driver ena_pci_driver = …; static int __init ena_init(void) { … } static void __exit ena_cleanup(void) { … } /****************************************************************************** ******************************** AENQ Handlers ******************************* *****************************************************************************/ /* ena_update_on_link_change: * Notify the network interface about the change in link status */ static void ena_update_on_link_change(void *adapter_data, struct ena_admin_aenq_entry *aenq_e) { … } static void ena_keep_alive_wd(void *adapter_data, struct ena_admin_aenq_entry *aenq_e) { … } static void ena_notification(void *adapter_data, struct ena_admin_aenq_entry *aenq_e) { … } /* This handler will called for unknown event group or unimplemented handlers*/ static void unimplemented_aenq_handler(void *data, struct ena_admin_aenq_entry *aenq_e) { … } static struct ena_aenq_handlers aenq_handlers = …; module_init(…) …; module_exit(ena_cleanup);
Codebase Browser
linux