// SPDX-License-Identifier: GPL-2.0-only /**************************************************************************** * Driver for Solarflare network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. * Copyright 2005-2013 Solarflare Communications Inc. */ #include <linux/module.h> #include <linux/pci.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/delay.h> #include <linux/notifier.h> #include <linux/ip.h> #include <linux/tcp.h> #include <linux/in.h> #include <linux/ethtool.h> #include <linux/topology.h> #include <linux/gfp.h> #include <linux/interrupt.h> #include "net_driver.h" #include "efx.h" #include "nic.h" #include "selftest.h" #include "workarounds.h" /************************************************************************** * * Type name strings * ************************************************************************** */ /* Loopback mode names (see LOOPBACK_MODE()) */ const unsigned int ef4_loopback_mode_max = …; const char *const ef4_loopback_mode_names[] = …; const unsigned int ef4_reset_type_max = …; const char *const ef4_reset_type_names[] = …; /* Reset workqueue. If any NIC has a hardware failure then a reset will be * queued onto this work queue. This is not a per-nic work queue, because * ef4_reset_work() acquires the rtnl lock, so resets are naturally serialised. */ static struct workqueue_struct *reset_workqueue; /* How often and how many times to poll for a reset while waiting for a * BIST that another function started to complete. */ #define BIST_WAIT_DELAY_MS … #define BIST_WAIT_DELAY_COUNT … /************************************************************************** * * Configurable values * *************************************************************************/ /* * Use separate channels for TX and RX events * * Set this to 1 to use separate channels for TX and RX. It allows us * to control interrupt affinity separately for TX and RX. * * This is only used in MSI-X interrupt mode */ bool ef4_separate_tx_channels; module_param(ef4_separate_tx_channels, bool, 0444); MODULE_PARM_DESC(…) …; /* This is the time (in jiffies) between invocations of the hardware * monitor. * On Falcon-based NICs, this will: * - Check the on-board hardware monitor; * - Poll the link state and reconfigure the hardware as necessary. * On Siena-based NICs for power systems with EEH support, this will give EEH a * chance to start. */ static unsigned int ef4_monitor_interval = …; /* Initial interrupt moderation settings. They can be modified after * module load with ethtool. * * The default for RX should strike a balance between increasing the * round-trip latency and reducing overhead. */ static unsigned int rx_irq_mod_usec = …; /* Initial interrupt moderation settings. They can be modified after * module load with ethtool. * * This default is chosen to ensure that a 10G link does not go idle * while a TX queue is stopped after it has become full. A queue is * restarted when it drops below half full. The time this takes (assuming * worst case 3 descriptors per packet and 1024 descriptors) is * 512 / 3 * 1.2 = 205 usec. */ static unsigned int tx_irq_mod_usec = …; /* This is the first interrupt mode to try out of: * 0 => MSI-X * 1 => MSI * 2 => legacy */ static unsigned int interrupt_mode; /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), * i.e. the number of CPUs among which we may distribute simultaneous * interrupt handling. * * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. * The default (0) means to assign an interrupt to each core. */ static unsigned int rss_cpus; module_param(rss_cpus, uint, 0444); MODULE_PARM_DESC(…) …; static bool phy_flash_cfg; module_param(phy_flash_cfg, bool, 0644); MODULE_PARM_DESC(…) …; static unsigned irq_adapt_low_thresh = …; module_param(irq_adapt_low_thresh, uint, 0644); MODULE_PARM_DESC(…) …; static unsigned irq_adapt_high_thresh = …; module_param(irq_adapt_high_thresh, uint, 0644); MODULE_PARM_DESC(…) …; static unsigned debug = …; module_param(debug, uint, 0); MODULE_PARM_DESC(…) …; /************************************************************************** * * Utility functions and prototypes * *************************************************************************/ static int ef4_soft_enable_interrupts(struct ef4_nic *efx); static void ef4_soft_disable_interrupts(struct ef4_nic *efx); static void ef4_remove_channel(struct ef4_channel *channel); static void ef4_remove_channels(struct ef4_nic *efx); static const struct ef4_channel_type ef4_default_channel_type; static void ef4_remove_port(struct ef4_nic *efx); static void ef4_init_napi_channel(struct ef4_channel *channel); static void ef4_fini_napi(struct ef4_nic *efx); static void ef4_fini_napi_channel(struct ef4_channel *channel); static void ef4_fini_struct(struct ef4_nic *efx); static void ef4_start_all(struct ef4_nic *efx); static void ef4_stop_all(struct ef4_nic *efx); #define EF4_ASSERT_RESET_SERIALISED(efx) … static int ef4_check_disabled(struct ef4_nic *efx) { … } /************************************************************************** * * Event queue processing * *************************************************************************/ /* Process channel's event queue * * This function is responsible for processing the event queue of a * single channel. The caller must guarantee that this function will * never be concurrently called more than once on the same channel, * though different channels may be being processed concurrently. */ static int ef4_process_channel(struct ef4_channel *channel, int budget) { … } /* NAPI poll handler * * NAPI guarantees serialisation of polls of the same device, which * provides the guarantee required by ef4_process_channel(). */ static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel) { … } static int ef4_poll(struct napi_struct *napi, int budget) { … } /* Create event queue * Event queue memory allocations are done only once. If the channel * is reset, the memory buffer will be reused; this guards against * errors during channel reset and also simplifies interrupt handling. */ static int ef4_probe_eventq(struct ef4_channel *channel) { … } /* Prepare channel's event queue */ static int ef4_init_eventq(struct ef4_channel *channel) { … } /* Enable event queue processing and NAPI */ void ef4_start_eventq(struct ef4_channel *channel) { … } /* Disable event queue processing and NAPI */ void ef4_stop_eventq(struct ef4_channel *channel) { … } static void ef4_fini_eventq(struct ef4_channel *channel) { … } static void ef4_remove_eventq(struct ef4_channel *channel) { … } /************************************************************************** * * Channel handling * *************************************************************************/ /* Allocate and initialise a channel structure. */ static struct ef4_channel * ef4_alloc_channel(struct ef4_nic *efx, int i, struct ef4_channel *old_channel) { … } /* Allocate and initialise a channel structure, copying parameters * (but not resources) from an old channel structure. */ static struct ef4_channel * ef4_copy_channel(const struct ef4_channel *old_channel) { … } static int ef4_probe_channel(struct ef4_channel *channel) { … } static void ef4_get_channel_name(struct ef4_channel *channel, char *buf, size_t len) { … } static void ef4_set_channel_names(struct ef4_nic *efx) { … } static int ef4_probe_channels(struct ef4_nic *efx) { … } /* Channels are shutdown and reinitialised whilst the NIC is running * to propagate configuration changes (mtu, checksum offload), or * to clear hardware error conditions */ static void ef4_start_datapath(struct ef4_nic *efx) { … } static void ef4_stop_datapath(struct ef4_nic *efx) { … } static void ef4_remove_channel(struct ef4_channel *channel) { … } static void ef4_remove_channels(struct ef4_nic *efx) { … } int ef4_realloc_channels(struct ef4_nic *efx, u32 rxq_entries, u32 txq_entries) { … } void ef4_schedule_slow_fill(struct ef4_rx_queue *rx_queue) { … } static const struct ef4_channel_type ef4_default_channel_type = …; int ef4_channel_dummy_op_int(struct ef4_channel *channel) { … } void ef4_channel_dummy_op_void(struct ef4_channel *channel) { … } /************************************************************************** * * Port handling * **************************************************************************/ /* This ensures that the kernel is kept informed (via * netif_carrier_on/off) of the link status, and also maintains the * link status's stop on the port's TX queue. */ void ef4_link_status_changed(struct ef4_nic *efx) { … } void ef4_link_set_advertising(struct ef4_nic *efx, u32 advertising) { … } void ef4_link_set_wanted_fc(struct ef4_nic *efx, u8 wanted_fc) { … } static void ef4_fini_port(struct ef4_nic *efx); /* We assume that efx->type->reconfigure_mac will always try to sync RX * filters and therefore needs to read-lock the filter table against freeing */ void ef4_mac_reconfigure(struct ef4_nic *efx) { … } /* Push loopback/power/transmit disable settings to the PHY, and reconfigure * the MAC appropriately. All other PHY configuration changes are pushed * through phy_op->set_link_ksettings(), and pushed asynchronously to the MAC * through ef4_monitor(). * * Callers must hold the mac_lock */ int __ef4_reconfigure_port(struct ef4_nic *efx) { … } /* Reinitialise the MAC to pick up new PHY settings, even if the port is * disabled. */ int ef4_reconfigure_port(struct ef4_nic *efx) { … } /* Asynchronous work item for changing MAC promiscuity and multicast * hash. Avoid a drain/rx_ingress enable by reconfiguring the current * MAC directly. */ static void ef4_mac_work(struct work_struct *data) { … } static int ef4_probe_port(struct ef4_nic *efx) { … } static int ef4_init_port(struct ef4_nic *efx) { … } static void ef4_start_port(struct ef4_nic *efx) { … } /* Cancel work for MAC reconfiguration, periodic hardware monitoring * and the async self-test, wait for them to finish and prevent them * being scheduled again. This doesn't cover online resets, which * should only be cancelled when removing the device. */ static void ef4_stop_port(struct ef4_nic *efx) { … } static void ef4_fini_port(struct ef4_nic *efx) { … } static void ef4_remove_port(struct ef4_nic *efx) { … } /************************************************************************** * * NIC handling * **************************************************************************/ static LIST_HEAD(ef4_primary_list); static LIST_HEAD(ef4_unassociated_list); static bool ef4_same_controller(struct ef4_nic *left, struct ef4_nic *right) { … } static void ef4_associate(struct ef4_nic *efx) { … } static void ef4_dissociate(struct ef4_nic *efx) { … } /* This configures the PCI device to enable I/O and DMA. */ static int ef4_init_io(struct ef4_nic *efx) { … } static void ef4_fini_io(struct ef4_nic *efx) { … } void ef4_set_default_rx_indir_table(struct ef4_nic *efx) { … } static unsigned int ef4_wanted_parallelism(struct ef4_nic *efx) { … } /* Probe the number and type of interrupts we are able to obtain, and * the resulting numbers of channels and RX queues. */ static int ef4_probe_interrupts(struct ef4_nic *efx) { … } static int ef4_soft_enable_interrupts(struct ef4_nic *efx) { … } static void ef4_soft_disable_interrupts(struct ef4_nic *efx) { … } static int ef4_enable_interrupts(struct ef4_nic *efx) { … } static void ef4_disable_interrupts(struct ef4_nic *efx) { … } static void ef4_remove_interrupts(struct ef4_nic *efx) { … } static void ef4_set_channels(struct ef4_nic *efx) { … } static int ef4_probe_nic(struct ef4_nic *efx) { … } static void ef4_remove_nic(struct ef4_nic *efx) { … } static int ef4_probe_filters(struct ef4_nic *efx) { … } static void ef4_remove_filters(struct ef4_nic *efx) { … } static void ef4_restore_filters(struct ef4_nic *efx) { … } /************************************************************************** * * NIC startup/shutdown * *************************************************************************/ static int ef4_probe_all(struct ef4_nic *efx) { … } /* If the interface is supposed to be running but is not, start * the hardware and software data path, regular activity for the port * (MAC statistics, link polling, etc.) and schedule the port to be * reconfigured. Interrupts must already be enabled. This function * is safe to call multiple times, so long as the NIC is not disabled. * Requires the RTNL lock. */ static void ef4_start_all(struct ef4_nic *efx) { … } /* Quiesce the hardware and software data path, and regular activity * for the port without bringing the link down. Safe to call multiple * times with the NIC in almost any state, but interrupts should be * enabled. Requires the RTNL lock. */ static void ef4_stop_all(struct ef4_nic *efx) { … } static void ef4_remove_all(struct ef4_nic *efx) { … } /************************************************************************** * * Interrupt moderation * **************************************************************************/ unsigned int ef4_usecs_to_ticks(struct ef4_nic *efx, unsigned int usecs) { … } unsigned int ef4_ticks_to_usecs(struct ef4_nic *efx, unsigned int ticks) { … } /* Set interrupt moderation parameters */ int ef4_init_irq_moderation(struct ef4_nic *efx, unsigned int tx_usecs, unsigned int rx_usecs, bool rx_adaptive, bool rx_may_override_tx) { … } void ef4_get_irq_moderation(struct ef4_nic *efx, unsigned int *tx_usecs, unsigned int *rx_usecs, bool *rx_adaptive) { … } /************************************************************************** * * Hardware monitor * **************************************************************************/ /* Run periodically off the general workqueue */ static void ef4_monitor(struct work_struct *data) { … } /************************************************************************** * * ioctls * *************************************************************************/ /* Net device ioctl * Context: process, rtnl_lock() held. */ static int ef4_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) { … } /************************************************************************** * * NAPI interface * **************************************************************************/ static void ef4_init_napi_channel(struct ef4_channel *channel) { … } static void ef4_init_napi(struct ef4_nic *efx) { … } static void ef4_fini_napi_channel(struct ef4_channel *channel) { … } static void ef4_fini_napi(struct ef4_nic *efx) { … } /************************************************************************** * * Kernel net device interface * *************************************************************************/ /* Context: process, rtnl_lock() held. */ int ef4_net_open(struct net_device *net_dev) { … } /* Context: process, rtnl_lock() held. * Note that the kernel will ignore our return code; this method * should really be a void. */ int ef4_net_stop(struct net_device *net_dev) { … } /* Context: process, rcu_read_lock or RTNL held, non-blocking. */ static void ef4_net_stats(struct net_device *net_dev, struct rtnl_link_stats64 *stats) { … } /* Context: netif_tx_lock held, BHs disabled. */ static void ef4_watchdog(struct net_device *net_dev, unsigned int txqueue) { … } /* Context: process, rtnl_lock() held. */ static int ef4_change_mtu(struct net_device *net_dev, int new_mtu) { … } static int ef4_set_mac_address(struct net_device *net_dev, void *data) { … } /* Context: netif_addr_lock held, BHs disabled. */ static void ef4_set_rx_mode(struct net_device *net_dev) { … } static int ef4_set_features(struct net_device *net_dev, netdev_features_t data) { … } static const struct net_device_ops ef4_netdev_ops = …; static void ef4_update_name(struct ef4_nic *efx) { … } static int ef4_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { … } static struct notifier_block ef4_netdev_notifier = …; static ssize_t phy_type_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(phy_type); static int ef4_register_netdev(struct ef4_nic *efx) { … } static void ef4_unregister_netdev(struct ef4_nic *efx) { … } /************************************************************************** * * Device reset and suspend * **************************************************************************/ /* Tears down the entire software state and most of the hardware state * before reset. */ void ef4_reset_down(struct ef4_nic *efx, enum reset_type method) { … } /* This function will always ensure that the locks acquired in * ef4_reset_down() are released. A failure return code indicates * that we were unable to reinitialise the hardware, and the * driver should be disabled. If ok is false, then the rx and tx * engines are not restarted, pending a RESET_DISABLE. */ int ef4_reset_up(struct ef4_nic *efx, enum reset_type method, bool ok) { … } /* Reset the NIC using the specified method. Note that the reset may * fail, in which case the card will be left in an unusable state. * * Caller must hold the rtnl_lock. */ int ef4_reset(struct ef4_nic *efx, enum reset_type method) { … } /* Try recovery mechanisms. * For now only EEH is supported. * Returns 0 if the recovery mechanisms are unsuccessful. * Returns a non-zero value otherwise. */ int ef4_try_recovery(struct ef4_nic *efx) { … } /* The worker thread exists so that code that cannot sleep can * schedule a reset for later. */ static void ef4_reset_work(struct work_struct *data) { … } void ef4_schedule_reset(struct ef4_nic *efx, enum reset_type type) { … } /************************************************************************** * * List of NICs we support * **************************************************************************/ /* PCI device ID table */ static const struct pci_device_id ef4_pci_table[] = …; /************************************************************************** * * Dummy PHY/MAC operations * * Can be used for some unimplemented operations * Needed so all function pointers are valid and do not have to be tested * before use * **************************************************************************/ int ef4_port_dummy_op_int(struct ef4_nic *efx) { … } void ef4_port_dummy_op_void(struct ef4_nic *efx) { … } static bool ef4_port_dummy_op_poll(struct ef4_nic *efx) { … } static const struct ef4_phy_operations ef4_dummy_phy_operations = …; /************************************************************************** * * Data housekeeping * **************************************************************************/ /* This zeroes out and then fills in the invariants in a struct * ef4_nic (including all sub-structures). */ static int ef4_init_struct(struct ef4_nic *efx, struct pci_dev *pci_dev, struct net_device *net_dev) { … } static void ef4_fini_struct(struct ef4_nic *efx) { … } void ef4_update_sw_stats(struct ef4_nic *efx, u64 *stats) { … } /************************************************************************** * * PCI interface * **************************************************************************/ /* Main body of final NIC shutdown code * This is called only at module unload (or hotplug removal). */ static void ef4_pci_remove_main(struct ef4_nic *efx) { … } /* Final NIC shutdown * This is called only at module unload (or hotplug removal). A PF can call * this on its VFs to ensure they are unbound first. */ static void ef4_pci_remove(struct pci_dev *pci_dev) { struct ef4_nic *efx; efx = pci_get_drvdata(pci_dev); if (!efx) return; /* Mark the NIC as fini, then stop the interface */ rtnl_lock(); ef4_dissociate(efx); dev_close(efx->net_dev); ef4_disable_interrupts(efx); efx->state = STATE_UNINIT; rtnl_unlock(); ef4_unregister_netdev(efx); ef4_mtd_remove(efx); ef4_pci_remove_main(efx); ef4_fini_io(efx); netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); ef4_fini_struct(efx); free_netdev(efx->net_dev); }; /* NIC VPD information * Called during probe to display the part number of the installed NIC. */ static void ef4_probe_vpd_strings(struct ef4_nic *efx) { … } /* Main body of NIC initialisation * This is called at module load (or hotplug insertion, theoretically). */ static int ef4_pci_probe_main(struct ef4_nic *efx) { … } /* NIC initialisation * * This is called at module load (or hotplug insertion, * theoretically). It sets up PCI mappings, resets the NIC, * sets up and registers the network devices with the kernel and hooks * the interrupt service routine. It does not prepare the device for * transmission; this is left to the first time one of the network * interfaces is brought up (i.e. ef4_net_open). */ static int ef4_pci_probe(struct pci_dev *pci_dev, const struct pci_device_id *entry) { … } static int ef4_pm_freeze(struct device *dev) { … } static int ef4_pm_thaw(struct device *dev) { … } static int ef4_pm_poweroff(struct device *dev) { … } /* Used for both resume and restore */ static int ef4_pm_resume(struct device *dev) { … } static int ef4_pm_suspend(struct device *dev) { … } static const struct dev_pm_ops ef4_pm_ops = …; /* A PCI error affecting this device was detected. * At this point MMIO and DMA may be disabled. * Stop the software path and request a slot reset. */ static pci_ers_result_t ef4_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { … } /* Fake a successful reset, which will be performed later in ef4_io_resume. */ static pci_ers_result_t ef4_io_slot_reset(struct pci_dev *pdev) { … } /* Perform the actual reset and resume I/O operations. */ static void ef4_io_resume(struct pci_dev *pdev) { … } /* For simplicity and reliability, we always require a slot reset and try to * reset the hardware when a pci error affecting the device is detected. * We leave both the link_reset and mmio_enabled callback unimplemented: * with our request for slot reset the mmio_enabled callback will never be * called, and the link_reset callback is not used by AER or EEH mechanisms. */ static const struct pci_error_handlers ef4_err_handlers = …; static struct pci_driver ef4_pci_driver = …; /************************************************************************** * * Kernel module interface * *************************************************************************/ module_param(interrupt_mode, uint, 0444); MODULE_PARM_DESC(…) …; static int __init ef4_init_module(void) { … } static void __exit ef4_exit_module(void) { … } module_init(…) …; module_exit(ef4_exit_module); MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_DEVICE_TABLE(pci, ef4_pci_table); MODULE_VERSION(…);
Codebase Browser
linux