// SPDX-License-Identifier: GPL-2.0-only /******************************************************************************* This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. ST Ethernet IPs are built around a Synopsys IP Core. Copyright(C) 2007-2011 STMicroelectronics Ltd Author: Giuseppe Cavallaro <[email protected]> Documentation available at: http://www.stlinux.com Support available at: https://bugzilla.stlinux.com/ *******************************************************************************/ #include <linux/clk.h> #include <linux/kernel.h> #include <linux/interrupt.h> #include <linux/ip.h> #include <linux/tcp.h> #include <linux/skbuff.h> #include <linux/ethtool.h> #include <linux/if_ether.h> #include <linux/crc32.h> #include <linux/mii.h> #include <linux/if.h> #include <linux/if_vlan.h> #include <linux/dma-mapping.h> #include <linux/slab.h> #include <linux/pm_runtime.h> #include <linux/prefetch.h> #include <linux/pinctrl/consumer.h> #ifdef CONFIG_DEBUG_FS #include <linux/debugfs.h> #include <linux/seq_file.h> #endif /* CONFIG_DEBUG_FS */ #include <linux/net_tstamp.h> #include <linux/phylink.h> #include <linux/udp.h> #include <linux/bpf_trace.h> #include <net/page_pool/helpers.h> #include <net/pkt_cls.h> #include <net/xdp_sock_drv.h> #include "stmmac_ptp.h" #include "stmmac.h" #include "stmmac_xdp.h" #include <linux/reset.h> #include <linux/of_mdio.h> #include "dwmac1000.h" #include "dwxgmac2.h" #include "hwif.h" /* As long as the interface is active, we keep the timestamping counter enabled * with fine resolution and binary rollover. This avoid non-monotonic behavior * (clock jumps) when changing timestamping settings at runtime. */ #define STMMAC_HWTS_ACTIVE … #define STMMAC_ALIGN(x) … #define TSO_MAX_BUFF_SIZE … /* Module parameters */ #define TX_TIMEO … static int watchdog = …; module_param(watchdog, int, 0644); MODULE_PARM_DESC(…) …; static int debug = …; module_param(debug, int, 0644); MODULE_PARM_DESC(…) …; static int phyaddr = …; module_param(phyaddr, int, 0444); MODULE_PARM_DESC(…) …; #define STMMAC_TX_THRESH(x) … #define STMMAC_RX_THRESH(x) … /* Limit to make sure XDP TX and slow path can coexist */ #define STMMAC_XSK_TX_BUDGET_MAX … #define STMMAC_TX_XSK_AVAIL … #define STMMAC_RX_FILL_BATCH … #define STMMAC_XDP_PASS … #define STMMAC_XDP_CONSUMED … #define STMMAC_XDP_TX … #define STMMAC_XDP_REDIRECT … static int flow_ctrl = …; module_param(flow_ctrl, int, 0644); MODULE_PARM_DESC(…) …; static int pause = …; module_param(pause, int, 0644); MODULE_PARM_DESC(…) …; #define TC_DEFAULT … static int tc = …; module_param(tc, int, 0644); MODULE_PARM_DESC(…) …; #define DEFAULT_BUFSIZE … static int buf_sz = …; module_param(buf_sz, int, 0644); MODULE_PARM_DESC(…) …; #define STMMAC_RX_COPYBREAK … static const u32 default_msg_level = …; #define STMMAC_DEFAULT_LPI_TIMER … static int eee_timer = …; module_param(eee_timer, int, 0644); MODULE_PARM_DESC(…) …; #define STMMAC_LPI_T(x) … /* By default the driver will use the ring mode to manage tx and rx descriptors, * but allow user to force to use the chain instead of the ring */ static unsigned int chain_mode; module_param(chain_mode, int, 0444); MODULE_PARM_DESC(…) …; static irqreturn_t stmmac_interrupt(int irq, void *dev_id); /* For MSI interrupts handling */ static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_queues_param(struct stmmac_priv *priv); static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, u32 rxmode, u32 chan); #ifdef CONFIG_DEBUG_FS static const struct net_device_ops stmmac_netdev_ops; static void stmmac_init_fs(struct net_device *dev); static void stmmac_exit_fs(struct net_device *dev); #endif #define STMMAC_COAL_TIMER(x) … int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) { … } EXPORT_SYMBOL_GPL(…); /** * stmmac_verify_args - verify the driver parameters. * Description: it checks the driver parameters and set a default in case of * errors. */ static void stmmac_verify_args(void) { … } static void __stmmac_disable_all_queues(struct stmmac_priv *priv) { … } /** * stmmac_disable_all_queues - Disable all queues * @priv: driver private structure */ static void stmmac_disable_all_queues(struct stmmac_priv *priv) { … } /** * stmmac_enable_all_queues - Enable all queues * @priv: driver private structure */ static void stmmac_enable_all_queues(struct stmmac_priv *priv) { … } static void stmmac_service_event_schedule(struct stmmac_priv *priv) { … } static void stmmac_global_err(struct stmmac_priv *priv) { … } /** * stmmac_clk_csr_set - dynamically set the MDC clock * @priv: driver private structure * Description: this is to dynamically set the MDC clock according to the csr * clock input. * Note: * If a specific clk_csr value is passed from the platform * this means that the CSR Clock Range selection cannot be * changed at run-time and it is fixed (as reported in the driver * documentation). Viceversa the driver will try to set the MDC * clock dynamically according to the actual clock input. */ static void stmmac_clk_csr_set(struct stmmac_priv *priv) { … } static void print_pkt(unsigned char *buf, int len) { … } static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) { … } /** * stmmac_rx_dirty - Get RX queue dirty * @priv: driver private structure * @queue: RX queue index */ static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) { … } static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) { … } /** * stmmac_enable_eee_mode - check and enter in LPI mode * @priv: driver private structure * Description: this function is to verify and enter in LPI mode in case of * EEE. */ static int stmmac_enable_eee_mode(struct stmmac_priv *priv) { … } /** * stmmac_disable_eee_mode - disable and exit from LPI mode * @priv: driver private structure * Description: this function is to exit and disable EEE in case of * LPI state is true. This is called by the xmit. */ void stmmac_disable_eee_mode(struct stmmac_priv *priv) { … } /** * stmmac_eee_ctrl_timer - EEE TX SW timer. * @t: timer_list struct containing private info * Description: * if there is no data transfer and if we are not in LPI state, * then MAC Transmitter can be moved to LPI state. */ static void stmmac_eee_ctrl_timer(struct timer_list *t) { … } /** * stmmac_eee_init - init EEE * @priv: driver private structure * Description: * if the GMAC supports the EEE (from the HW cap reg) and the phy device * can also manage EEE, this function enable the LPI state and start related * timer. */ bool stmmac_eee_init(struct stmmac_priv *priv) { … } /* stmmac_get_tx_hwtstamp - get HW TX timestamps * @priv: driver private structure * @p : descriptor pointer * @skb : the socket buffer * Description : * This function will read timestamp from the descriptor & pass it to stack. * and also perform some sanity checks. */ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, struct sk_buff *skb) { … } /* stmmac_get_rx_hwtstamp - get HW RX timestamps * @priv: driver private structure * @p : descriptor pointer * @np : next descriptor pointer * @skb : the socket buffer * Description : * This function will read received packet's timestamp from the descriptor * and pass it to stack. It also perform some sanity checks. */ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, struct dma_desc *np, struct sk_buff *skb) { … } /** * stmmac_hwtstamp_set - control hardware timestamping. * @dev: device pointer. * @ifr: An IOCTL specific structure, that can contain a pointer to * a proprietary structure used to pass information to the driver. * Description: * This function configures the MAC to enable/disable both outgoing(TX) * and incoming(RX) packets time stamping based on user input. * Return Value: * 0 on success and an appropriate -ve integer on failure. */ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) { … } /** * stmmac_hwtstamp_get - read hardware timestamping. * @dev: device pointer. * @ifr: An IOCTL specific structure, that can contain a pointer to * a proprietary structure used to pass information to the driver. * Description: * This function obtain the current hardware timestamping settings * as requested. */ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) { … } /** * stmmac_init_tstamp_counter - init hardware timestamping counter * @priv: driver private structure * @systime_flags: timestamping flags * Description: * Initialize hardware counter for packet timestamping. * This is valid as long as the interface is open and not suspended. * Will be rerun after resuming from suspend, case in which the timestamping * flags updated by stmmac_hwtstamp_set() also need to be restored. */ int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) { … } EXPORT_SYMBOL_GPL(…); /** * stmmac_init_ptp - init PTP * @priv: driver private structure * Description: this is to verify if the HW supports the PTPv1 or PTPv2. * This is done by looking at the HW cap. register. * This function also registers the ptp driver. */ static int stmmac_init_ptp(struct stmmac_priv *priv) { … } static void stmmac_release_ptp(struct stmmac_priv *priv) { … } /** * stmmac_mac_flow_ctrl - Configure flow control in all queues * @priv: driver private structure * @duplex: duplex passed to the next function * Description: It is used for configuring the flow control in all queues */ static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) { … } static unsigned long stmmac_mac_get_caps(struct phylink_config *config, phy_interface_t interface) { … } static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, phy_interface_t interface) { … } static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { … } static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) { … } static void stmmac_mac_link_down(struct phylink_config *config, unsigned int mode, phy_interface_t interface) { … } static void stmmac_mac_link_up(struct phylink_config *config, struct phy_device *phy, unsigned int mode, phy_interface_t interface, int speed, int duplex, bool tx_pause, bool rx_pause) { … } static const struct phylink_mac_ops stmmac_phylink_mac_ops = …; /** * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported * @priv: driver private structure * Description: this is to verify if the HW supports the PCS. * Physical Coding Sublayer (PCS) interface that can be used when the MAC is * configured for the TBI, RTBI, or SGMII PHY interface. */ static void stmmac_check_pcs_mode(struct stmmac_priv *priv) { … } /** * stmmac_init_phy - PHY initialization * @dev: net device structure * Description: it initializes the driver's PHY state, and attaches the PHY * to the mac driver. * Return value: * 0 on success */ static int stmmac_init_phy(struct net_device *dev) { … } static int stmmac_phy_setup(struct stmmac_priv *priv) { … } static void stmmac_display_rx_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } static void stmmac_display_tx_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } static void stmmac_display_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } static int stmmac_set_bfsize(int mtu, int bufsize) { … } /** * stmmac_clear_rx_descriptors - clear RX descriptors * @priv: driver private structure * @dma_conf: structure to take the dma data * @queue: RX queue index * Description: this function is called to clear the RX descriptors * in case of both basic and extended descriptors are used. */ static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } /** * stmmac_clear_tx_descriptors - clear tx descriptors * @priv: driver private structure * @dma_conf: structure to take the dma data * @queue: TX queue index. * Description: this function is called to clear the TX descriptors * in case of both basic and extended descriptors are used. */ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } /** * stmmac_clear_descriptors - clear descriptors * @priv: driver private structure * @dma_conf: structure to take the dma data * Description: this function is called to clear the TX and RX descriptors * in case of both basic and extended descriptors are used. */ static void stmmac_clear_descriptors(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * stmmac_init_rx_buffers - init the RX descriptor buffer. * @priv: driver private structure * @dma_conf: structure to take the dma data * @p: descriptor pointer * @i: descriptor index * @flags: gfp flag * @queue: RX queue index * Description: this function is called to allocate a receive buffer, perform * the DMA mapping and init the descriptor. */ static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, struct dma_desc *p, int i, gfp_t flags, u32 queue) { … } /** * stmmac_free_rx_buffer - free RX dma buffers * @priv: private structure * @rx_q: RX queue * @i: buffer index. */ static void stmmac_free_rx_buffer(struct stmmac_priv *priv, struct stmmac_rx_queue *rx_q, int i) { … } /** * stmmac_free_tx_buffer - free RX dma buffers * @priv: private structure * @dma_conf: structure to take the dma data * @queue: RX queue index * @i: buffer index. */ static void stmmac_free_tx_buffer(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue, int i) { … } /** * dma_free_rx_skbufs - free RX dma buffers * @priv: private structure * @dma_conf: structure to take the dma data * @queue: RX queue index */ static void dma_free_rx_skbufs(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue, gfp_t flags) { … } /** * dma_free_rx_xskbufs - free RX dma buffers from XSK pool * @priv: private structure * @dma_conf: structure to take the dma data * @queue: RX queue index */ static void dma_free_rx_xskbufs(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) { … } /** * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) * @priv: driver private structure * @dma_conf: structure to take the dma data * @queue: RX queue index * @flags: gfp flag. * Description: this function initializes the DMA RX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue, gfp_t flags) { … } static int init_dma_rx_desc_rings(struct net_device *dev, struct stmmac_dma_conf *dma_conf, gfp_t flags) { … } /** * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) * @priv: driver private structure * @dma_conf: structure to take the dma data * @queue: TX queue index * Description: this function initializes the DMA TX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static int init_dma_tx_desc_rings(struct net_device *dev, struct stmmac_dma_conf *dma_conf) { … } /** * init_dma_desc_rings - init the RX/TX descriptor rings * @dev: net device structure * @dma_conf: structure to take the dma data * @flags: gfp flag. * Description: this function initializes the DMA RX/TX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ static int init_dma_desc_rings(struct net_device *dev, struct stmmac_dma_conf *dma_conf, gfp_t flags) { … } /** * dma_free_tx_skbufs - free TX dma buffers * @priv: private structure * @dma_conf: structure to take the dma data * @queue: TX queue index */ static void dma_free_tx_skbufs(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } /** * stmmac_free_tx_skbufs - free TX skb buffers * @priv: private structure */ static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) { … } /** * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) * @priv: private structure * @dma_conf: structure to take the dma data * @queue: RX queue index */ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static void free_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) * @priv: private structure * @dma_conf: structure to take the dma data * @queue: TX queue index */ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static void free_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). * @priv: private structure * @dma_conf: structure to take the dma data * @queue: RX queue index * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). * @priv: private structure * @dma_conf: structure to take the dma data * @queue: TX queue index * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf, u32 queue) { … } static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * alloc_dma_desc_resources - alloc TX/RX resources. * @priv: private structure * @dma_conf: structure to take the dma data * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ static int alloc_dma_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * free_dma_desc_resources - free dma desc resources * @priv: private structure * @dma_conf: structure to take the dma data */ static void free_dma_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { … } /** * stmmac_mac_enable_rx_queues - Enable MAC rx queues * @priv: driver private structure * Description: It is used for enabling the rx queues in the MAC */ static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) { … } /** * stmmac_start_rx_dma - start RX DMA channel * @priv: driver private structure * @chan: RX channel index * Description: * This starts a RX DMA channel */ static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) { … } /** * stmmac_start_tx_dma - start TX DMA channel * @priv: driver private structure * @chan: TX channel index * Description: * This starts a TX DMA channel */ static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) { … } /** * stmmac_stop_rx_dma - stop RX DMA channel * @priv: driver private structure * @chan: RX channel index * Description: * This stops a RX DMA channel */ static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) { … } /** * stmmac_stop_tx_dma - stop TX DMA channel * @priv: driver private structure * @chan: TX channel index * Description: * This stops a TX DMA channel */ static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) { … } static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) { … } /** * stmmac_start_all_dma - start all RX and TX DMA channels * @priv: driver private structure * Description: * This starts all the RX and TX DMA channels */ static void stmmac_start_all_dma(struct stmmac_priv *priv) { … } /** * stmmac_stop_all_dma - stop all RX and TX DMA channels * @priv: driver private structure * Description: * This stops the RX and TX DMA channels */ static void stmmac_stop_all_dma(struct stmmac_priv *priv) { … } /** * stmmac_dma_operation_mode - HW DMA operation mode * @priv: driver private structure * Description: it is used for configuring the DMA operation mode register in * order to program the tx/rx DMA thresholds or Store-And-Forward mode. */ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) { … } static void stmmac_xsk_request_timestamp(void *_priv) { … } static u64 stmmac_xsk_fill_timestamp(void *_priv) { … } static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = …; static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) { … } static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) { … } /** * stmmac_tx_clean - to manage the transmission completion * @priv: driver private structure * @budget: napi budget limiting this functions packet handling * @queue: TX queue index * @pending_packets: signal to arm the TX coal timer * Description: it reclaims the transmit resources after transmission completes. * If some packets still needs to be handled, due to TX coalesce, set * pending_packets to true to make NAPI arm the TX coal timer. */ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, bool *pending_packets) { … } /** * stmmac_tx_err - to manage the tx error * @priv: driver private structure * @chan: channel index * Description: it cleans the descriptors and restarts the transmission * in case of transmission errors. */ static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) { … } /** * stmmac_set_dma_operation_mode - Set DMA operation mode by channel * @priv: driver private structure * @txmode: TX operating mode * @rxmode: RX operating mode * @chan: channel index * Description: it is used for configuring of the DMA operation mode in * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward * mode. */ static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, u32 rxmode, u32 chan) { … } static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) { … } static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) { … } /** * stmmac_dma_interrupt - DMA ISR * @priv: driver private structure * Description: this is the DMA ISR. It is called by the main ISR. * It calls the dwmac dma routine and schedule poll method in case of some * work can be done. */ static void stmmac_dma_interrupt(struct stmmac_priv *priv) { … } /** * stmmac_mmc_setup: setup the Mac Management Counters (MMC) * @priv: driver private structure * Description: this masks the MMC irq, in fact, the counters are managed in SW. */ static void stmmac_mmc_setup(struct stmmac_priv *priv) { … } /** * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. * @priv: driver private structure * Description: * new GMAC chip generations have a new register to indicate the * presence of the optional feature/functions. * This can be also used to override the value passed through the * platform and necessary for old MAC10/100 and GMAC chips. */ static int stmmac_get_hw_features(struct stmmac_priv *priv) { … } /** * stmmac_check_ether_addr - check if the MAC addr is valid * @priv: driver private structure * Description: * it is to verify if the MAC address is valid, in case of failures it * generates a random MAC address */ static void stmmac_check_ether_addr(struct stmmac_priv *priv) { … } /** * stmmac_init_dma_engine - DMA init. * @priv: driver private structure * Description: * It inits the DMA invoking the specific MAC/GMAC callback. * Some DMA parameters can be passed from the platform; * in case of these are not passed a default is kept for the MAC or GMAC. */ static int stmmac_init_dma_engine(struct stmmac_priv *priv) { … } static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) { … } /** * stmmac_tx_timer - mitigation sw timer for tx. * @t: data pointer * Description: * This is the timer handler to directly invoke the stmmac_tx_clean. */ static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) { … } /** * stmmac_init_coalesce - init mitigation options. * @priv: driver private structure * Description: * This inits the coalesce parameters: i.e. timer rate, * timer handler and default threshold used for enabling the * interrupt on completion bit. */ static void stmmac_init_coalesce(struct stmmac_priv *priv) { … } static void stmmac_set_rings_length(struct stmmac_priv *priv) { … } /** * stmmac_set_tx_queue_weight - Set TX queue weight * @priv: driver private structure * Description: It is used for setting TX queues weight */ static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) { … } /** * stmmac_configure_cbs - Configure CBS in TX queue * @priv: driver private structure * Description: It is used for configuring CBS in AVB TX queues */ static void stmmac_configure_cbs(struct stmmac_priv *priv) { … } /** * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel * @priv: driver private structure * Description: It is used for mapping RX queues to RX dma channels */ static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) { … } /** * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority * @priv: driver private structure * Description: It is used for configuring the RX Queue Priority */ static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) { … } /** * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority * @priv: driver private structure * Description: It is used for configuring the TX Queue Priority */ static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) { … } /** * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing * @priv: driver private structure * Description: It is used for configuring the RX queue routing */ static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) { … } static void stmmac_mac_config_rss(struct stmmac_priv *priv) { … } /** * stmmac_mtl_configuration - Configure MTL * @priv: driver private structure * Description: It is used for configurring MTL */ static void stmmac_mtl_configuration(struct stmmac_priv *priv) { … } static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) { … } static int stmmac_fpe_start_wq(struct stmmac_priv *priv) { … } /** * stmmac_hw_setup - setup mac in a usable state. * @dev : pointer to the device structure. * @ptp_register: register PTP if set * Description: * this is the main function to setup the HW in a usable state because the * dma engine is reset, the core registers are configured (e.g. AXI, * Checksum features, timers). The DMA is ready to start receiving and * transmitting. * Return value: * 0 on success and an appropriate (-)ve integer as defined in errno.h * file on failure. */ static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) { … } static void stmmac_hw_teardown(struct net_device *dev) { … } static void stmmac_free_irq(struct net_device *dev, enum request_irq_err irq_err, int irq_idx) { … } static int stmmac_request_irq_multi_msi(struct net_device *dev) { … } static int stmmac_request_irq_single(struct net_device *dev) { … } static int stmmac_request_irq(struct net_device *dev) { … } /** * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue * @priv: driver private structure * @mtu: MTU to setup the dma queue and buf with * Description: Allocate and generate a dma_conf based on the provided MTU. * Allocate the Tx/Rx DMA queue and init them. * Return value: * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. */ static struct stmmac_dma_conf * stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) { … } /** * __stmmac_open - open entry point of the driver * @dev : pointer to the device structure. * @dma_conf : structure to take the dma data * Description: * This function is the open entry point of the driver. * Return value: * 0 on success and an appropriate (-)ve integer as defined in errno.h * file on failure. */ static int __stmmac_open(struct net_device *dev, struct stmmac_dma_conf *dma_conf) { … } static int stmmac_open(struct net_device *dev) { … } static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) { … } /** * stmmac_release - close entry point of the driver * @dev : device pointer. * Description: * This is the stop entry point of the driver. */ static int stmmac_release(struct net_device *dev) { … } static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, struct stmmac_tx_queue *tx_q) { … } /** * stmmac_tso_allocator - close entry point of the driver * @priv: driver private structure * @des: buffer start address * @total_len: total length to fill in descriptors * @last_segment: condition for the last descriptor * @queue: TX queue index * Description: * This function fills descriptor and request new descriptors according to * buffer length to fill */ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, int total_len, bool last_segment, u32 queue) { … } static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) { … } /** * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) * @skb : the socket buffer * @dev : device pointer * Description: this is the transmit function that is called on TSO frames * (support available on GMAC4 and newer chips). * Diagram below show the ring programming in case of TSO frames: * * First Descriptor * -------- * | DES0 |---> buffer1 = L2/L3/L4 header * | DES1 |---> TCP Payload (can continue on next descr...) * | DES2 |---> buffer 1 and 2 len * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] * -------- * | * ... * | * -------- * | DES0 | --| Split TCP Payload on Buffers 1 and 2 * | DES1 | --| * | DES2 | --> buffer 1 and 2 len * | DES3 | * -------- * * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. */ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) { … } /** * stmmac_has_ip_ethertype() - Check if packet has IP ethertype * @skb: socket buffer to check * * Check if a packet has an ethertype that will trigger the IP header checks * and IP/TCP checksum engine of the stmmac core. * * Return: true if the ethertype can trigger the checksum engine, false * otherwise */ static bool stmmac_has_ip_ethertype(struct sk_buff *skb) { … } /** * stmmac_xmit - Tx entry point of the driver * @skb : the socket buffer * @dev : device pointer * Description : this is the tx entry point of the driver. * It programs the chain or the ring and supports oversized frames * and SG feature. */ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) { … } static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) { … } /** * stmmac_rx_refill - refill used skb preallocated buffers * @priv: driver private structure * @queue: RX queue index * Description : this is to reallocate the skb for the reception process * that is based on zero-copy. */ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) { … } static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, struct dma_desc *p, int status, unsigned int len) { … } static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, struct dma_desc *p, int status, unsigned int len) { … } static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, struct xdp_frame *xdpf, bool dma_map) { … } static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, int cpu) { … } static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, struct xdp_buff *xdp) { … } static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, struct bpf_prog *prog, struct xdp_buff *xdp) { … } static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, struct xdp_buff *xdp) { … } static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, int xdp_status) { … } static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, struct xdp_buff *xdp) { … } static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, struct dma_desc *p, struct dma_desc *np, struct xdp_buff *xdp) { … } static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) { … } static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) { … } static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) { … } /** * stmmac_rx - manage the receive process * @priv: driver private structure * @limit: napi bugget * @queue: RX queue index. * Description : this the function called by the napi poll method. * It gets all the frames inside the ring. */ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) { … } static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) { … } static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) { … } static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) { … } /** * stmmac_tx_timeout * @dev : Pointer to net device structure * @txqueue: the index of the hanging transmit queue * Description: this function is called when a packet transmission fails to * complete within a reasonable time. The driver will mark the error in the * netdev structure and arrange for the device to be reset to a sane state * in order to transmit a new packet. */ static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) { … } /** * stmmac_set_rx_mode - entry point for multicast addressing * @dev : pointer to the device structure * Description: * This function is a driver entry point which gets called by the kernel * whenever multicast addresses must be enabled/disabled. * Return value: * void. */ static void stmmac_set_rx_mode(struct net_device *dev) { … } /** * stmmac_change_mtu - entry point to change MTU size for the device. * @dev : device pointer. * @new_mtu : the new MTU size for the device. * Description: the Maximum Transfer Unit (MTU) is used by the network layer * to drive packet transmission. Ethernet has an MTU of 1500 octets * (ETH_DATA_LEN). This value can be changed with ifconfig. * Return value: * 0 on success and an appropriate (-)ve integer as defined in errno.h * file on failure. */ static int stmmac_change_mtu(struct net_device *dev, int new_mtu) { … } static netdev_features_t stmmac_fix_features(struct net_device *dev, netdev_features_t features) { … } static int stmmac_set_features(struct net_device *netdev, netdev_features_t features) { … } static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) { … } static void stmmac_common_interrupt(struct stmmac_priv *priv) { … } /** * stmmac_interrupt - main ISR * @irq: interrupt number. * @dev_id: to pass the net device pointer. * Description: this is the main driver interrupt service routine. * It can call: * o DMA service routine (to manage incoming frame reception and transmission * status) * o Core interrupts to manage: remote wake-up, management counter, LPI * interrupts. */ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) { … } static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) { … } static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) { … } static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) { … } static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) { … } /** * stmmac_ioctl - Entry point for the Ioctl * @dev: Device pointer. * @rq: An IOCTL specefic structure, that can contain a pointer to * a proprietary structure used to pass information to the driver. * @cmd: IOCTL command * Description: * Currently it supports the phy_mii_ioctl(...) and HW time stamping. */ static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) { … } static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { … } static LIST_HEAD(stmmac_block_cb_list); static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, void *type_data) { … } static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, struct net_device *sb_dev) { … } static int stmmac_set_mac_address(struct net_device *ndev, void *addr) { … } #ifdef CONFIG_DEBUG_FS static struct dentry *stmmac_fs_dir; static void sysfs_display_ring(void *head, int size, int extend_desc, struct seq_file *seq, dma_addr_t dma_phy_addr) { … } static int stmmac_rings_status_show(struct seq_file *seq, void *v) { … } DEFINE_SHOW_ATTRIBUTE(…); static int stmmac_dma_cap_show(struct seq_file *seq, void *v) { … } DEFINE_SHOW_ATTRIBUTE(…); /* Use network device events to rename debugfs file entries. */ static int stmmac_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { … } static struct notifier_block stmmac_notifier = …; static void stmmac_init_fs(struct net_device *dev) { … } static void stmmac_exit_fs(struct net_device *dev) { … } #endif /* CONFIG_DEBUG_FS */ static u32 stmmac_vid_crc32_le(__le16 vid_le) { … } static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) { … } static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) { … } static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { … } static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) { … } static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags) { … } void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) { … } void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) { … } void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) { … } void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) { … } void stmmac_xdp_release(struct net_device *dev) { … } int stmmac_xdp_open(struct net_device *dev) { … } int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) { … } static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) { … } static const struct net_device_ops stmmac_netdev_ops = …; static void stmmac_reset_subtask(struct stmmac_priv *priv) { … } static void stmmac_service_task(struct work_struct *work) { … } /** * stmmac_hw_init - Init the MAC device * @priv: driver private structure * Description: this function is to configure the MAC device according to * some platform parameters or the HW capability register. It prepares the * driver to use either ring or chain modes and to setup either enhanced or * normal descriptors. */ static int stmmac_hw_init(struct stmmac_priv *priv) { … } static void stmmac_napi_add(struct net_device *dev) { … } static void stmmac_napi_del(struct net_device *dev) { … } int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) { … } int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) { … } #define SEND_VERIFY_MPAKCET_FMT … static void stmmac_fpe_lp_task(struct work_struct *work) { … } void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) { … } static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) { … } static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = …; /** * stmmac_dvr_probe * @device: device pointer * @plat_dat: platform data pointer * @res: stmmac resource pointer * Description: this is the main probe function used to * call the alloc_etherdev, allocate the priv structure. * Return: * returns 0 on success, otherwise errno. */ int stmmac_dvr_probe(struct device *device, struct plat_stmmacenet_data *plat_dat, struct stmmac_resources *res) { … } EXPORT_SYMBOL_GPL(…); /** * stmmac_dvr_remove * @dev: device pointer * Description: this function resets the TX/RX processes, disables the MAC RX/TX * changes the link status, releases the DMA descriptor rings. */ void stmmac_dvr_remove(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); /** * stmmac_suspend - suspend callback * @dev: device pointer * Description: this is the function to suspend the device and it is called * by the platform driver to stop the network queue, release the resources, * program the PMT register (for WoL), clean and release driver resources. */ int stmmac_suspend(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) { … } static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) { … } /** * stmmac_reset_queues_param - reset queue parameters * @priv: device pointer */ static void stmmac_reset_queues_param(struct stmmac_priv *priv) { … } /** * stmmac_resume - resume callback * @dev: device pointer * Description: when resume this function is invoked to setup the DMA and CORE * in a usable state. */ int stmmac_resume(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); #ifndef MODULE static int __init stmmac_cmdline_opt(char *str) { … } __setup(…); #endif /* MODULE */ static int __init stmmac_init(void) { … } static void __exit stmmac_exit(void) { … } module_init(…) … module_exit(…) MODULE_DESCRIPTION(…) …; MODULE_AUTHOR(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux