// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom GENET (Gigabit Ethernet) controller driver * * Copyright (c) 2014-2024 Broadcom */ #define pr_fmt(fmt) … #include <linux/acpi.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/types.h> #include <linux/fcntl.h> #include <linux/interrupt.h> #include <linux/string.h> #include <linux/if_ether.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/delay.h> #include <linux/platform_device.h> #include <linux/dma-mapping.h> #include <linux/pm.h> #include <linux/clk.h> #include <net/arp.h> #include <linux/mii.h> #include <linux/ethtool.h> #include <linux/netdevice.h> #include <linux/inetdevice.h> #include <linux/etherdevice.h> #include <linux/skbuff.h> #include <linux/in.h> #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/phy.h> #include <linux/platform_data/bcmgenet.h> #include <linux/unaligned.h> #include "bcmgenet.h" /* Maximum number of hardware queues, downsized if needed */ #define GENET_MAX_MQ_CNT … /* Default highest priority queue for multi queue support */ #define GENET_Q0_PRIORITY … #define GENET_Q16_RX_BD_CNT … #define GENET_Q16_TX_BD_CNT … #define RX_BUF_LENGTH … #define SKB_ALIGNMENT … /* Tx/Rx DMA register offset, skip 256 descriptors */ #define WORDS_PER_BD(p) … #define DMA_DESC_SIZE … #define GENET_TDMA_REG_OFF … #define GENET_RDMA_REG_OFF … /* Forward declarations */ static void bcmgenet_set_rx_mode(struct net_device *dev); static inline void bcmgenet_writel(u32 value, void __iomem *offset) { … } static inline u32 bcmgenet_readl(void __iomem *offset) { … } static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv, void __iomem *d, u32 value) { … } static inline void dmadesc_set_addr(struct bcmgenet_priv *priv, void __iomem *d, dma_addr_t addr) { … } /* Combined address + length/status setter */ static inline void dmadesc_set(struct bcmgenet_priv *priv, void __iomem *d, dma_addr_t addr, u32 val) { … } #define GENET_VER_FMT … #define GENET_MSG_DEFAULT … static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv) { … } static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val) { … } /* These macros are defined to deal with register map change * between GENET1.1 and GENET2. Only those currently being used * by driver are defined. */ static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv) { … } static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val) { … } static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv) { … } static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val) { … } /* RX/TX DMA register accessors */ enum dma_reg { … }; static const u8 bcmgenet_dma_regs_v3plus[] = …; static const u8 bcmgenet_dma_regs_v2[] = …; static const u8 bcmgenet_dma_regs_v1[] = …; /* Set at runtime once bcmgenet version is known */ static const u8 *bcmgenet_dma_regs; static inline struct bcmgenet_priv *dev_to_priv(struct device *dev) { … } static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv, enum dma_reg r) { … } static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv, u32 val, enum dma_reg r) { … } static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv, enum dma_reg r) { … } static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv, u32 val, enum dma_reg r) { … } /* RDMA/TDMA ring registers and accessors * we merge the common fields and just prefix with T/D the registers * having different meaning depending on the direction */ enum dma_ring_reg { … }; /* GENET v4 supports 40-bits pointer addressing * for obvious reasons the LO and HI word parts * are contiguous, but this offsets the other * registers. */ static const u8 genet_dma_ring_regs_v4[] = …; static const u8 genet_dma_ring_regs_v123[] = …; /* Set at runtime once GENET version is known */ static const u8 *genet_dma_ring_regs; static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv, unsigned int ring, enum dma_ring_reg r) { … } static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv, unsigned int ring, u32 val, enum dma_ring_reg r) { … } static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv, unsigned int ring, enum dma_ring_reg r) { … } static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv, unsigned int ring, u32 val, enum dma_ring_reg r) { … } static void bcmgenet_hfb_enable_filter(struct bcmgenet_priv *priv, u32 f_index) { … } static void bcmgenet_hfb_disable_filter(struct bcmgenet_priv *priv, u32 f_index) { … } static void bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv *priv, u32 f_index, u32 rx_queue) { … } static void bcmgenet_hfb_set_filter_length(struct bcmgenet_priv *priv, u32 f_index, u32 f_length) { … } static int bcmgenet_hfb_validate_mask(void *mask, size_t size) { … } #define VALIDATE_MASK(x) … static int bcmgenet_hfb_insert_data(struct bcmgenet_priv *priv, u32 f_index, u32 offset, void *val, void *mask, size_t size) { … } static void bcmgenet_hfb_create_rxnfc_filter(struct bcmgenet_priv *priv, struct bcmgenet_rxnfc_rule *rule) { … } /* bcmgenet_hfb_clear * * Clear Hardware Filter Block and disable all filtering. */ static void bcmgenet_hfb_clear_filter(struct bcmgenet_priv *priv, u32 f_index) { … } static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv) { … } static void bcmgenet_hfb_init(struct bcmgenet_priv *priv) { … } static int bcmgenet_begin(struct net_device *dev) { … } static void bcmgenet_complete(struct net_device *dev) { … } static int bcmgenet_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *cmd) { … } static int bcmgenet_set_link_ksettings(struct net_device *dev, const struct ethtool_link_ksettings *cmd) { … } static int bcmgenet_set_features(struct net_device *dev, netdev_features_t features) { … } static u32 bcmgenet_get_msglevel(struct net_device *dev) { … } static void bcmgenet_set_msglevel(struct net_device *dev, u32 level) { … } static int bcmgenet_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { … } static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring, u32 usecs, u32 pkts) { … } static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring, struct ethtool_coalesce *ec) { … } static int bcmgenet_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { … } static void bcmgenet_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause) { … } static int bcmgenet_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause) { … } /* standard ethtool support functions. */ enum bcmgenet_stat_type { … }; struct bcmgenet_stats { … }; #define STAT_NETDEV(m) … #define STAT_GENET_MIB(str, m, _type) … #define STAT_GENET_MIB_RX(str, m) … #define STAT_GENET_MIB_TX(str, m) … #define STAT_GENET_RUNT(str, m) … #define STAT_GENET_SOFT_MIB(str, m) … #define STAT_GENET_MISC(str, m, offset) … #define STAT_GENET_Q(num) … /* There is a 0xC gap between the end of RX and beginning of TX stats and then * between the end of TX stats and the beginning of the RX RUNT */ #define BCMGENET_STAT_OFFSET … /* Hardware counters must be kept in sync because the order/offset * is important here (order in structure declaration = order in hardware) */ static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = …; #define BCMGENET_STATS_LEN … static void bcmgenet_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { … } static int bcmgenet_get_sset_count(struct net_device *dev, int string_set) { … } static void bcmgenet_get_strings(struct net_device *dev, u32 stringset, u8 *data) { … } static u32 bcmgenet_update_stat_misc(struct bcmgenet_priv *priv, u16 offset) { … } static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv) { … } static void bcmgenet_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { … } void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, bool tx_lpi_enabled) { … } static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) { … } static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) { … } static int bcmgenet_validate_flow(struct net_device *dev, struct ethtool_rxnfc *cmd) { … } static int bcmgenet_insert_flow(struct net_device *dev, struct ethtool_rxnfc *cmd) { … } static int bcmgenet_delete_flow(struct net_device *dev, struct ethtool_rxnfc *cmd) { … } static int bcmgenet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) { … } static int bcmgenet_get_flow(struct net_device *dev, struct ethtool_rxnfc *cmd, int loc) { … } static int bcmgenet_get_num_flows(struct bcmgenet_priv *priv) { … } static int bcmgenet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, u32 *rule_locs) { … } /* standard ethtool support functions. */ static const struct ethtool_ops bcmgenet_ethtool_ops = …; /* Power down the unimac, based on mode. */ static int bcmgenet_power_down(struct bcmgenet_priv *priv, enum bcmgenet_power_mode mode) { … } static void bcmgenet_power_up(struct bcmgenet_priv *priv, enum bcmgenet_power_mode mode) { … } static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv, struct bcmgenet_tx_ring *ring) { … } static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv, struct bcmgenet_tx_ring *ring) { … } static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring) { … } static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring) { … } static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring) { … } static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring) { … } static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring) { … } static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring) { … } static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring) { … } static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring) { … } /* Simple helper to free a transmit control block's resources * Returns an skb when the last transmit control block associated with the * skb is freed. The skb should be freed by the caller if necessary. */ static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev, struct enet_cb *cb) { … } /* Simple helper to free a receive control block's resources */ static struct sk_buff *bcmgenet_free_rx_cb(struct device *dev, struct enet_cb *cb) { … } /* Unlocked version of the reclaim routine */ static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev, struct bcmgenet_tx_ring *ring) { … } static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, struct bcmgenet_tx_ring *ring) { … } static int bcmgenet_tx_poll(struct napi_struct *napi, int budget) { … } static void bcmgenet_tx_reclaim_all(struct net_device *dev) { … } /* Reallocate the SKB to put enough headroom in front of it and insert * the transmit checksum offsets in the descriptors */ static struct sk_buff *bcmgenet_add_tsb(struct net_device *dev, struct sk_buff *skb) { … } static void bcmgenet_hide_tsb(struct sk_buff *skb) { … } static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) { … } static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv, struct enet_cb *cb) { … } /* bcmgenet_desc_rx - descriptor based rx process. * this could be called from bottom half, or from NAPI polling method. */ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, unsigned int budget) { … } /* Rx NAPI polling method */ static int bcmgenet_rx_poll(struct napi_struct *napi, int budget) { … } static void bcmgenet_dim_work(struct work_struct *work) { … } /* Assign skb to RX DMA descriptor. */ static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv, struct bcmgenet_rx_ring *ring) { … } static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv) { … } static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable) { … } static void reset_umac(struct bcmgenet_priv *priv) { … } static void bcmgenet_intr_disable(struct bcmgenet_priv *priv) { … } static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv) { … } static void init_umac(struct bcmgenet_priv *priv) { … } static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring, void (*cb)(struct work_struct *work)) { … } static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring) { … } /* Initialize a Tx ring along with corresponding hardware registers */ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, unsigned int index, unsigned int size, unsigned int start_ptr, unsigned int end_ptr) { … } /* Initialize a RDMA ring */ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, unsigned int index, unsigned int size, unsigned int start_ptr, unsigned int end_ptr) { … } static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv) { … } static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv) { … } static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv) { … } /* Initialize Tx queues * * Queues 0-3 are priority-based, each one has 32 descriptors, * with queue 0 being the highest priority queue. * * Queue 16 is the default Tx queue with * GENET_Q16_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors. * * The transmit control block pool is then partitioned as follows: * - Tx queue 0 uses tx_cbs[0..31] * - Tx queue 1 uses tx_cbs[32..63] * - Tx queue 2 uses tx_cbs[64..95] * - Tx queue 3 uses tx_cbs[96..127] * - Tx queue 16 uses tx_cbs[128..255] */ static void bcmgenet_init_tx_queues(struct net_device *dev) { … } static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv) { … } static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv) { … } static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv) { … } /* Initialize Rx queues * * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be * used to direct traffic to these queues. * * Queue 16 is the default Rx queue with GENET_Q16_RX_BD_CNT descriptors. */ static int bcmgenet_init_rx_queues(struct net_device *dev) { … } static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv) { … } static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) { … } /* init_edma: Initialize DMA control register */ static int bcmgenet_init_dma(struct bcmgenet_priv *priv) { … } /* Interrupt bottom half */ static void bcmgenet_irq_task(struct work_struct *work) { … } /* bcmgenet_isr1: handle Rx and Tx priority queues */ static irqreturn_t bcmgenet_isr1(int irq, void *dev_id) { … } /* bcmgenet_isr0: handle Rx and Tx default queues + other stuff */ static irqreturn_t bcmgenet_isr0(int irq, void *dev_id) { … } static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id) { … } static void bcmgenet_umac_reset(struct bcmgenet_priv *priv) { … } static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv, const unsigned char *addr) { … } static void bcmgenet_get_hw_addr(struct bcmgenet_priv *priv, unsigned char *addr) { … } /* Returns a reusable dma control register value */ static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv, bool flush_rx) { … } static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl) { … } static void bcmgenet_netif_start(struct net_device *dev) { … } static int bcmgenet_open(struct net_device *dev) { … } static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) { … } static int bcmgenet_close(struct net_device *dev) { … } static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring) { … } static void bcmgenet_timeout(struct net_device *dev, unsigned int txqueue) { … } #define MAX_MDF_FILTER … static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv, const unsigned char *addr, int *i) { … } static void bcmgenet_set_rx_mode(struct net_device *dev) { … } /* Set the hardware MAC address. */ static int bcmgenet_set_mac_addr(struct net_device *dev, void *p) { … } static struct net_device_stats *bcmgenet_get_stats(struct net_device *dev) { … } static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier) { … } static const struct net_device_ops bcmgenet_netdev_ops = …; /* Array of GENET hardware parameters/characteristics */ static struct bcmgenet_hw_params bcmgenet_hw_params[] = …; /* Infer hardware parameters from the detected GENET version */ static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv) { … } struct bcmgenet_plat_data { … }; static const struct bcmgenet_plat_data v1_plat_data = …; static const struct bcmgenet_plat_data v2_plat_data = …; static const struct bcmgenet_plat_data v3_plat_data = …; static const struct bcmgenet_plat_data v4_plat_data = …; static const struct bcmgenet_plat_data v5_plat_data = …; static const struct bcmgenet_plat_data bcm2711_plat_data = …; static const struct bcmgenet_plat_data bcm7712_plat_data = …; static const struct of_device_id bcmgenet_match[] = …; MODULE_DEVICE_TABLE(of, bcmgenet_match); static int bcmgenet_probe(struct platform_device *pdev) { … } static void bcmgenet_remove(struct platform_device *pdev) { … } static void bcmgenet_shutdown(struct platform_device *pdev) { … } #ifdef CONFIG_PM_SLEEP static int bcmgenet_resume_noirq(struct device *d) { … } static int bcmgenet_resume(struct device *d) { … } static int bcmgenet_suspend(struct device *d) { … } static int bcmgenet_suspend_noirq(struct device *d) { … } #else #define bcmgenet_suspend … #define bcmgenet_suspend_noirq … #define bcmgenet_resume … #define bcmgenet_resume_noirq … #endif /* CONFIG_PM_SLEEP */ static const struct dev_pm_ops bcmgenet_pm_ops = …; static const struct acpi_device_id genet_acpi_match[] = …; MODULE_DEVICE_TABLE(acpi, genet_acpi_match); static struct platform_driver bcmgenet_driver = …; module_platform_driver(…) …; MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_ALIAS(…) …; MODULE_LICENSE(…) …; MODULE_SOFTDEP(…) …;
Codebase Browser
linux