// SPDX-License-Identifier: GPL-2.0+ /* Copyright (c) Tehuti Networks Ltd. */ #include <linux/bitfield.h> #include <linux/ethtool.h> #include <linux/firmware.h> #include <linux/if_vlan.h> #include <linux/iopoll.h> #include <linux/netdevice.h> #include <linux/pci.h> #include <linux/phylink.h> #include <linux/vmalloc.h> #include <net/netdev_queues.h> #include <net/page_pool/helpers.h> #include "tn40.h" #define TN40_SHORT_PACKET_SIZE … #define TN40_FIRMWARE_NAME … static void tn40_enable_interrupts(struct tn40_priv *priv) { … } static void tn40_disable_interrupts(struct tn40_priv *priv) { … } static int tn40_fifo_alloc(struct tn40_priv *priv, struct tn40_fifo *f, int fsz_type, u16 reg_cfg0, u16 reg_cfg1, u16 reg_rptr, u16 reg_wptr) { … } static void tn40_fifo_free(struct tn40_priv *priv, struct tn40_fifo *f) { … } static struct tn40_rxdb *tn40_rxdb_alloc(int nelem) { … } static void tn40_rxdb_free(struct tn40_rxdb *db) { … } static int tn40_rxdb_alloc_elem(struct tn40_rxdb *db) { … } static void *tn40_rxdb_addr_elem(struct tn40_rxdb *db, unsigned int n) { … } static int tn40_rxdb_available(struct tn40_rxdb *db) { … } static void tn40_rxdb_free_elem(struct tn40_rxdb *db, unsigned int n) { … } /** * tn40_create_rx_ring - Initialize RX all related HW and SW resources * @priv: NIC private structure * * create_rx_ring creates rxf and rxd fifos, updates the relevant HW registers, * preallocates skbs for rx. It assumes that Rx is disabled in HW funcs are * grouped for better cache usage * * RxD fifo is smaller then RxF fifo by design. Upon high load, RxD will be * filled and packets will be dropped by the NIC without getting into the host * or generating interrupts. In this situation the host has no chance of * processing all the packets. Dropping packets by the NIC is cheaper, since it * takes 0 CPU cycles. * * Return: 0 on success and negative value on error. */ static int tn40_create_rx_ring(struct tn40_priv *priv) { … } static void tn40_rx_free_buffers(struct tn40_priv *priv) { … } static void tn40_destroy_rx_ring(struct tn40_priv *priv) { … } static void tn40_set_rx_desc(struct tn40_priv *priv, int idx, u64 dma) { … } /** * tn40_rx_alloc_buffers - Fill rxf fifo with buffers. * * @priv: NIC's private structure * * rx_alloc_buffers allocates buffers via the page pool API, builds rxf descs * and pushes them (rxf descr) into the rxf fifo. The pages are stored in rxdb. * To calculate the free space, we uses the cached values of RPTR and WPTR * when needed. This function also updates RPTR and WPTR. */ static void tn40_rx_alloc_buffers(struct tn40_priv *priv) { … } static void tn40_recycle_rx_buffer(struct tn40_priv *priv, struct tn40_rxd_desc *rxdd) { … } static int tn40_rx_receive(struct tn40_priv *priv, int budget) { … } /* TX HW/SW interaction overview * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * There are 2 types of TX communication channels between driver and NIC. * 1) TX Free Fifo - TXF - Holds ack descriptors for sent packets. * 2) TX Data Fifo - TXD - Holds descriptors of full buffers. * * Currently the NIC supports TSO, checksumming and gather DMA * UFO and IP fragmentation is on the way. * * RX SW Data Structures * ~~~~~~~~~~~~~~~~~~~~~ * TXDB is used to keep track of all skbs owned by SW and their DMA addresses. * For TX case, ownership lasts from getting the packet via hard_xmit and * until the HW acknowledges sending the packet by TXF descriptors. * TXDB is implemented as a cyclic buffer. * * FIFO objects keep info about the fifo's size and location, relevant HW * registers, usage and skb db. Each RXD and RXF fifo has their own fifo * structure. Implemented as simple struct. * * TX SW Execution Flow * ~~~~~~~~~~~~~~~~~~~~ * OS calls the driver's hard_xmit method with a packet to send. The driver * creates DMA mappings, builds TXD descriptors and kicks the HW by updating * TXD WPTR. * * When a packet is sent, The HW write a TXF descriptor and the SW * frees the original skb. To prevent TXD fifo overflow without * reading HW registers every time, the SW deploys "tx level" * technique. Upon startup, the tx level is initialized to TXD fifo * length. For every sent packet, the SW gets its TXD descriptor size * (from a pre-calculated array) and subtracts it from tx level. The * size is also stored in txdb. When a TXF ack arrives, the SW fetched * the size of the original TXD descriptor from the txdb and adds it * to the tx level. When the Tx level drops below some predefined * threshold, the driver stops the TX queue. When the TX level rises * above that level, the tx queue is enabled again. * * This technique avoids excessive reading of RPTR and WPTR registers. * As our benchmarks shows, it adds 1.5 Gbit/sec to NIC's throughput. */ static void tn40_do_tx_db_ptr_next(struct tn40_txdb *db, struct tn40_tx_map **pptr) { … } static void tn40_tx_db_inc_rptr(struct tn40_txdb *db) { … } static void tn40_tx_db_inc_wptr(struct tn40_txdb *db) { … } static int tn40_tx_db_init(struct tn40_txdb *d, int sz_type) { … } static void tn40_tx_db_close(struct tn40_txdb *d) { … } /* Sizes of tx desc (including padding if needed) as function of the SKB's * frag number * 7 - is number of lwords in txd with one phys buffer * 3 - is number of lwords used for every additional phys buffer * for (i = 0; i < TN40_MAX_PBL; i++) { * lwords = 7 + (i * 3); * if (lwords & 1) * lwords++; pad it with 1 lword * tn40_txd_sizes[i].bytes = lwords << 2; * tn40_txd_sizes[i].qwords = lwords >> 1; * } */ static struct { … } tn40_txd_sizes[] = …; static void tn40_pbl_set(struct tn40_pbl *pbl, dma_addr_t dma, int len) { … } static void tn40_txdb_set(struct tn40_txdb *db, dma_addr_t dma, int len) { … } struct tn40_mapping_info { … }; /** * tn40_tx_map_skb - create and store DMA mappings for skb's data blocks * @priv: NIC private structure * @skb: socket buffer to map * @txdd: pointer to tx descriptor to be updated * @pkt_len: pointer to unsigned long value * * This function creates DMA mappings for skb's data blocks and writes them to * PBL of a new tx descriptor. It also stores them in the tx db, so they could * be unmapped after the data has been sent. It is the responsibility of the * caller to make sure that there is enough space in the txdb. The last * element holds a pointer to skb itself and is marked with a zero length. * * Return: 0 on success and negative value on error. */ static int tn40_tx_map_skb(struct tn40_priv *priv, struct sk_buff *skb, struct tn40_txd_desc *txdd, unsigned int *pkt_len) { … } static int tn40_create_tx_ring(struct tn40_priv *priv) { … } /** * tn40_tx_space - Calculate the available space in the TX fifo. * @priv: NIC private structure * * Return: available space in TX fifo in bytes */ static int tn40_tx_space(struct tn40_priv *priv) { … } #define TN40_TXD_FULL_CHECKSUM … static netdev_tx_t tn40_start_xmit(struct sk_buff *skb, struct net_device *ndev) { … } static void tn40_tx_cleanup(struct tn40_priv *priv) { … } static void tn40_tx_free_skbs(struct tn40_priv *priv) { … } static void tn40_destroy_tx_ring(struct tn40_priv *priv) { … } /** * tn40_tx_push_desc - Push a descriptor to TxD fifo. * * @priv: NIC private structure * @data: desc's data * @size: desc's size * * This function pushes desc to TxD fifo and overlaps it if needed. * * This function does not check for available space, nor does it check * that the data size is smaller than the fifo size. Checking for * space is the responsibility of the caller. */ static void tn40_tx_push_desc(struct tn40_priv *priv, void *data, int size) { … } /** * tn40_tx_push_desc_safe - push descriptor to TxD fifo in a safe way. * * @priv: NIC private structure * @data: descriptor data * @size: descriptor size * * This function does check for available space and, if necessary, * waits for the NIC to read existing data before writing new data. */ static void tn40_tx_push_desc_safe(struct tn40_priv *priv, void *data, int size) { … } int tn40_set_link_speed(struct tn40_priv *priv, u32 speed) { … } static void tn40_link_changed(struct tn40_priv *priv) { … } static void tn40_isr_extra(struct tn40_priv *priv, u32 isr) { … } static irqreturn_t tn40_isr_napi(int irq, void *dev) { … } static int tn40_poll(struct napi_struct *napi, int budget) { … } static int tn40_fw_load(struct tn40_priv *priv) { … } static void tn40_restore_mac(struct net_device *ndev, struct tn40_priv *priv) { … } static void tn40_hw_start(struct tn40_priv *priv) { … } static int tn40_hw_reset(struct tn40_priv *priv) { … } static void tn40_sw_reset(struct tn40_priv *priv) { … } static int tn40_start(struct tn40_priv *priv) { … } static void tn40_stop(struct tn40_priv *priv) { … } static int tn40_close(struct net_device *ndev) { … } static int tn40_open(struct net_device *dev) { … } static void __tn40_vlan_rx_vid(struct net_device *ndev, uint16_t vid, int enable) { … } static int tn40_vlan_rx_add_vid(struct net_device *ndev, __always_unused __be16 proto, u16 vid) { … } static int tn40_vlan_rx_kill_vid(struct net_device *ndev, __always_unused __be16 proto, u16 vid) { … } static void tn40_setmulti(struct net_device *ndev) { … } static int tn40_set_mac(struct net_device *ndev, void *p) { … } static void tn40_mac_init(struct tn40_priv *priv) { … } static void tn40_get_stats(struct net_device *ndev, struct rtnl_link_stats64 *stats) { … } static const struct net_device_ops tn40_netdev_ops = …; static int tn40_ethtool_get_link_ksettings(struct net_device *ndev, struct ethtool_link_ksettings *cmd) { … } static const struct ethtool_ops tn40_ethtool_ops = …; static void tn40_get_queue_stats_rx(struct net_device *ndev, int idx, struct netdev_queue_stats_rx *stats) { … } static void tn40_get_queue_stats_tx(struct net_device *ndev, int idx, struct netdev_queue_stats_tx *stats) { … } static void tn40_get_base_stats(struct net_device *ndev, struct netdev_queue_stats_rx *rx, struct netdev_queue_stats_tx *tx) { … } static const struct netdev_stat_ops tn40_stat_ops = …; static int tn40_priv_init(struct tn40_priv *priv) { … } static struct net_device *tn40_netdev_alloc(struct pci_dev *pdev) { … } static int tn40_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { … } static void tn40_remove(struct pci_dev *pdev) { … } static const struct pci_device_id tn40_id_table[] = …; static struct pci_driver tn40_driver = …; module_pci_driver(…) …; MODULE_DEVICE_TABLE(pci, tn40_id_table); MODULE_LICENSE(…) …; MODULE_FIRMWARE(…); MODULE_DESCRIPTION(…) …;
Codebase Browser
linux