// SPDX-License-Identifier: GPL-2.0-only /***************************************************************************** * * * File: sge.c * * $Revision: 1.26 $ * * $Date: 2005/06/21 18:29:48 $ * * Description: * * DMA engine. * * part of the Chelsio 10Gb Ethernet Driver. * * * * * * http://www.chelsio.com * * * * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * * All rights reserved. * * * * Maintainers: [email protected] * * * * Authors: Dimitrios Michailidis <[email protected]> * * Tina Yang <[email protected]> * * Felix Marti <[email protected]> * * Scott Bardone <[email protected]> * * Kurt Ottaway <[email protected]> * * Frank DiMambro <[email protected]> * * * * History: * * * ****************************************************************************/ #include "common.h" #include <linux/types.h> #include <linux/errno.h> #include <linux/pci.h> #include <linux/ktime.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/if_vlan.h> #include <linux/skbuff.h> #include <linux/mm.h> #include <linux/tcp.h> #include <linux/ip.h> #include <linux/in.h> #include <linux/if_arp.h> #include <linux/slab.h> #include <linux/prefetch.h> #include "cpl5_cmd.h" #include "sge.h" #include "regs.h" #include "espi.h" /* This belongs in if_ether.h */ #define ETH_P_CPL5 … #define SGE_CMDQ_N … #define SGE_FREELQ_N … #define SGE_CMDQ0_E_N … #define SGE_CMDQ1_E_N … #define SGE_FREEL_SIZE … #define SGE_JUMBO_FREEL_SIZE … #define SGE_FREEL_REFILL_THRESH … #define SGE_RESPQ_E_N … #define SGE_INTRTIMER_NRES … #define SGE_RX_SM_BUF_SIZE … #define SGE_TX_DESC_MAX_PLEN … #define SGE_RESPQ_REPLENISH_THRES … /* * Period of the TX buffer reclaim timer. This timer does not need to run * frequently as TX buffers are usually reclaimed by new TX packets. */ #define TX_RECLAIM_PERIOD … #define M_CMD_LEN … #define V_CMD_LEN(v) … #define G_CMD_LEN(v) … #define V_CMD_GEN1(v) … #define V_CMD_GEN2(v) … #define F_CMD_DATAVALID … #define F_CMD_SOP … #define V_CMD_EOP(v) … /* * Command queue, receive buffer list, and response queue descriptors. */ #if defined(__BIG_ENDIAN_BITFIELD) struct cmdQ_e { u32 addr_lo; u32 len_gen; u32 flags; u32 addr_hi; }; struct freelQ_e { u32 addr_lo; u32 len_gen; u32 gen2; u32 addr_hi; }; struct respQ_e { u32 Qsleeping : 4; u32 Cmdq1CreditReturn : 5; u32 Cmdq1DmaComplete : 5; u32 Cmdq0CreditReturn : 5; u32 Cmdq0DmaComplete : 5; u32 FreelistQid : 2; u32 CreditValid : 1; u32 DataValid : 1; u32 Offload : 1; u32 Eop : 1; u32 Sop : 1; u32 GenerationBit : 1; u32 BufferLength; }; #elif defined(__LITTLE_ENDIAN_BITFIELD) struct cmdQ_e { … }; struct freelQ_e { … }; struct respQ_e { … } ; #endif /* * SW Context Command and Freelist Queue Descriptors */ struct cmdQ_ce { … }; struct freelQ_ce { … }; /* * SW command, freelist and response rings */ struct cmdQ { … }; struct freelQ { … }; struct respQ { … }; /* Bit flags for cmdQ.status */ enum { … }; /* T204 TX SW scheduler */ /* Per T204 TX port */ struct sched_port { … }; /* Per T204 device */ struct sched { … }; static void restart_sched(struct tasklet_struct *t); /* * Main SGE data structure * * Interrupts are handled by a single CPU and it is likely that on a MP system * the application is migrated to another CPU. In that scenario, we try to * separate the RX(in irq context) and TX state in order to decrease memory * contention. */ struct sge { … }; static const u8 ch_mac_addr[ETH_ALEN] = …; /* * stop tasklet and free all pending skb's */ static void tx_sched_stop(struct sge *sge) { … } /* * t1_sched_update_parms() is called when the MTU or link speed changes. It * re-computes scheduler parameters to scope with the change. */ unsigned int t1_sched_update_parms(struct sge *sge, unsigned int port, unsigned int mtu, unsigned int speed) { … } #if 0 /* * t1_sched_max_avail_bytes() tells the scheduler the maximum amount of * data that can be pushed per port. */ void t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val) { struct sched *s = sge->tx_sched; unsigned int i; s->max_avail = val; for (i = 0; i < MAX_NPORTS; i++) t1_sched_update_parms(sge, i, 0, 0); } /* * t1_sched_set_drain_bits_per_us() tells the scheduler at which rate a port * is draining. */ void t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port, unsigned int val) { struct sched *s = sge->tx_sched; struct sched_port *p = &s->p[port]; p->drain_bits_per_1024ns = val * 1024 / 1000; t1_sched_update_parms(sge, port, 0, 0); } #endif /* 0 */ /* * tx_sched_init() allocates resources and does basic initialization. */ static int tx_sched_init(struct sge *sge) { … } /* * sched_update_avail() computes the delta since the last time it was called * and updates the per port quota (number of bits that can be sent to the any * port). */ static inline int sched_update_avail(struct sge *sge) { … } /* * sched_skb() is called from two different places. In the tx path, any * packet generating load on an output port will call sched_skb() * (skb != NULL). In addition, sched_skb() is called from the irq/soft irq * context (skb == NULL). * The scheduler only returns a skb (which will then be sent) if the * length of the skb is <= the current quota of the output port. */ static struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb, unsigned int credits) { … } /* * PIO to indicate that memory mapped Q contains valid descriptor(s). */ static inline void doorbell_pio(struct adapter *adapter, u32 val) { … } /* * Frees all RX buffers on the freelist Q. The caller must make sure that * the SGE is turned off before calling this function. */ static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q) { … } /* * Free RX free list and response queue resources. */ static void free_rx_resources(struct sge *sge) { … } /* * Allocates basic RX resources, consisting of memory mapped freelist Qs and a * response queue. */ static int alloc_rx_resources(struct sge *sge, struct sge_params *p) { … } /* * Reclaims n TX descriptors and frees the buffers associated with them. */ static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n) { … } /* * Free TX resources. * * Assumes that SGE is stopped and all interrupts are disabled. */ static void free_tx_resources(struct sge *sge) { … } /* * Allocates basic TX resources, consisting of memory mapped command Qs. */ static int alloc_tx_resources(struct sge *sge, struct sge_params *p) { … } static inline void setup_ring_params(struct adapter *adapter, u64 addr, u32 size, int base_reg_lo, int base_reg_hi, int size_reg) { … } /* * Enable/disable VLAN acceleration. */ void t1_vlan_mode(struct adapter *adapter, netdev_features_t features) { … } /* * Programs the various SGE registers. However, the engine is not yet enabled, * but sge->sge_control is setup and ready to go. */ static void configure_sge(struct sge *sge, struct sge_params *p) { … } /* * Return the payload capacity of the jumbo free-list buffers. */ static inline unsigned int jumbo_payload_capacity(const struct sge *sge) { … } /* * Frees all SGE related resources and the sge structure itself */ void t1_sge_destroy(struct sge *sge) { … } /* * Allocates new RX buffers on the freelist Q (and tracks them on the freelist * context Q) until the Q is full or alloc_skb fails. * * It is possible that the generation bits already match, indicating that the * buffer is already valid and nothing needs to be done. This happens when we * copied a received buffer into a new sk_buff during the interrupt processing. * * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad), * we specify a RX_OFFSET in order to make sure that the IP header is 4B * aligned. */ static void refill_free_list(struct sge *sge, struct freelQ *q) { … } /* * Calls refill_free_list for both free lists. If we cannot fill at least 1/4 * of both rings, we go into 'few interrupt mode' in order to give the system * time to free up resources. */ static void freelQs_empty(struct sge *sge) { … } #define SGE_PL_INTR_MASK … #define SGE_INT_FATAL … #define SGE_INT_ENABLE … /* * Disable SGE Interrupts */ void t1_sge_intr_disable(struct sge *sge) { … } /* * Enable SGE interrupts. */ void t1_sge_intr_enable(struct sge *sge) { … } /* * Clear SGE interrupts. */ void t1_sge_intr_clear(struct sge *sge) { … } /* * SGE 'Error' interrupt handler */ bool t1_sge_intr_error_handler(struct sge *sge) { … } const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge) { … } void t1_sge_get_port_stats(const struct sge *sge, int port, struct sge_port_stats *ss) { … } /** * recycle_fl_buf - recycle a free list buffer * @fl: the free list * @idx: index of buffer to recycle * * Recycles the specified buffer on the given free list by adding it at * the next available slot on the list. */ static void recycle_fl_buf(struct freelQ *fl, int idx) { … } static int copybreak __read_mostly = …; module_param(copybreak, int, 0); MODULE_PARM_DESC(…) …; /** * get_packet - return the next ingress packet buffer * @adapter: the adapter that received the packet * @fl: the SGE free list holding the packet * @len: the actual packet length, excluding any SGE padding * * Get the next packet from a free list and complete setup of the * sk_buff. If the packet is small we make a copy and recycle the * original buffer, otherwise we use the original buffer itself. If a * positive drop threshold is supplied packets are dropped and their * buffers recycled if (a) the number of remaining buffers is under the * threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ static inline struct sk_buff *get_packet(struct adapter *adapter, struct freelQ *fl, unsigned int len) { … } /** * unexpected_offload - handle an unexpected offload packet * @adapter: the adapter * @fl: the free list that received the packet * * Called when we receive an unexpected offload packet (e.g., the TOE * function is disabled or the card is a NIC). Prints a message and * recycles the buffer. */ static void unexpected_offload(struct adapter *adapter, struct freelQ *fl) { … } /* * T1/T2 SGE limits the maximum DMA size per TX descriptor to * SGE_TX_DESC_MAX_PLEN (16KB). If the PAGE_SIZE is larger than 16KB, the * stack might send more than SGE_TX_DESC_MAX_PLEN in a contiguous manner. * Note that the *_large_page_tx_descs stuff will be optimized out when * PAGE_SIZE <= SGE_TX_DESC_MAX_PLEN. * * compute_large_page_descs() computes how many additional descriptors are * required to break down the stack's request. */ static inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb) { … } /* * Write a cmdQ entry. * * Since this function writes the 'flags' field, it must not be used to * write the first cmdQ entry. */ static inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping, unsigned int len, unsigned int gen, unsigned int eop) { … } /* * See comment for previous function. * * write_tx_descs_large_page() writes additional SGE tx descriptors if * *desc_len exceeds HW's capability. */ static inline unsigned int write_large_page_tx_descs(unsigned int pidx, struct cmdQ_e **e, struct cmdQ_ce **ce, unsigned int *gen, dma_addr_t *desc_mapping, unsigned int *desc_len, unsigned int nfrags, struct cmdQ *q) { … } /* * Write the command descriptors to transmit the given skb starting at * descriptor pidx with the given generation. */ static inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb, unsigned int pidx, unsigned int gen, struct cmdQ *q) { … } /* * Clean up completed Tx buffers. */ static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q) { … } /* * Called from tasklet. Checks the scheduler for any * pending skbs that can be sent. */ static void restart_sched(struct tasklet_struct *t) { … } /** * sge_rx - process an ingress ethernet packet * @sge: the sge structure * @fl: the free list that contains the packet buffer * @len: the packet length * * Process an ingress ethernet packet and deliver it to the stack. */ static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len) { … } /* * Returns true if a command queue has enough available descriptors that * we can resume Tx operation after temporarily disabling its packet queue. */ static inline int enough_free_Tx_descs(const struct cmdQ *q) { … } /* * Called when sufficient space has become available in the SGE command queues * after the Tx packet schedulers have been suspended to restart the Tx path. */ static void restart_tx_queues(struct sge *sge) { … } /* * update_tx_info is called from the interrupt handler/NAPI to return cmdQ0 * information. */ static unsigned int update_tx_info(struct adapter *adapter, unsigned int flags, unsigned int pr0) { … } /* * Process SGE responses, up to the supplied budget. Returns the number of * responses processed. A negative budget is effectively unlimited. */ static int process_responses(struct adapter *adapter, int budget) { … } static inline int responses_pending(const struct adapter *adapter) { … } /* * A simpler version of process_responses() that handles only pure (i.e., * non data-carrying) responses. Such respones are too light-weight to justify * calling a softirq when using NAPI, so we handle them specially in hard * interrupt context. The function is called with a pointer to a response, * which the caller must ensure is a valid pure response. Returns 1 if it * encounters a valid data-carrying response, 0 otherwise. */ static int process_pure_responses(struct adapter *adapter) { … } /* * Handler for new data events when using NAPI. This does not need any locking * or protection from interrupts as data interrupts are off at this point and * other adapter interrupts do not interfere. */ int t1_poll(struct napi_struct *napi, int budget) { … } irqreturn_t t1_interrupt_thread(int irq, void *data) { … } irqreturn_t t1_interrupt(int irq, void *data) { … } /* * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it. * * The code figures out how many entries the sk_buff will require in the * cmdQ and updates the cmdQ data structure with the state once the enqueue * has complete. Then, it doesn't access the global structure anymore, but * uses the corresponding fields on the stack. In conjunction with a spinlock * around that code, we can make the function reentrant without holding the * lock when we actually enqueue (which might be expensive, especially on * architectures with IO MMUs). * * This runs with softirqs disabled. */ static int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter, unsigned int qid, struct net_device *dev) { … } #define MK_ETH_TYPE_MSS(type, mss) … /* * eth_hdr_len - return the length of an Ethernet header * @data: pointer to the start of the Ethernet header * * Returns the length of an Ethernet header, including optional VLAN tag. */ static inline int eth_hdr_len(const void *data) { … } /* * Adds the CPL header to the sk_buff and passes it to t1_sge_tx. */ netdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev) { … } /* * Callback for the Tx buffer reclaim timer. Runs with softirqs disabled. */ static void sge_tx_reclaim_cb(struct timer_list *t) { … } /* * Propagate changes of the SGE coalescing parameters to the HW. */ int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p) { … } /* * Allocates both RX and TX resources and configures the SGE. However, * the hardware is not enabled yet. */ int t1_sge_configure(struct sge *sge, struct sge_params *p) { … } /* * Disables the DMA engine. */ void t1_sge_stop(struct sge *sge) { … } /* * Enables the DMA engine. */ void t1_sge_start(struct sge *sge) { … } /* * Callback for the T2 ESPI 'stuck packet feature' workaorund */ static void espibug_workaround_t204(struct timer_list *t) { … } static void espibug_workaround(struct timer_list *t) { … } /* * Creates a t1_sge structure and returns suggested resource parameters. */ struct sge *t1_sge_create(struct adapter *adapter, struct sge_params *p) { … }
Codebase Browser
linux