/* * This file is part of the Chelsio T4 Ethernet driver for Linux. * * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include <linux/skbuff.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/dma-mapping.h> #include <linux/jiffies.h> #include <linux/prefetch.h> #include <linux/export.h> #include <net/xfrm.h> #include <net/ipv6.h> #include <net/tcp.h> #include <net/busy_poll.h> #ifdef CONFIG_CHELSIO_T4_FCOE #include <scsi/fc/fc_fcoe.h> #endif /* CONFIG_CHELSIO_T4_FCOE */ #include "cxgb4.h" #include "t4_regs.h" #include "t4_values.h" #include "t4_msg.h" #include "t4fw_api.h" #include "cxgb4_ptp.h" #include "cxgb4_uld.h" #include "cxgb4_tc_mqprio.h" #include "sched.h" /* * Rx buffer size. We use largish buffers if possible but settle for single * pages under memory shortage. */ #if PAGE_SHIFT >= 16 #define FL_PG_ORDER … #else #define FL_PG_ORDER … #endif /* RX_PULL_LEN should be <= RX_COPY_THRES */ #define RX_COPY_THRES … #define RX_PULL_LEN … /* * Main body length for sk_buffs used for Rx Ethernet packets with fragments. * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room. */ #define RX_PKT_SKB_LEN … /* * Max number of Tx descriptors we clean up at a time. Should be modest as * freeing skbs isn't cheap and it happens while holding locks. We just need * to free packets faster than they arrive, we eventually catch up and keep * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should * also match the CIDX Flush Threshold. */ #define MAX_TX_RECLAIM … /* * Max number of Rx buffers we replenish at a time. Again keep this modest, * allocating buffers isn't cheap either. */ #define MAX_RX_REFILL … /* * Period of the Rx queue check timer. This timer is infrequent as it has * something to do only when the system experiences severe memory shortage. */ #define RX_QCHECK_PERIOD … /* * Period of the Tx queue check timer. */ #define TX_QCHECK_PERIOD … /* * Max number of Tx descriptors to be reclaimed by the Tx timer. */ #define MAX_TIMER_TX_RECLAIM … /* * Timer index used when backing off due to memory shortage. */ #define NOMEM_TMR_IDX … /* * Suspension threshold for non-Ethernet Tx queues. We require enough room * for a full sized WR. */ #define TXQ_STOP_THRES … /* * Max Tx descriptor space we allow for an Ethernet packet to be inlined * into a WR. */ #define MAX_IMM_TX_PKT_LEN … /* * Max size of a WR sent through a control Tx queue. */ #define MAX_CTRL_WR_LEN … struct rx_sw_desc { … }; /* * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs. * We could easily support more but there doesn't seem to be much need for * that ... */ #define FL_MTU_SMALL … #define FL_MTU_LARGE … static inline unsigned int fl_mtu_bufsize(struct adapter *adapter, unsigned int mtu) { … } #define FL_MTU_SMALL_BUFSIZE(adapter) … #define FL_MTU_LARGE_BUFSIZE(adapter) … /* * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses * these to specify the buffer size as an index into the SGE Free List Buffer * Size register array. We also use bit 4, when the buffer has been unmapped * for DMA, but this is of course never sent to the hardware and is only used * to prevent double unmappings. All of the above requires that the Free List * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal * Free List Buffer alignment is 32 bytes, this works out for us ... */ enum { … }; static int timer_pkt_quota[] = …; #define MIN_NAPI_WORK … static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d) { … } static inline bool is_buf_mapped(const struct rx_sw_desc *d) { … } /** * txq_avail - return the number of available slots in a Tx queue * @q: the Tx queue * * Returns the number of descriptors in a Tx queue available to write new * packets. */ static inline unsigned int txq_avail(const struct sge_txq *q) { … } /** * fl_cap - return the capacity of a free-buffer list * @fl: the FL * * Returns the capacity of a free-buffer list. The capacity is less than * the size because one descriptor needs to be left unpopulated, otherwise * HW will think the FL is empty. */ static inline unsigned int fl_cap(const struct sge_fl *fl) { … } /** * fl_starving - return whether a Free List is starving. * @adapter: pointer to the adapter * @fl: the Free List * * Tests specified Free List to see whether the number of buffers * available to the hardware has falled below our "starvation" * threshold. */ static inline bool fl_starving(const struct adapter *adapter, const struct sge_fl *fl) { … } int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, dma_addr_t *addr) { … } EXPORT_SYMBOL(…); static void unmap_skb(struct device *dev, const struct sk_buff *skb, const dma_addr_t *addr) { … } #ifdef CONFIG_NEED_DMA_MAP_STATE /** * deferred_unmap_destructor - unmap a packet when it is freed * @skb: the packet * * This is the packet destructor used for Tx packets that need to remain * mapped until they are freed rather than until their Tx descriptors are * freed. */ static void deferred_unmap_destructor(struct sk_buff *skb) { … } #endif /** * free_tx_desc - reclaims Tx descriptors and their buffers * @adap: the adapter * @q: the Tx queue to reclaim descriptors from * @n: the number of descriptors to reclaim * @unmap: whether the buffers should be unmapped for DMA * * Reclaims Tx descriptors from an SGE Tx queue and frees the associated * Tx buffers. Called with the Tx queue lock held. */ void free_tx_desc(struct adapter *adap, struct sge_txq *q, unsigned int n, bool unmap) { … } /* * Return the number of reclaimable descriptors in a Tx queue. */ static inline int reclaimable(const struct sge_txq *q) { … } /** * reclaim_completed_tx - reclaims completed TX Descriptors * @adap: the adapter * @q: the Tx queue to reclaim completed descriptors from * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1 * @unmap: whether the buffers should be unmapped for DMA * * Reclaims Tx Descriptors that the SGE has indicated it has processed, * and frees the associated buffers if possible. If @max == -1, then * we'll use a defaiult maximum. Called with the TX Queue locked. */ static inline int reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, int maxreclaim, bool unmap) { … } /** * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors * @adap: the adapter * @q: the Tx queue to reclaim completed descriptors from * @unmap: whether the buffers should be unmapped for DMA * * Reclaims Tx descriptors that the SGE has indicated it has processed, * and frees the associated buffers if possible. Called with the Tx * queue locked. */ void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, bool unmap) { … } EXPORT_SYMBOL(…); static inline int get_buf_size(struct adapter *adapter, const struct rx_sw_desc *d) { … } /** * free_rx_bufs - free the Rx buffers on an SGE free list * @adap: the adapter * @q: the SGE free list to free buffers from * @n: how many buffers to free * * Release the next @n buffers on an SGE free-buffer Rx queue. The * buffers must be made inaccessible to HW before calling this function. */ static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n) { … } /** * unmap_rx_buf - unmap the current Rx buffer on an SGE free list * @adap: the adapter * @q: the SGE free list * * Unmap the current buffer on an SGE free-buffer Rx queue. The * buffer must be made inaccessible to HW before calling this function. * * This is similar to @free_rx_bufs above but does not free the buffer. * Do note that the FL still loses any further access to the buffer. */ static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q) { … } static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) { … } static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg, dma_addr_t mapping) { … } /** * refill_fl - refill an SGE Rx buffer ring * @adap: the adapter * @q: the ring to refill * @n: the number of new buffers to allocate * @gfp: the gfp flags for the allocations * * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, * allocated with the supplied gfp flags. The caller must assure that * @n does not exceed the queue's capacity. If afterwards the queue is * found critically low mark it as starving in the bitmap of starving FLs. * * Returns the number of buffers allocated. */ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp) { … } static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) { … } /** * alloc_ring - allocate resources for an SGE descriptor ring * @dev: the PCI device's core device * @nelem: the number of descriptors * @elem_size: the size of each descriptor * @sw_size: the size of the SW state associated with each ring element * @phys: the physical address of the allocated ring * @metadata: address of the array holding the SW state for the ring * @stat_size: extra space in HW ring for status information * @node: preferred node for memory allocations * * Allocates resources for an SGE descriptor ring, such as Tx queues, * free buffer lists, or response queues. Each SGE ring requires * space for its HW descriptors plus, optionally, space for the SW state * associated with each HW entry (the metadata). The function returns * three values: the virtual address for the HW ring (the return value * of the function), the bus address of the HW ring, and the address * of the SW ring. */ static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size, size_t sw_size, dma_addr_t *phys, void *metadata, size_t stat_size, int node) { … } /** * sgl_len - calculates the size of an SGL of the given capacity * @n: the number of SGL entries * * Calculates the number of flits needed for a scatter/gather list that * can hold the given number of entries. */ static inline unsigned int sgl_len(unsigned int n) { … } /** * flits_to_desc - returns the num of Tx descriptors for the given flits * @n: the number of flits * * Returns the number of Tx descriptors needed for the supplied number * of flits. */ static inline unsigned int flits_to_desc(unsigned int n) { … } /** * is_eth_imm - can an Ethernet packet be sent as immediate data? * @skb: the packet * @chip_ver: chip version * * Returns whether an Ethernet packet is small enough to fit as * immediate data. Return value corresponds to headroom required. */ static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver) { … } /** * calc_tx_flits - calculate the number of flits for a packet Tx WR * @skb: the packet * @chip_ver: chip version * * Returns the number of flits needed for a Tx WR for the given Ethernet * packet, including the needed WR and CPL headers. */ static inline unsigned int calc_tx_flits(const struct sk_buff *skb, unsigned int chip_ver) { … } /** * cxgb4_write_sgl - populate a scatter/gather list for a packet * @skb: the packet * @q: the Tx queue we are writing into * @sgl: starting location for writing the SGL * @end: points right after the end of the SGL * @start: start offset into skb main-body data to include in the SGL * @addr: the list of bus addresses for the SGL elements * * Generates a gather list for the buffers that make up a packet. * The caller must provide adequate space for the SGL that will be written. * The SGL includes all of the packet's page fragments and the data in its * main body except for the first @start bytes. @sgl must be 16-byte * aligned and within a Tx descriptor with available space. @end points * right after the end of the SGL but does not account for any potential * wrap around, i.e., @end > @sgl. */ void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, struct ulptx_sgl *sgl, u64 *end, unsigned int start, const dma_addr_t *addr) { … } EXPORT_SYMBOL(…); /* cxgb4_write_partial_sgl - populate SGL for partial packet * @skb: the packet * @q: the Tx queue we are writing into * @sgl: starting location for writing the SGL * @end: points right after the end of the SGL * @addr: the list of bus addresses for the SGL elements * @start: start offset in the SKB where partial data starts * @len: length of data from @start to send out * * This API will handle sending out partial data of a skb if required. * Unlike cxgb4_write_sgl, @start can be any offset into the skb data, * and @len will decide how much data after @start offset to send out. */ void cxgb4_write_partial_sgl(const struct sk_buff *skb, struct sge_txq *q, struct ulptx_sgl *sgl, u64 *end, const dma_addr_t *addr, u32 start, u32 len) { … } EXPORT_SYMBOL(…); /* This function copies 64 byte coalesced work request to * memory mapped BAR2 space. For coalesced WR SGE fetches * data from the FIFO instead of from Host. */ static void cxgb_pio_copy(u64 __iomem *dst, u64 *src) { … } /** * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell * @adap: the adapter * @q: the Tx queue * @n: number of new descriptors to give to HW * * Ring the doorbel for a Tx queue. */ inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) { … } EXPORT_SYMBOL(…); /** * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors * @skb: the packet * @q: the Tx queue where the packet will be inlined * @pos: starting position in the Tx queue where to inline the packet * * Inline a packet's contents directly into Tx descriptors, starting at * the given position within the Tx DMA ring. * Most of the complexity of this operation is dealing with wrap arounds * in the middle of the packet we want to inline. */ void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, void *pos) { … } EXPORT_SYMBOL(…); static void *inline_tx_skb_header(const struct sk_buff *skb, const struct sge_txq *q, void *pos, int length) { … } /* * Figure out what HW csum a packet wants and return the appropriate control * bits. */ static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb) { … } static void eth_txq_stop(struct sge_eth_txq *q) { … } static inline void txq_advance(struct sge_txq *q, unsigned int n) { … } #ifdef CONFIG_CHELSIO_T4_FCOE static inline int cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap, const struct port_info *pi, u64 *cntrl) { … } #endif /* CONFIG_CHELSIO_T4_FCOE */ /* Returns tunnel type if hardware supports offloading of the same. * It is called only for T5 and onwards. */ enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb) { … } static inline void t6_fill_tnl_lso(struct sk_buff *skb, struct cpl_tx_tnl_lso *tnl_lso, enum cpl_tx_tnl_lso_type tnl_type) { … } static inline void *write_tso_wr(struct adapter *adap, struct sk_buff *skb, struct cpl_tx_pkt_lso_core *lso) { … } /** * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update * @adap: the adapter * @eq: the Ethernet TX Queue * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1 * * We're typically called here to update the state of an Ethernet TX * Queue with respect to the hardware's progress in consuming the TX * Work Requests that we've put on that Egress Queue. This happens * when we get Egress Queue Update messages and also prophylactically * in regular timer-based Ethernet TX Queue maintenance. */ int t4_sge_eth_txq_egress_update(struct adapter *adap, struct sge_eth_txq *eq, int maxreclaim) { … } static inline int cxgb4_validate_skb(struct sk_buff *skb, struct net_device *dev, u32 min_pkt_len) { … } static void *write_eo_udp_wr(struct sk_buff *skb, struct fw_eth_tx_eo_wr *wr, u32 hdr_len) { … } /** * cxgb4_eth_xmit - add a packet to an Ethernet Tx queue * @skb: the packet * @dev: the egress net device * * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. */ static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev) { … } /* Constants ... */ enum { … }; /** * t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data? * @skb: the packet * * Returns whether an Ethernet packet is small enough to fit completely as * immediate data. */ static inline int t4vf_is_eth_imm(const struct sk_buff *skb) { … } /** * t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR * @skb: the packet * * Returns the number of flits needed for a TX Work Request for the * given Ethernet packet, including the needed WR and CPL headers. */ static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff *skb) { … } /** * cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue * @skb: the packet * @dev: the egress net device * * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled. */ static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb, struct net_device *dev) { … } /** * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs * @q: the SGE control Tx queue * * This is a variant of cxgb4_reclaim_completed_tx() that is used * for Tx queues that send only immediate data (presently just * the control queues) and thus do not have any sk_buffs to release. */ static inline void reclaim_completed_tx_imm(struct sge_txq *q) { … } static inline void eosw_txq_advance_index(u32 *idx, u32 n, u32 max) { … } void cxgb4_eosw_txq_free_desc(struct adapter *adap, struct sge_eosw_txq *eosw_txq, u32 ndesc) { … } static inline void eosw_txq_advance(struct sge_eosw_txq *eosw_txq, u32 n) { … } static inline int eosw_txq_enqueue(struct sge_eosw_txq *eosw_txq, struct sk_buff *skb) { … } static inline struct sk_buff *eosw_txq_peek(struct sge_eosw_txq *eosw_txq) { … } static inline u8 ethofld_calc_tx_flits(struct adapter *adap, struct sk_buff *skb, u32 hdr_len) { … } static void *write_eo_wr(struct adapter *adap, struct sge_eosw_txq *eosw_txq, struct sk_buff *skb, struct fw_eth_tx_eo_wr *wr, u32 hdr_len, u32 wrlen) { … } static int ethofld_hard_xmit(struct net_device *dev, struct sge_eosw_txq *eosw_txq) { … } static void ethofld_xmit(struct net_device *dev, struct sge_eosw_txq *eosw_txq) { … } static netdev_tx_t cxgb4_ethofld_xmit(struct sk_buff *skb, struct net_device *dev) { … } netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev) { … } static void eosw_txq_flush_pending_skbs(struct sge_eosw_txq *eosw_txq) { … } /** * cxgb4_ethofld_send_flowc - Send ETHOFLD flowc request to bind eotid to tc. * @dev: netdevice * @eotid: ETHOFLD tid to bind/unbind * @tc: traffic class. If set to FW_SCHED_CLS_NONE, then unbinds the @eotid * * Send a FLOWC work request to bind an ETHOFLD TID to a traffic class. * If @tc is set to FW_SCHED_CLS_NONE, then the @eotid is unbound from * a traffic class. */ int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc) { … } /** * is_imm - check whether a packet can be sent as immediate data * @skb: the packet * * Returns true if a packet can be sent as a WR with immediate data. */ static inline int is_imm(const struct sk_buff *skb) { … } /** * ctrlq_check_stop - check if a control queue is full and should stop * @q: the queue * @wr: most recent WR written to the queue * * Check if a control queue has become full and should be stopped. * We clean up control queue descriptors very lazily, only when we are out. * If the queue is still full after reclaiming any completed descriptors * we suspend it and have the last WR wake it up. */ static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr) { … } #define CXGB4_SELFTEST_LB_STR … int cxgb4_selftest_lb_pkt(struct net_device *netdev) { … } /** * ctrl_xmit - send a packet through an SGE control Tx queue * @q: the control queue * @skb: the packet * * Send a packet through an SGE control Tx queue. Packets sent through * a control queue must fit entirely as immediate data. */ static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb) { … } /** * restart_ctrlq - restart a suspended control queue * @t: pointer to the tasklet associated with this handler * * Resumes transmission on a suspended Tx control queue. */ static void restart_ctrlq(struct tasklet_struct *t) { … } /** * t4_mgmt_tx - send a management message * @adap: the adapter * @skb: the packet containing the management message * * Send a management message through control queue 0. */ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) { … } /** * is_ofld_imm - check whether a packet can be sent as immediate data * @skb: the packet * * Returns true if a packet can be sent as an offload WR with immediate * data. * FW_OFLD_TX_DATA_WR limits the payload to 255 bytes due to 8-bit field. * However, FW_ULPTX_WR commands have a 256 byte immediate only * payload limit. */ static inline int is_ofld_imm(const struct sk_buff *skb) { … } /** * calc_tx_flits_ofld - calculate # of flits for an offload packet * @skb: the packet * * Returns the number of flits needed for the given offload packet. * These packets are already fully constructed and no additional headers * will be added. */ static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) { … } /** * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion * @q: the queue to stop * * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting * inability to map packets. A periodic timer attempts to restart * queues so marked. */ static void txq_stop_maperr(struct sge_uld_txq *q) { … } /** * ofldtxq_stop - stop an offload Tx queue that has become full * @q: the queue to stop * @wr: the Work Request causing the queue to become full * * Stops an offload Tx queue that has become full and modifies the packet * being written to request a wakeup. */ static void ofldtxq_stop(struct sge_uld_txq *q, struct fw_wr_hdr *wr) { … } /** * service_ofldq - service/restart a suspended offload queue * @q: the offload queue * * Services an offload Tx queue by moving packets from its Pending Send * Queue to the Hardware TX ring. The function starts and ends with the * Send Queue locked, but drops the lock while putting the skb at the * head of the Send Queue onto the Hardware TX Ring. Dropping the lock * allows more skbs to be added to the Send Queue by other threads. * The packet being processed at the head of the Pending Send Queue is * left on the queue in case we experience DMA Mapping errors, etc. * and need to give up and restart later. * * service_ofldq() can be thought of as a task which opportunistically * uses other threads execution contexts. We use the Offload Queue * boolean "service_ofldq_running" to make sure that only one instance * is ever running at a time ... */ static void service_ofldq(struct sge_uld_txq *q) __must_hold(&q->sendq.lock) { … } /** * ofld_xmit - send a packet through an offload queue * @q: the Tx offload queue * @skb: the packet * * Send an offload packet through an SGE offload queue. */ static int ofld_xmit(struct sge_uld_txq *q, struct sk_buff *skb) { … } /** * restart_ofldq - restart a suspended offload queue * @t: pointer to the tasklet associated with this handler * * Resumes transmission on a suspended Tx offload queue. */ static void restart_ofldq(struct tasklet_struct *t) { … } /** * skb_txq - return the Tx queue an offload packet should use * @skb: the packet * * Returns the Tx queue an offload packet should use as indicated by bits * 1-15 in the packet's queue_mapping. */ static inline unsigned int skb_txq(const struct sk_buff *skb) { … } /** * is_ctrl_pkt - return whether an offload packet is a control packet * @skb: the packet * * Returns whether an offload packet should use an OFLD or a CTRL * Tx queue as indicated by bit 0 in the packet's queue_mapping. */ static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb) { … } static inline int uld_send(struct adapter *adap, struct sk_buff *skb, unsigned int tx_uld_type) { … } /** * t4_ofld_send - send an offload packet * @adap: the adapter * @skb: the packet * * Sends an offload packet. We use the packet queue_mapping to select the * appropriate Tx queue as follows: bit 0 indicates whether the packet * should be sent as regular or control, bits 1-15 select the queue. */ int t4_ofld_send(struct adapter *adap, struct sk_buff *skb) { … } /** * cxgb4_ofld_send - send an offload packet * @dev: the net device * @skb: the packet * * Sends an offload packet. This is an exported version of @t4_ofld_send, * intended for ULDs. */ int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb) { … } EXPORT_SYMBOL(…); static void *inline_tx_header(const void *src, const struct sge_txq *q, void *pos, int length) { … } /** * ofld_xmit_direct - copy a WR into offload queue * @q: the Tx offload queue * @src: location of WR * @len: WR length * * Copy an immediate WR into an uncontended SGE offload queue. */ static int ofld_xmit_direct(struct sge_uld_txq *q, const void *src, unsigned int len) { … } int cxgb4_immdata_send(struct net_device *dev, unsigned int idx, const void *src, unsigned int len) { … } EXPORT_SYMBOL(…); /** * t4_crypto_send - send crypto packet * @adap: the adapter * @skb: the packet * * Sends crypto packet. We use the packet queue_mapping to select the * appropriate Tx queue as follows: bit 0 indicates whether the packet * should be sent as regular or control, bits 1-15 select the queue. */ static int t4_crypto_send(struct adapter *adap, struct sk_buff *skb) { … } /** * cxgb4_crypto_send - send crypto packet * @dev: the net device * @skb: the packet * * Sends crypto packet. This is an exported version of @t4_crypto_send, * intended for ULDs. */ int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb) { … } EXPORT_SYMBOL(…); static inline void copy_frags(struct sk_buff *skb, const struct pkt_gl *gl, unsigned int offset) { … } /** * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list * @gl: the gather list * @skb_len: size of sk_buff main body if it carries fragments * @pull_len: amount of data to move to the sk_buff's main body * * Builds an sk_buff from the given packet gather list. Returns the * sk_buff or %NULL if sk_buff allocation failed. */ struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl, unsigned int skb_len, unsigned int pull_len) { … } EXPORT_SYMBOL(…); /** * t4_pktgl_free - free a packet gather list * @gl: the gather list * * Releases the pages of a packet gather list. We do not own the last * page on the list and do not free it. */ static void t4_pktgl_free(const struct pkt_gl *gl) { … } /* * Process an MPS trace packet. Give it an unused protocol number so it won't * be delivered to anyone and send it to the stack for capture. */ static noinline int handle_trace_pkt(struct adapter *adap, const struct pkt_gl *gl) { … } /** * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp * @adap: the adapter * @hwtstamps: time stamp structure to update * @sgetstamp: 60bit iqe timestamp * * Every ingress queue entry has the 60-bit timestamp, convert that timestamp * which is in Core Clock ticks into ktime_t and assign it **/ static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap, struct skb_shared_hwtstamps *hwtstamps, u64 sgetstamp) { … } static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl, const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len) { … } enum { … }; /** * t4_systim_to_hwstamp - read hardware time stamp * @adapter: the adapter * @skb: the packet * * Read Time Stamp from MPS packet and insert in skb which * is forwarded to PTP application */ static noinline int t4_systim_to_hwstamp(struct adapter *adapter, struct sk_buff *skb) { … } /** * t4_rx_hststamp - Recv PTP Event Message * @adapter: the adapter * @rsp: the response queue descriptor holding the RX_PKT message * @rxq: the response queue holding the RX_PKT message * @skb: the packet * * PTP enabled and MPS packet, read HW timestamp */ static int t4_rx_hststamp(struct adapter *adapter, const __be64 *rsp, struct sge_eth_rxq *rxq, struct sk_buff *skb) { … } /** * t4_tx_hststamp - Loopback PTP Transmit Event Message * @adapter: the adapter * @skb: the packet * @dev: the ingress net device * * Read hardware timestamp for the loopback PTP Tx event message */ static int t4_tx_hststamp(struct adapter *adapter, struct sk_buff *skb, struct net_device *dev) { … } /** * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue * @rsp: Response Entry pointer into Response Queue * @gl: Gather List pointer * * For adapters which support the SGE Doorbell Queue Timer facility, * we configure the Ethernet TX Queues to send CIDX Updates to the * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE * messages. This adds a small load to PCIe Link RX bandwidth and, * potentially, higher CPU Interrupt load, but allows us to respond * much more quickly to the CIDX Updates. This is important for * Upper Layer Software which isn't willing to have a large amount * of TX Data outstanding before receiving DMA Completions. */ static void t4_tx_completion_handler(struct sge_rspq *rspq, const __be64 *rsp, const struct pkt_gl *gl) { … } static int cxgb4_validate_lb_pkt(struct port_info *pi, const struct pkt_gl *si) { … } /** * t4_ethrx_handler - process an ingress ethernet packet * @q: the response queue that received the packet * @rsp: the response queue descriptor holding the RX_PKT message * @si: the gather list of packet fragments * * Process an ingress ethernet packet and deliver it to the stack. */ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct pkt_gl *si) { … } /** * restore_rx_bufs - put back a packet's Rx buffers * @si: the packet gather list * @q: the SGE free list * @frags: number of FL buffers to restore * * Puts back on an FL the Rx buffers associated with @si. The buffers * have already been unmapped and are left unmapped, we mark them so to * prevent further unmapping attempts. * * This function undoes a series of @unmap_rx_buf calls when we find out * that the current packet can't be processed right away afterall and we * need to come back to it later. This is a very rare event and there's * no effort to make this particularly efficient. */ static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q, int frags) { … } /** * is_new_response - check if a response is newly written * @r: the response descriptor * @q: the response queue * * Returns true if a response descriptor contains a yet unprocessed * response. */ static inline bool is_new_response(const struct rsp_ctrl *r, const struct sge_rspq *q) { … } /** * rspq_next - advance to the next entry in a response queue * @q: the queue * * Updates the state of a response queue to advance it to the next entry. */ static inline void rspq_next(struct sge_rspq *q) { … } /** * process_responses - process responses from an SGE response queue * @q: the ingress queue to process * @budget: how many responses can be processed in this round * * Process responses from an SGE response queue up to the supplied budget. * Responses include received packets as well as control messages from FW * or HW. * * Additionally choose the interrupt holdoff time for the next interrupt * on this queue. If the system is under memory shortage use a fairly * long delay to help recovery. */ static int process_responses(struct sge_rspq *q, int budget) { … } /** * napi_rx_handler - the NAPI handler for Rx processing * @napi: the napi instance * @budget: how many packets we can process in this round * * Handler for new data events when using NAPI. This does not need any * locking or protection from interrupts as data interrupts are off at * this point and other adapter interrupts do not interfere (the latter * in not a concern at all with MSI-X as non-data interrupts then have * a separate handler). */ static int napi_rx_handler(struct napi_struct *napi, int budget) { … } void cxgb4_ethofld_restart(struct tasklet_struct *t) { … } /* cxgb4_ethofld_rx_handler - Process ETHOFLD Tx completions * @q: the response queue that received the packet * @rsp: the response queue descriptor holding the CPL message * @si: the gather list of packet fragments * * Process a ETHOFLD Tx completion. Increment the cidx here, but * free up the descriptors in a tasklet later. */ int cxgb4_ethofld_rx_handler(struct sge_rspq *q, const __be64 *rsp, const struct pkt_gl *si) { … } /* * The MSI-X interrupt handler for an SGE response queue. */ irqreturn_t t4_sge_intr_msix(int irq, void *cookie) { … } /* * Process the indirect interrupt entries in the interrupt queue and kick off * NAPI for each queue that has generated an entry. */ static unsigned int process_intrq(struct adapter *adap) { … } /* * The MSI interrupt handler, which handles data events from SGE response queues * as well as error and other async events as they all use the same MSI vector. */ static irqreturn_t t4_intr_msi(int irq, void *cookie) { … } /* * Interrupt handler for legacy INTx interrupts. * Handles data events from SGE response queues as well as error and other * async events as they all use the same interrupt line. */ static irqreturn_t t4_intr_intx(int irq, void *cookie) { … } /** * t4_intr_handler - select the top-level interrupt handler * @adap: the adapter * * Selects the top-level interrupt handler based on the type of interrupts * (MSI-X, MSI, or INTx). */ irq_handler_t t4_intr_handler(struct adapter *adap) { … } static void sge_rx_timer_cb(struct timer_list *t) { … } static void sge_tx_timer_cb(struct timer_list *t) { … } /** * bar2_address - return the BAR2 address for an SGE Queue's Registers * @adapter: the adapter * @qid: the SGE Queue ID * @qtype: the SGE Queue Type (Egress or Ingress) * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues * * Returns the BAR2 address for the SGE Queue Registers associated with * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID" * Registers are supported (e.g. the Write Combining Doorbell Buffer). */ static void __iomem *bar2_address(struct adapter *adapter, unsigned int qid, enum t4_bar2_qtype qtype, unsigned int *pbar2_qid) { … } /* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map */ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, struct net_device *dev, int intr_idx, struct sge_fl *fl, rspq_handler_t hnd, rspq_flush_handler_t flush_hnd, int cong) { … } static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id) { … } /** * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue * @adap: the adapter * @txq: the SGE Ethernet TX Queue to initialize * @dev: the Linux Network Device * @netdevq: the corresponding Linux TX Queue * @iqid: the Ingress Queue to which to deliver CIDX Update messages * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers */ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, struct net_device *dev, struct netdev_queue *netdevq, unsigned int iqid, u8 dbqt) { … } int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq, struct net_device *dev, unsigned int iqid, unsigned int cmplqid) { … } int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid, unsigned int cmplqid) { … } static int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_txq *q, struct net_device *dev, u32 cmd, u32 iqid) { … } int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq, struct net_device *dev, unsigned int iqid, unsigned int uld_type) { … } int t4_sge_alloc_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq, struct net_device *dev, u32 iqid) { … } void free_txq(struct adapter *adap, struct sge_txq *q) { … } void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl) { … } /** * t4_free_ofld_rxqs - free a block of consecutive Rx queues * @adap: the adapter * @n: number of queues * @q: pointer to first queue * * Release the resources of a consecutive block of offload Rx queues. */ void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q) { … } void t4_sge_free_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq) { … } /** * t4_free_sge_resources - free SGE resources * @adap: the adapter * * Frees resources used by the SGE queue sets. */ void t4_free_sge_resources(struct adapter *adap) { … } void t4_sge_start(struct adapter *adap) { … } /** * t4_sge_stop - disable SGE operation * @adap: the adapter * * Stop tasklets and timers associated with the DMA engine. Note that * this is effective only if measures have been taken to disable any HW * events that may restart them. */ void t4_sge_stop(struct adapter *adap) { … } /** * t4_sge_init_soft - grab core SGE values needed by SGE code * @adap: the adapter * * We need to grab the SGE operating parameters that we need to have * in order to do our job and make sure we can live with them. */ static int t4_sge_init_soft(struct adapter *adap) { … } /** * t4_sge_init - initialize SGE * @adap: the adapter * * Perform low-level SGE code initialization needed every time after a * chip reset. */ int t4_sge_init(struct adapter *adap) { … }
Codebase Browser
linux