fbnic_txrx.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) Meta Platforms, Inc. and affiliates. */

#include <linux/bitfield.h>
#include <linux/iopoll.h>
#include <linux/pci.h>
#include <net/netdev_queues.h>
#include <net/page_pool/helpers.h>

#include "fbnic.h"
#include "fbnic_csr.h"
#include "fbnic_netdev.h"
#include "fbnic_txrx.h"

struct fbnic_xmit_cb { … };

#define FBNIC_XMIT_CB(__skb) …

static u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring)
{ … }

static u32 fbnic_ring_rd32(struct fbnic_ring *ring, unsigned int csr)
{ … }

static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val)
{ … }

static unsigned int fbnic_desc_unused(struct fbnic_ring *ring)
{ … }

static unsigned int fbnic_desc_used(struct fbnic_ring *ring)
{ … }

static struct netdev_queue *txring_txq(const struct net_device *dev,
				       const struct fbnic_ring *ring)
{ … }

static int fbnic_maybe_stop_tx(const struct net_device *dev,
			       struct fbnic_ring *ring,
			       const unsigned int size)
{ … }

static bool fbnic_tx_sent_queue(struct sk_buff *skb, struct fbnic_ring *ring)
{ … }

static void fbnic_unmap_single_twd(struct device *dev, __le64 *twd)
{ … }

static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd)
{ … }

#define FBNIC_TWD_TYPE(_type) …

static bool
fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta)
{ … }

static void
fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq)
{ … }

static bool
fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta)
{ … }

#define FBNIC_MIN_FRAME_LEN …

static netdev_tx_t
fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring)
{ … }

netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev)
{ … }

netdev_features_t
fbnic_features_check(struct sk_buff *skb, struct net_device *dev,
		     netdev_features_t features)
{ … }

static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget,
			     struct fbnic_ring *ring, bool discard,
			     unsigned int hw_head)
{ … }

static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx,
				 struct page *page)
{ … }

static struct page *fbnic_page_pool_get(struct fbnic_ring *ring,
					unsigned int idx)
{ … }

static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx,
				  struct fbnic_napi_vector *nv, int budget)
{ … }

static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget,
			    struct fbnic_q_triad *qt, s32 head0)
{ … }

static void
fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt,
		int napi_budget)
{ … }

static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,
			    struct fbnic_ring *ring, unsigned int hw_head)
{ … }

static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page)
{ … }

static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq)
{ … }

static unsigned int fbnic_hdr_pg_start(unsigned int pg_off)
{ … }

static unsigned int fbnic_hdr_pg_end(unsigned int pg_off, unsigned int len)
{ … }

static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd,
			      struct fbnic_pkt_buff *pkt,
			      struct fbnic_q_triad *qt)
{ … }

static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,
			      struct fbnic_pkt_buff *pkt,
			      struct fbnic_q_triad *qt)
{ … }

static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv,
			       struct fbnic_pkt_buff *pkt, int budget)
{ … }

static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv,
				       struct fbnic_pkt_buff *pkt)
{ … }

static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd)
{ … }

static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv,
				      u64 rcd, struct sk_buff *skb,
				      struct fbnic_q_triad *qt)
{ … }

static bool fbnic_rcd_metadata_err(u64 rcd)
{ … }

static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
			   struct fbnic_q_triad *qt, int budget)
{ … }

static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv)
{ … }

static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv)
{ … }

static int fbnic_poll(struct napi_struct *napi, int budget)
{ … }

static irqreturn_t fbnic_msix_clean_rings(int __always_unused irq, void *data)
{ … }

static void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,
					     struct fbnic_ring *rxr)
{ … }

static void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
					     struct fbnic_ring *txr)
{ … }

static void fbnic_remove_tx_ring(struct fbnic_net *fbn,
				 struct fbnic_ring *txr)
{ … }

static void fbnic_remove_rx_ring(struct fbnic_net *fbn,
				 struct fbnic_ring *rxr)
{ … }

static void fbnic_free_napi_vector(struct fbnic_net *fbn,
				   struct fbnic_napi_vector *nv)
{ … }

void fbnic_free_napi_vectors(struct fbnic_net *fbn)
{ … }

static void fbnic_name_napi_vector(struct fbnic_napi_vector *nv)
{ … }

#define FBNIC_PAGE_POOL_FLAGS …

static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn,
				    struct fbnic_napi_vector *nv)
{ … }

static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell,
			    int q_idx, u8 flags)
{ … }

static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
				   unsigned int v_count, unsigned int v_idx,
				   unsigned int txq_count, unsigned int txq_idx,
				   unsigned int rxq_count, unsigned int rxq_idx)
{ … }

int fbnic_alloc_napi_vectors(struct fbnic_net *fbn)
{ … }

static void fbnic_free_ring_resources(struct device *dev,
				      struct fbnic_ring *ring)
{ … }

static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn,
				    struct fbnic_ring *txr)
{ … }

static int fbnic_alloc_tx_ring_buffer(struct fbnic_ring *txr)
{ … }

static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn,
					 struct fbnic_ring *txr)
{ … }

static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn,
				    struct fbnic_ring *rxr)
{ … }

static int fbnic_alloc_rx_ring_buffer(struct fbnic_ring *rxr)
{ … }

static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn,
					 struct fbnic_ring *rxr)
{ … }

static void fbnic_free_qt_resources(struct fbnic_net *fbn,
				    struct fbnic_q_triad *qt)
{ … }

static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn,
				       struct fbnic_q_triad *qt)
{ … }

static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn,
				       struct fbnic_q_triad *qt)
{ … }

static void fbnic_free_nv_resources(struct fbnic_net *fbn,
				    struct fbnic_napi_vector *nv)
{ … }

static int fbnic_alloc_nv_resources(struct fbnic_net *fbn,
				    struct fbnic_napi_vector *nv)
{ … }

void fbnic_free_resources(struct fbnic_net *fbn)
{ … }

int fbnic_alloc_resources(struct fbnic_net *fbn)
{ … }

static void fbnic_disable_twq0(struct fbnic_ring *txr)
{ … }

static void fbnic_disable_tcq(struct fbnic_ring *txr)
{ … }

static void fbnic_disable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq)
{ … }

static void fbnic_disable_rcq(struct fbnic_ring *rxr)
{ … }

void fbnic_napi_disable(struct fbnic_net *fbn)
{ … }

void fbnic_disable(struct fbnic_net *fbn)
{ … }

static void fbnic_tx_flush(struct fbnic_dev *fbd)
{ … }

static void fbnic_tx_flush_off(struct fbnic_dev *fbd)
{ … }

struct fbnic_idle_regs { … };

static bool fbnic_all_idle(struct fbnic_dev *fbd,
			   const struct fbnic_idle_regs *regs,
			   unsigned int nregs)
{ … }

static void fbnic_idle_dump(struct fbnic_dev *fbd,
			    const struct fbnic_idle_regs *regs,
			    unsigned int nregs, const char *dir, int err)
{ … }

int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail)
{ … }

void fbnic_flush(struct fbnic_net *fbn)
{ … }

void fbnic_fill(struct fbnic_net *fbn)
{ … }

static void fbnic_enable_twq0(struct fbnic_ring *twq)
{ … }

static void fbnic_enable_tcq(struct fbnic_napi_vector *nv,
			     struct fbnic_ring *tcq)
{ … }

static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq)
{ … }

static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv,
				       struct fbnic_ring *rcq)
{ … }

static void fbnic_enable_rcq(struct fbnic_napi_vector *nv,
			     struct fbnic_ring *rcq)
{ … }

void fbnic_enable(struct fbnic_net *fbn)
{ … }

static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv)
{ … }

void fbnic_napi_enable(struct fbnic_net *fbn)
{ … }

void fbnic_napi_depletion_check(struct net_device *netdev)
{ … }
linux/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
