linux/drivers/net/ethernet/freescale/enetc/enetc.c

// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/* Copyright 2017-2019 NXP */

#include "enetc.h"
#include <linux/bpf_trace.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/vmalloc.h>
#include <linux/ptp_classify.h>
#include <net/ip6_checksum.h>
#include <net/pkt_sched.h>
#include <net/tso.h>

u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
{ … }
EXPORT_SYMBOL_GPL(…);

static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
					 u8 preemptible_tcs)
{ … }

static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
{ … }

static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
							struct enetc_bdr *tx_ring)
{ … }

static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
{ … }

static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
{ … }

static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
				struct enetc_tx_swbd *tx_swbd)
{ … }

static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
				struct enetc_tx_swbd *tx_swbd)
{ … }

/* Let H/W know BD ring has been updated */
static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
{ … }

static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
			   u8 *msgtype, u8 *twostep,
			   u16 *correction_offset, u16 *body_offset)
{ … }

static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
{ … }

static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
				 struct enetc_tx_swbd *tx_swbd,
				 union enetc_tx_bd *txbd, int *i, int hdr_len,
				 int data_len)
{ … }

static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
				 struct enetc_tx_swbd *tx_swbd,
				 union enetc_tx_bd *txbd, char *data,
				 int size, bool last_bd)
{ … }

static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
				 char *hdr, int hdr_len, int *l4_hdr_len)
{ … }

static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
				    struct sk_buff *skb, char *hdr, int len,
				    __wsum sum)
{ … }

static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
{ … }

static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
				    struct net_device *ndev)
{ … }

netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

static irqreturn_t enetc_msix(int irq, void *data)
{ … }

static void enetc_rx_dim_work(struct work_struct *w)
{ … }

static void enetc_rx_net_dim(struct enetc_int_vector *v)
{ … }

static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
{ … }

static bool enetc_page_reusable(struct page *page)
{ … }

static void enetc_reuse_page(struct enetc_bdr *rx_ring,
			     struct enetc_rx_swbd *old)
{ … }

static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
				u64 *tstamp)
{ … }

static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
{ … }

static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
				      struct enetc_tx_swbd *tx_swbd)
{ … }

static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
{ … }

static bool enetc_new_page(struct enetc_bdr *rx_ring,
			   struct enetc_rx_swbd *rx_swbd)
{ … }

static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
{ … }

#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
static void enetc_get_rx_tstamp(struct net_device *ndev,
				union enetc_rx_bd *rxbd,
				struct sk_buff *skb)
{ … }
#endif

static void enetc_get_offloads(struct enetc_bdr *rx_ring,
			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
{ … }

/* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
 * mapped buffers.
 */
static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
					       int i, u16 size)
{ … }

/* Reuse the current page without performing half-page buffer flipping */
static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
			      struct enetc_rx_swbd *rx_swbd)
{ … }

/* Reuse the current page by performing half-page buffer flipping */
static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
			       struct enetc_rx_swbd *rx_swbd)
{ … }

static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
						int i, u16 size)
{ … }

static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
				     u16 size, struct sk_buff *skb)
{ … }

static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
					      u32 bd_status,
					      union enetc_rx_bd **rxbd, int *i)
{ … }

static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
				       u32 bd_status, union enetc_rx_bd **rxbd,
				       int *i, int *cleaned_cnt, int buffer_size)
{ … }

#define ENETC_RXBD_BUNDLE …

static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
			       struct napi_struct *napi, int work_limit)
{ … }

static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
				  struct enetc_tx_swbd *tx_swbd,
				  int frm_len)
{ … }

/* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
 * descriptors.
 */
static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
{ … }

static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
					  struct enetc_tx_swbd *xdp_tx_arr,
					  struct xdp_frame *xdp_frame)
{ … }

int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
		   struct xdp_frame **frames, u32 flags)
{ … }
EXPORT_SYMBOL_GPL(…);

static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
				     struct xdp_buff *xdp_buff, u16 size)
{ … }

static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
				     u16 size, struct xdp_buff *xdp_buff)
{ … }

static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
				 union enetc_rx_bd **rxbd, int *i,
				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
{ … }

/* Convert RX buffer descriptors to TX buffer descriptors. These will be
 * recycled back into the RX ring in enetc_clean_tx_ring.
 */
static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
					struct enetc_bdr *rx_ring,
					int rx_ring_first, int rx_ring_last)
{ … }

static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
			   int rx_ring_last)
{ … }

static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
				   struct napi_struct *napi, int work_limit,
				   struct bpf_prog *prog)
{ … }

static int enetc_poll(struct napi_struct *napi, int budget)
{ … }

/* Probing and Init */
#define ENETC_MAX_RFS_SIZE …
void enetc_get_si_caps(struct enetc_si *si)
{ … }
EXPORT_SYMBOL_GPL(…);

static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
{ … }

static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
{ … }

static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
				   struct device *dev, size_t bd_count)
{ … }

static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
{ … }

static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
{ … }

static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
				    size_t num_resources)
{ … }

static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
				   struct device *dev, size_t bd_count,
				   bool extended)
{ … }

static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
{ … }

static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
{ … }

static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
				    size_t num_resources)
{ … }

static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
				     const struct enetc_bdr_resource *res)
{ … }

static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
				     const struct enetc_bdr_resource *res)
{ … }

static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
				      const struct enetc_bdr_resource *res)
{ … }

static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
				      const struct enetc_bdr_resource *res)
{ … }

static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
{ … }

static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
{ … }

static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
{ … }

static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
{ … }

int enetc_configure_si(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_free_si_resources(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
{ … }

static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
			      bool extended)
{ … }

static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
{ … }

static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
{ … }

static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
{ … }

static void enetc_enable_bdrs(struct enetc_ndev_priv *priv)
{ … }

static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
{ … }

static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
{ … }

static void enetc_disable_bdrs(struct enetc_ndev_priv *priv)
{ … }

static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
{ … }

static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
{ … }

static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
{ … }

static void enetc_free_irqs(struct enetc_ndev_priv *priv)
{ … }

static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
{ … }

static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
{ … }

static int enetc_phylink_connect(struct net_device *ndev)
{ … }

static void enetc_tx_onestep_tstamp(struct work_struct *work)
{ … }

static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
{ … }

void enetc_start(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

int enetc_open(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_stop(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

int enetc_close(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
			     void *ctx)
{ … }

static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
{ … }

void enetc_reset_tc_mqprio(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
{ … }
EXPORT_SYMBOL_GPL(…);

static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
{ … }

static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
				struct netlink_ext_ack *extack)
{ … }

int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
{ … }
EXPORT_SYMBOL_GPL(…);

struct net_device_stats *enetc_get_stats(struct net_device *ndev)
{ … }
EXPORT_SYMBOL_GPL(…);

static int enetc_set_rss(struct net_device *ndev, int en)
{ … }

static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
{ … }

static void enetc_enable_txvlan(struct net_device *ndev, bool en)
{ … }

void enetc_set_features(struct net_device *ndev, netdev_features_t features)
{ … }
EXPORT_SYMBOL_GPL(…);

#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
{ … }

static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
{ … }
#endif

int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
{ … }
EXPORT_SYMBOL_GPL(…);

int enetc_alloc_msix(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_free_msix(struct enetc_ndev_priv *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

static void enetc_kfree_si(struct enetc_si *si)
{ … }

static void enetc_detect_errata(struct enetc_si *si)
{ … }

int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
{ … }
EXPORT_SYMBOL_GPL(…);

void enetc_pci_remove(struct pci_dev *pdev)
{ … }
EXPORT_SYMBOL_GPL(…);

MODULE_DESCRIPTION(…) …;
MODULE_LICENSE(…) …;