linux/drivers/net/ethernet/netronome/nfp/nfdk/dp.c

// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2015-2019 Netronome Systems, Inc. */

#include <linux/bpf_trace.h>
#include <linux/netdevice.h>
#include <linux/overflow.h>
#include <linux/sizes.h>
#include <linux/bitfield.h>
#include <net/xfrm.h>

#include "../nfp_app.h"
#include "../nfp_net.h"
#include "../nfp_net_dp.h"
#include "../crypto/crypto.h"
#include "../crypto/fw.h"
#include "nfdk.h"

static int nfp_nfdk_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring)
{ … }

static int nfp_nfdk_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring)
{ … }

static void nfp_nfdk_tx_ring_stop(struct netdev_queue *nd_q,
				  struct nfp_net_tx_ring *tx_ring)
{ … }

static __le64
nfp_nfdk_tx_tso(struct nfp_net_r_vector *r_vec, struct nfp_nfdk_tx_buf *txbuf,
		struct sk_buff *skb)
{ … }

static u8
nfp_nfdk_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
		 unsigned int pkt_cnt, struct sk_buff *skb, u64 flags)
{ … }

static int
nfp_nfdk_tx_maybe_close_block(struct nfp_net_tx_ring *tx_ring,
			      struct sk_buff *skb)
{ … }

static int
nfp_nfdk_prep_tx_meta(struct nfp_net_dp *dp, struct nfp_app *app,
		      struct sk_buff *skb, bool *ipsec)
{ … }

/**
 * nfp_nfdk_tx() - Main transmit entry point
 * @skb:    SKB to transmit
 * @netdev: netdev structure
 *
 * Return: NETDEV_TX_OK on success.
 */
netdev_tx_t nfp_nfdk_tx(struct sk_buff *skb, struct net_device *netdev)
{ … }

/**
 * nfp_nfdk_tx_complete() - Handled completed TX packets
 * @tx_ring:	TX ring structure
 * @budget:	NAPI budget (only used as bool to determine if in NAPI context)
 */
static void nfp_nfdk_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget)
{ … }

/* Receive processing */
static void *
nfp_nfdk_napi_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr)
{ … }

/**
 * nfp_nfdk_rx_give_one() - Put mapped skb on the software and hardware rings
 * @dp:		NFP Net data path struct
 * @rx_ring:	RX ring structure
 * @frag:	page fragment buffer
 * @dma_addr:	DMA address of skb mapping
 */
static void
nfp_nfdk_rx_give_one(const struct nfp_net_dp *dp,
		     struct nfp_net_rx_ring *rx_ring,
		     void *frag, dma_addr_t dma_addr)
{ … }

/**
 * nfp_nfdk_rx_ring_fill_freelist() - Give buffers from the ring to FW
 * @dp:	     NFP Net data path struct
 * @rx_ring: RX ring to fill
 */
void nfp_nfdk_rx_ring_fill_freelist(struct nfp_net_dp *dp,
				    struct nfp_net_rx_ring *rx_ring)
{ … }

/**
 * nfp_nfdk_rx_csum_has_errors() - group check if rxd has any csum errors
 * @flags: RX descriptor flags field in CPU byte order
 */
static int nfp_nfdk_rx_csum_has_errors(u16 flags)
{ … }

/**
 * nfp_nfdk_rx_csum() - set SKB checksum field based on RX descriptor flags
 * @dp:  NFP Net data path struct
 * @r_vec: per-ring structure
 * @rxd: Pointer to RX descriptor
 * @meta: Parsed metadata prepend
 * @skb: Pointer to SKB
 */
static void
nfp_nfdk_rx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
		 struct nfp_net_rx_desc *rxd, struct nfp_meta_parsed *meta,
		 struct sk_buff *skb)
{ … }

static void
nfp_nfdk_set_hash(struct net_device *netdev, struct nfp_meta_parsed *meta,
		  unsigned int type, __be32 *hash)
{ … }

static bool
nfp_nfdk_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,
		    void *data, void *pkt, unsigned int pkt_len, int meta_len)
{ … }

static void
nfp_nfdk_rx_drop(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
		 struct nfp_net_rx_ring *rx_ring, struct nfp_net_rx_buf *rxbuf,
		 struct sk_buff *skb)
{ … }

static bool nfp_nfdk_xdp_complete(struct nfp_net_tx_ring *tx_ring)
{ … }

static bool
nfp_nfdk_tx_xdp_buf(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring,
		    struct nfp_net_tx_ring *tx_ring,
		    struct nfp_net_rx_buf *rxbuf, unsigned int dma_off,
		    unsigned int pkt_len, bool *completed)
{ … }

/**
 * nfp_nfdk_rx() - receive up to @budget packets on @rx_ring
 * @rx_ring:   RX ring to receive from
 * @budget:    NAPI budget
 *
 * Note, this function is separated out from the napi poll function to
 * more cleanly separate packet receive code from other bookkeeping
 * functions performed in the napi poll function.
 *
 * Return: Number of packets received.
 */
static int nfp_nfdk_rx(struct nfp_net_rx_ring *rx_ring, int budget)
{ … }

/**
 * nfp_nfdk_poll() - napi poll function
 * @napi:    NAPI structure
 * @budget:  NAPI budget
 *
 * Return: number of packets polled.
 */
int nfp_nfdk_poll(struct napi_struct *napi, int budget)
{ … }

/* Control device data path
 */

bool
nfp_nfdk_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,
		     struct sk_buff *skb, bool old)
{ … }

static void __nfp_ctrl_tx_queued(struct nfp_net_r_vector *r_vec)
{ … }

static bool
nfp_ctrl_meta_ok(struct nfp_net *nn, void *data, unsigned int meta_len)
{ … }

static bool
nfp_ctrl_rx_one(struct nfp_net *nn, struct nfp_net_dp *dp,
		struct nfp_net_r_vector *r_vec, struct nfp_net_rx_ring *rx_ring)
{ … }

static bool nfp_ctrl_rx(struct nfp_net_r_vector *r_vec)
{ … }

void nfp_nfdk_ctrl_poll(struct tasklet_struct *t)
{ … }