linux/drivers/net/ethernet/google/gve/gve_rx.c

// SPDX-License-Identifier: (GPL-2.0 OR MIT)
/* Google virtual Ethernet (gve) driver
 *
 * Copyright (C) 2015-2021 Google, Inc.
 */

#include "gve.h"
#include "gve_adminq.h"
#include "gve_utils.h"
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <net/xdp.h>
#include <net/xdp_sock_drv.h>

static void gve_rx_free_buffer(struct device *dev,
			       struct gve_rx_slot_page_info *page_info,
			       union gve_rx_data_slot *data_slot)
{ … }

static void gve_rx_unfill_pages(struct gve_priv *priv,
				struct gve_rx_ring *rx,
				struct gve_rx_alloc_rings_cfg *cfg)
{ … }

static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
{ … }

static void gve_rx_init_ring_state_gqi(struct gve_rx_ring *rx)
{ … }

static void gve_rx_reset_ring_gqi(struct gve_priv *priv, int idx)
{ … }

void gve_rx_stop_ring_gqi(struct gve_priv *priv, int idx)
{ … }

void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
			  struct gve_rx_alloc_rings_cfg *cfg)
{ … }

static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
			     dma_addr_t addr, struct page *page, __be64 *slot_addr)
{ … }

static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
			       struct gve_rx_slot_page_info *page_info,
			       union gve_rx_data_slot *data_slot,
			       struct gve_rx_ring *rx)
{ … }

static int gve_rx_prefill_pages(struct gve_rx_ring *rx,
				struct gve_rx_alloc_rings_cfg *cfg)
{ … }

void gve_rx_start_ring_gqi(struct gve_priv *priv, int idx)
{ … }

int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
			  struct gve_rx_alloc_rings_cfg *cfg,
			  struct gve_rx_ring *rx,
			  int idx)
{ … }

int gve_rx_alloc_rings_gqi(struct gve_priv *priv,
			   struct gve_rx_alloc_rings_cfg *cfg)
{ … }

void gve_rx_free_rings_gqi(struct gve_priv *priv,
			   struct gve_rx_alloc_rings_cfg *cfg)
{ … }

void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
{ … }

static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
{ … }

static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
					struct gve_rx_slot_page_info *page_info,
					unsigned int truesize, u16 len,
					struct gve_rx_ctx *ctx)
{ … }

static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
{ … }

static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
{ … }

static struct sk_buff *
gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
		      struct gve_rx_slot_page_info *page_info, u16 len,
		      struct napi_struct *napi,
		      union gve_rx_data_slot *data_slot,
		      u16 packet_buffer_size, struct gve_rx_ctx *ctx)
{ … }

static struct sk_buff *gve_rx_copy_to_pool(struct gve_rx_ring *rx,
					   struct gve_rx_slot_page_info *page_info,
					   u16 len, struct napi_struct *napi)
{ … }

static struct sk_buff *
gve_rx_qpl(struct device *dev, struct net_device *netdev,
	   struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
	   u16 len, struct napi_struct *napi,
	   union gve_rx_data_slot *data_slot)
{ … }

static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
				  struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
				  u16 len, union gve_rx_data_slot *data_slot,
				  bool is_only_frag)
{ … }

static int gve_xsk_pool_redirect(struct net_device *dev,
				 struct gve_rx_ring *rx,
				 void *data, int len,
				 struct bpf_prog *xdp_prog)
{ … }

static int gve_xdp_redirect(struct net_device *dev, struct gve_rx_ring *rx,
			    struct xdp_buff *orig, struct bpf_prog *xdp_prog)
{ … }

static void gve_xdp_done(struct gve_priv *priv, struct gve_rx_ring *rx,
			 struct xdp_buff *xdp, struct bpf_prog *xprog,
			 int xdp_act)
{ … }

#define GVE_PKTCONT_BIT_IS_SET(x) …
static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
		   struct gve_rx_desc *desc, u32 idx,
		   struct gve_rx_cnts *cnts)
{ … }

bool gve_rx_work_pending(struct gve_rx_ring *rx)
{ … }

static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
{ … }

static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
			     netdev_features_t feat)
{ … }

int gve_rx_poll(struct gve_notify_block *block, int budget)
{ … }