linux/net/core/gro.c

// SPDX-License-Identifier: GPL-2.0-or-later
#include <net/gro.h>
#include <net/dst_metadata.h>
#include <net/busy_poll.h>
#include <trace/events/net.h>
#include <linux/skbuff_ref.h>

#define MAX_GRO_SKBS …

/* This should be increased if a protocol with a bigger head is added. */
#define GRO_MAX_HEAD …

static DEFINE_SPINLOCK(offload_lock);

/**
 *	dev_add_offload - register offload handlers
 *	@po: protocol offload declaration
 *
 *	Add protocol offload handlers to the networking stack. The passed
 *	&proto_offload is linked into kernel lists and may not be freed until
 *	it has been removed from the kernel lists.
 *
 *	This call does not sleep therefore it can not
 *	guarantee all CPU's that are in middle of receiving packets
 *	will see the new offload handlers (until the next received packet).
 */
void dev_add_offload(struct packet_offload *po)
{ … }
EXPORT_SYMBOL(…);

/**
 *	__dev_remove_offload	 - remove offload handler
 *	@po: packet offload declaration
 *
 *	Remove a protocol offload handler that was previously added to the
 *	kernel offload handlers by dev_add_offload(). The passed &offload_type
 *	is removed from the kernel lists and can be freed or reused once this
 *	function returns.
 *
 *      The packet type might still be in use by receivers
 *	and must not be freed until after all the CPU's have gone
 *	through a quiescent state.
 */
static void __dev_remove_offload(struct packet_offload *po)
{ … }

/**
 *	dev_remove_offload	 - remove packet offload handler
 *	@po: packet offload declaration
 *
 *	Remove a packet offload handler that was previously added to the kernel
 *	offload handlers by dev_add_offload(). The passed &offload_type is
 *	removed from the kernel lists and can be freed or reused once this
 *	function returns.
 *
 *	This call sleeps to guarantee that no CPU is looking at the packet
 *	type after return.
 */
void dev_remove_offload(struct packet_offload *po)
{ … }
EXPORT_SYMBOL(…);


int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
{ … }

int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
{ … }


static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
{ … }

static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
				   bool flush_old)
{ … }

/* napi->gro_hash[].list contains packets ordered by age.
 * youngest packets at the head of it.
 * Complete skbs in reverse order to reduce latencies.
 */
void napi_gro_flush(struct napi_struct *napi, bool flush_old)
{ … }
EXPORT_SYMBOL(…);

static unsigned long gro_list_prepare_tc_ext(const struct sk_buff *skb,
					     const struct sk_buff *p,
					     unsigned long diffs)
{ … }

static void gro_list_prepare(const struct list_head *head,
			     const struct sk_buff *skb)
{ … }

static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
{ … }

static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
{ … }

static void gro_try_pull_from_frag0(struct sk_buff *skb)
{ … }

static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
{ … }

static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{ … }

struct packet_offload *gro_find_receive_by_type(__be16 type)
{ … }
EXPORT_SYMBOL(…);

struct packet_offload *gro_find_complete_by_type(__be16 type)
{ … }
EXPORT_SYMBOL(…);

static gro_result_t napi_skb_finish(struct napi_struct *napi,
				    struct sk_buff *skb,
				    gro_result_t ret)
{ … }

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{ … }
EXPORT_SYMBOL(…);

static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
{ … }

struct sk_buff *napi_get_frags(struct napi_struct *napi)
{ … }
EXPORT_SYMBOL(…);

static gro_result_t napi_frags_finish(struct napi_struct *napi,
				      struct sk_buff *skb,
				      gro_result_t ret)
{ … }

/* Upper GRO stack assumes network header starts at gro_offset=0
 * Drivers could call both napi_gro_frags() and napi_gro_receive()
 * We copy ethernet header into skb->data to have a common layout.
 */
static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
{ … }

gro_result_t napi_gro_frags(struct napi_struct *napi)
{ … }
EXPORT_SYMBOL(…);

/* Compute the checksum from gro_offset and return the folded value
 * after adding in any pseudo checksum.
 */
__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
{ … }
EXPORT_SYMBOL(…);