linux/net/core/skmsg.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */

#include <linux/skmsg.h>
#include <linux/skbuff.h>
#include <linux/scatterlist.h>

#include <net/sock.h>
#include <net/tcp.h>
#include <net/tls.h>
#include <trace/events/sock.h>

static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce)
{ … }

int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
		 int elem_first_coalesce)
{ … }
EXPORT_SYMBOL_GPL(…);

int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
		 u32 off, u32 len)
{ … }
EXPORT_SYMBOL_GPL(…);

void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
{ … }
EXPORT_SYMBOL_GPL(…);

void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes)
{ … }
EXPORT_SYMBOL_GPL(…);

static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i,
			    bool charge)
{ … }

static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i,
			 bool charge)
{ … }

int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg)
{ … }
EXPORT_SYMBOL_GPL(…);

int sk_msg_free(struct sock *sk, struct sk_msg *msg)
{ … }
EXPORT_SYMBOL_GPL(…);

static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg,
				  u32 bytes, bool charge)
{ … }

void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes)
{ … }
EXPORT_SYMBOL_GPL(…);

void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
				  u32 bytes)
{ … }

void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len)
{ … }
EXPORT_SYMBOL_GPL(…);

int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
			      struct sk_msg *msg, u32 bytes)
{ … }
EXPORT_SYMBOL_GPL(…);

int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
			     struct sk_msg *msg, u32 bytes)
{ … }
EXPORT_SYMBOL_GPL(…);

/* Receive sk_msg from psock->ingress_msg to @msg. */
int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
		   int len, int flags)
{ … }
EXPORT_SYMBOL_GPL(…);

bool sk_msg_is_readable(struct sock *sk)
{ … }
EXPORT_SYMBOL_GPL(…);

static struct sk_msg *alloc_sk_msg(gfp_t gfp)
{ … }

static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
						  struct sk_buff *skb)
{ … }

static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
					u32 off, u32 len,
					struct sk_psock *psock,
					struct sock *sk,
					struct sk_msg *msg)
{ … }

static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
				     u32 off, u32 len);

static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
				u32 off, u32 len)
{ … }

/* Puts an skb on the ingress queue of the socket already assigned to the
 * skb. In this case we do not need to check memory limits or skb_set_owner_r
 * because the skb is already accounted for here.
 */
static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
				     u32 off, u32 len)
{ … }

static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
			       u32 off, u32 len, bool ingress)
{ … }

static void sk_psock_skb_state(struct sk_psock *psock,
			       struct sk_psock_work_state *state,
			       int len, int off)
{ … }

static void sk_psock_backlog(struct work_struct *work)
{ … }

struct sk_psock *sk_psock_init(struct sock *sk, int node)
{ … }
EXPORT_SYMBOL_GPL(…);

struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
{ … }

static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
{ … }

static void __sk_psock_zap_ingress(struct sk_psock *psock)
{ … }

static void sk_psock_link_destroy(struct sk_psock *psock)
{ … }

void sk_psock_stop(struct sk_psock *psock)
{ … }

static void sk_psock_done_strp(struct sk_psock *psock);

static void sk_psock_destroy(struct work_struct *work)
{ … }

void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
{ … }
EXPORT_SYMBOL_GPL(…);

static int sk_psock_map_verd(int verdict, bool redir)
{ … }

int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
			 struct sk_msg *msg)
{ … }
EXPORT_SYMBOL_GPL(…);

static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb)
{ … }

static void sk_psock_tls_verdict_apply(struct sk_buff *skb,
				       struct sk_psock *from, int verdict)
{ … }

int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
{ … }
EXPORT_SYMBOL_GPL(…);

static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
				  int verdict)
{ … }

static void sk_psock_write_space(struct sock *sk)
{ … }

#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
{ … }

static int sk_psock_strp_read_done(struct strparser *strp, int err)
{ … }

static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
{ … }

/* Called with socket lock held. */
static void sk_psock_strp_data_ready(struct sock *sk)
{ … }

int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
{ … }

void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
{ … }

void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
{ … }

static void sk_psock_done_strp(struct sk_psock *psock)
{ … }
#else
static void sk_psock_done_strp(struct sk_psock *psock)
{
}
#endif /* CONFIG_BPF_STREAM_PARSER */

static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
{ … }

static void sk_psock_verdict_data_ready(struct sock *sk)
{ … }

void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
{ … }

void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
{ … }