linux/net/core/sock_reuseport.c

// SPDX-License-Identifier: GPL-2.0
/*
 * To speed up listener socket lookup, create an array to store all sockets
 * listening on the same port.  This allows a decision to be made after finding
 * the first socket.  An optional BPF program can also be configured for
 * selecting the socket index from the array of available sockets.
 */

#include <net/ip.h>
#include <net/sock_reuseport.h>
#include <linux/bpf.h>
#include <linux/idr.h>
#include <linux/filter.h>
#include <linux/rcupdate.h>

#define INIT_SOCKS …

DEFINE_SPINLOCK(…);

static DEFINE_IDA(reuseport_ida);
static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
			       struct sock_reuseport *reuse, bool bind_inany);

void reuseport_has_conns_set(struct sock *sk)
{ … }
EXPORT_SYMBOL(…);

static void __reuseport_get_incoming_cpu(struct sock_reuseport *reuse)
{ … }

static void __reuseport_put_incoming_cpu(struct sock_reuseport *reuse)
{ … }

static void reuseport_get_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{ … }

static void reuseport_put_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{ … }

void reuseport_update_incoming_cpu(struct sock *sk, int val)
{ … }

static int reuseport_sock_index(struct sock *sk,
				const struct sock_reuseport *reuse,
				bool closed)
{ … }

static void __reuseport_add_sock(struct sock *sk,
				 struct sock_reuseport *reuse)
{ … }

static bool __reuseport_detach_sock(struct sock *sk,
				    struct sock_reuseport *reuse)
{ … }

static void __reuseport_add_closed_sock(struct sock *sk,
					struct sock_reuseport *reuse)
{ … }

static bool __reuseport_detach_closed_sock(struct sock *sk,
					   struct sock_reuseport *reuse)
{ … }

static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
{ … }

int reuseport_alloc(struct sock *sk, bool bind_inany)
{ … }
EXPORT_SYMBOL(…);

static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
{ … }

static void reuseport_free_rcu(struct rcu_head *head)
{ … }

/**
 *  reuseport_add_sock - Add a socket to the reuseport group of another.
 *  @sk:  New socket to add to the group.
 *  @sk2: Socket belonging to the existing reuseport group.
 *  @bind_inany: Whether or not the group is bound to a local INANY address.
 *
 *  May return ENOMEM and not add socket to group under memory pressure.
 */
int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
{ … }
EXPORT_SYMBOL(…);

static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
			       struct sock_reuseport *reuse, bool bind_inany)
{ … }

void reuseport_detach_sock(struct sock *sk)
{ … }
EXPORT_SYMBOL(…);

void reuseport_stop_listen_sock(struct sock *sk)
{ … }
EXPORT_SYMBOL(…);

static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
				   struct bpf_prog *prog, struct sk_buff *skb,
				   int hdr_len)
{ … }

static struct sock *reuseport_select_sock_by_hash(struct sock_reuseport *reuse,
						  u32 hash, u16 num_socks)
{ … }

/**
 *  reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: First socket in the group.
 *  @hash: When no BPF filter is available, use this hash to select.
 *  @skb: skb to run through BPF filter.
 *  @hdr_len: BPF filter expects skb data pointer at payload data.  If
 *    the skb does not yet point at the payload, this parameter represents
 *    how far the pointer needs to advance to reach the payload.
 *  Returns a socket that should receive the packet (or NULL on error).
 */
struct sock *reuseport_select_sock(struct sock *sk,
				   u32 hash,
				   struct sk_buff *skb,
				   int hdr_len)
{ … }
EXPORT_SYMBOL(…);

/**
 *  reuseport_migrate_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: close()ed or shutdown()ed socket in the group.
 *  @migrating_sk: ESTABLISHED/SYN_RECV full socket in the accept queue or
 *    NEW_SYN_RECV request socket during 3WHS.
 *  @skb: skb to run through BPF filter.
 *  Returns a socket (with sk_refcnt +1) that should accept the child socket
 *  (or NULL on error).
 */
struct sock *reuseport_migrate_sock(struct sock *sk,
				    struct sock *migrating_sk,
				    struct sk_buff *skb)
{ … }
EXPORT_SYMBOL(…);

int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
{ … }
EXPORT_SYMBOL(…);

int reuseport_detach_prog(struct sock *sk)
{ … }
EXPORT_SYMBOL(…);