sock_map.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */

#include <linux/bpf.h>
#include <linux/btf_ids.h>
#include <linux/filter.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/net.h>
#include <linux/workqueue.h>
#include <linux/skmsg.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/sock_diag.h>
#include <net/udp.h>

struct bpf_stab { … };

#define SOCK_CREATE_FLAG_MASK …

/* This mutex is used to
 *  - protect race between prog/link attach/detach and link prog update, and
 *  - protect race between releasing and accessing map in bpf_link.
 * A single global mutex lock is used since it is expected contention is low.
 */
static DEFINE_MUTEX(sockmap_mutex);

static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
				struct bpf_prog *old, struct bpf_link *link,
				u32 which);
static struct sk_psock_progs *sock_map_progs(struct bpf_map *map);

static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
{ … }

int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
{ … }

int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
{ … }

static void sock_map_sk_acquire(struct sock *sk)
	__acquires(&sk->sk_lock.slock)
{ … }

static void sock_map_sk_release(struct sock *sk)
	__releases(&sk->sk_lock.slock)
{ … }

static void sock_map_add_link(struct sk_psock *psock,
			      struct sk_psock_link *link,
			      struct bpf_map *map, void *link_raw)
{ … }

static void sock_map_del_link(struct sock *sk,
			      struct sk_psock *psock, void *link_raw)
{ … }

static void sock_map_unref(struct sock *sk, void *link_raw)
{ … }

static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
{ … }

static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
{ … }

static int sock_map_link(struct bpf_map *map, struct sock *sk)
{ … }

static void sock_map_free(struct bpf_map *map)
{ … }

static void sock_map_release_progs(struct bpf_map *map)
{ … }

static struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
{ … }

static void *sock_map_lookup(struct bpf_map *map, void *key)
{ … }

static void *sock_map_lookup_sys(struct bpf_map *map, void *key)
{ … }

static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
			     struct sock **psk)
{ … }

static void sock_map_delete_from_link(struct bpf_map *map, struct sock *sk,
				      void *link_raw)
{ … }

static long sock_map_delete_elem(struct bpf_map *map, void *key)
{ … }

static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next)
{ … }

static int sock_map_update_common(struct bpf_map *map, u32 idx,
				  struct sock *sk, u64 flags)
{ … }

static bool sock_map_op_okay(const struct bpf_sock_ops_kern *ops)
{ … }

static bool sock_map_redirect_allowed(const struct sock *sk)
{ … }

static bool sock_map_sk_is_suitable(const struct sock *sk)
{ … }

static bool sock_map_sk_state_allowed(const struct sock *sk)
{ … }

static int sock_hash_update_common(struct bpf_map *map, void *key,
				   struct sock *sk, u64 flags);

int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
			     u64 flags)
{ … }

static long sock_map_update_elem(struct bpf_map *map, void *key,
				 void *value, u64 flags)
{ … }

BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops,
	   struct bpf_map *, map, void *, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_sock_map_update_proto = …;

BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
	   struct bpf_map *, map, u32, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_sk_redirect_map_proto = …;

BPF_CALL_4(bpf_msg_redirect_map, struct sk_msg *, msg,
	   struct bpf_map *, map, u32, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_msg_redirect_map_proto = …;

struct sock_map_seq_info { … };

struct bpf_iter__sockmap { … };

DEFINE_BPF_ITER_FUNC(sockmap, struct bpf_iter_meta *meta,
		     struct bpf_map *map, void *key,
		     struct sock *sk)

static void *sock_map_seq_lookup_elem(struct sock_map_seq_info *info)
{ … }

static void *sock_map_seq_start(struct seq_file *seq, loff_t *pos)
	__acquires(rcu)
{ … }

static void *sock_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
	__must_hold(rcu)
{ … }

static int sock_map_seq_show(struct seq_file *seq, void *v)
	__must_hold(rcu)
{ … }

static void sock_map_seq_stop(struct seq_file *seq, void *v)
	__releases(rcu)
{ … }

static const struct seq_operations sock_map_seq_ops = …;

static int sock_map_init_seq_private(void *priv_data,
				     struct bpf_iter_aux_info *aux)
{ … }

static void sock_map_fini_seq_private(void *priv_data)
{ … }

static u64 sock_map_mem_usage(const struct bpf_map *map)
{ … }

static const struct bpf_iter_seq_info sock_map_iter_seq_info = …;

BTF_ID_LIST_SINGLE(sock_map_btf_ids, struct, bpf_stab)
const struct bpf_map_ops sock_map_ops = …;

struct bpf_shtab_elem { … };

struct bpf_shtab_bucket { … };

struct bpf_shtab { … };

static inline u32 sock_hash_bucket_hash(const void *key, u32 len)
{ … }

static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,
							u32 hash)
{ … }

static struct bpf_shtab_elem *
sock_hash_lookup_elem_raw(struct hlist_head *head, u32 hash, void *key,
			  u32 key_size)
{ … }

static struct sock *__sock_hash_lookup_elem(struct bpf_map *map, void *key)
{ … }

static void sock_hash_free_elem(struct bpf_shtab *htab,
				struct bpf_shtab_elem *elem)
{ … }

static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,
				       void *link_raw)
{ … }

static long sock_hash_delete_elem(struct bpf_map *map, void *key)
{ … }

static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,
						   void *key, u32 key_size,
						   u32 hash, struct sock *sk,
						   struct bpf_shtab_elem *old)
{ … }

static int sock_hash_update_common(struct bpf_map *map, void *key,
				   struct sock *sk, u64 flags)
{ … }

static int sock_hash_get_next_key(struct bpf_map *map, void *key,
				  void *key_next)
{ … }

static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
{ … }

static void sock_hash_free(struct bpf_map *map)
{ … }

static void *sock_hash_lookup_sys(struct bpf_map *map, void *key)
{ … }

static void *sock_hash_lookup(struct bpf_map *map, void *key)
{ … }

static void sock_hash_release_progs(struct bpf_map *map)
{ … }

BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, sops,
	   struct bpf_map *, map, void *, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_sock_hash_update_proto = …;

BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb,
	   struct bpf_map *, map, void *, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_sk_redirect_hash_proto = …;

BPF_CALL_4(bpf_msg_redirect_hash, struct sk_msg *, msg,
	   struct bpf_map *, map, void *, key, u64, flags)
{ … }

const struct bpf_func_proto bpf_msg_redirect_hash_proto = …;

struct sock_hash_seq_info { … };

static void *sock_hash_seq_find_next(struct sock_hash_seq_info *info,
				     struct bpf_shtab_elem *prev_elem)
{ … }

static void *sock_hash_seq_start(struct seq_file *seq, loff_t *pos)
	__acquires(rcu)
{ … }

static void *sock_hash_seq_next(struct seq_file *seq, void *v, loff_t *pos)
	__must_hold(rcu)
{ … }

static int sock_hash_seq_show(struct seq_file *seq, void *v)
	__must_hold(rcu)
{ … }

static void sock_hash_seq_stop(struct seq_file *seq, void *v)
	__releases(rcu)
{ … }

static const struct seq_operations sock_hash_seq_ops = …;

static int sock_hash_init_seq_private(void *priv_data,
				      struct bpf_iter_aux_info *aux)
{ … }

static void sock_hash_fini_seq_private(void *priv_data)
{ … }

static u64 sock_hash_mem_usage(const struct bpf_map *map)
{ … }

static const struct bpf_iter_seq_info sock_hash_iter_seq_info = …;

BTF_ID_LIST_SINGLE(sock_hash_map_btf_ids, struct, bpf_shtab)
const struct bpf_map_ops sock_hash_ops = …;

static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
{ … }

static int sock_map_prog_link_lookup(struct bpf_map *map, struct bpf_prog ***pprog,
				     struct bpf_link ***plink, u32 which)
{ … }

/* Handle the following four cases:
 * prog_attach: prog != NULL, old == NULL, link == NULL
 * prog_detach: prog == NULL, old != NULL, link == NULL
 * link_attach: prog != NULL, old == NULL, link != NULL
 * link_detach: prog == NULL, old != NULL, link != NULL
 */
static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
				struct bpf_prog *old, struct bpf_link *link,
				u32 which)
{ … }

int sock_map_bpf_prog_query(const union bpf_attr *attr,
			    union bpf_attr __user *uattr)
{ … }

static void sock_map_unlink(struct sock *sk, struct sk_psock_link *link)
{ … }

static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)
{ … }

void sock_map_unhash(struct sock *sk)
{ … }
EXPORT_SYMBOL_GPL(…);

void sock_map_destroy(struct sock *sk)
{ … }
EXPORT_SYMBOL_GPL(…);

void sock_map_close(struct sock *sk, long timeout)
{ … }
EXPORT_SYMBOL_GPL(…);

struct sockmap_link { … };

static void sock_map_link_release(struct bpf_link *link)
{ … }

static int sock_map_link_detach(struct bpf_link *link)
{ … }

static void sock_map_link_dealloc(struct bpf_link *link)
{ … }

/* Handle the following two cases:
 * case 1: link != NULL, prog != NULL, old != NULL
 * case 2: link != NULL, prog != NULL, old == NULL
 */
static int sock_map_link_update_prog(struct bpf_link *link,
				     struct bpf_prog *prog,
				     struct bpf_prog *old)
{ … }

static u32 sock_map_link_get_map_id(const struct sockmap_link *sockmap_link)
{ … }

static int sock_map_link_fill_info(const struct bpf_link *link,
				   struct bpf_link_info *info)
{ … }

static void sock_map_link_show_fdinfo(const struct bpf_link *link,
				      struct seq_file *seq)
{ … }

static const struct bpf_link_ops sock_map_link_ops = …;

int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
{ … }

static int sock_map_iter_attach_target(struct bpf_prog *prog,
				       union bpf_iter_link_info *linfo,
				       struct bpf_iter_aux_info *aux)
{ … }

static void sock_map_iter_detach_target(struct bpf_iter_aux_info *aux)
{ … }

static struct bpf_iter_reg sock_map_iter_reg = …;

static int __init bpf_sockmap_iter_init(void)
{ … }
late_initcall(bpf_sockmap_iter_init);
linux/net/core/sock_map.c
