linux/kernel/bpf/bpf_struct_ops.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2019 Facebook */

#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/btf.h>
#include <linux/filter.h>
#include <linux/slab.h>
#include <linux/numa.h>
#include <linux/seq_file.h>
#include <linux/refcount.h>
#include <linux/mutex.h>
#include <linux/btf_ids.h>
#include <linux/rcupdate_wait.h>
#include <linux/poll.h>

struct bpf_struct_ops_value { … };

#define MAX_TRAMP_IMAGE_PAGES …

struct bpf_struct_ops_map { … };

struct bpf_struct_ops_link { … };

static DEFINE_MUTEX(update_mutex);

#define VALUE_PREFIX …
#define VALUE_PREFIX_LEN …

const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = …;

const struct bpf_prog_ops bpf_struct_ops_prog_ops = …;

BTF_ID_LIST(st_ops_ids)
BTF_ID(…)
BTF_ID(…)

enum { … };

extern struct btf *btf_vmlinux;

static bool is_valid_value_type(struct btf *btf, s32 value_id,
				const struct btf_type *type,
				const char *value_name)
{ … }

static void *bpf_struct_ops_image_alloc(void)
{ … }

void bpf_struct_ops_image_free(void *image)
{ … }

#define MAYBE_NULL_SUFFIX …
#define MAX_STUB_NAME …

/* Return the type info of a stub function, if it exists.
 *
 * The name of a stub function is made up of the name of the struct_ops and
 * the name of the function pointer member, separated by "__". For example,
 * if the struct_ops type is named "foo_ops" and the function pointer
 * member is named "bar", the stub function name would be "foo_ops__bar".
 */
static const struct btf_type *
find_stub_func_proto(const struct btf *btf, const char *st_op_name,
		     const char *member_name)
{ … }

/* Prepare argument info for every nullable argument of a member of a
 * struct_ops type.
 *
 * Initialize a struct bpf_struct_ops_arg_info according to type info of
 * the arguments of a stub function. (Check kCFI for more information about
 * stub functions.)
 *
 * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info
 * to provide an array of struct bpf_ctx_arg_aux, which in turn provides
 * the information that used by the verifier to check the arguments of the
 * BPF struct_ops program assigned to the member. Here, we only care about
 * the arguments that are marked as __nullable.
 *
 * The array of struct bpf_ctx_arg_aux is eventually assigned to
 * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the
 * verifier. (See check_struct_ops_btf_id())
 *
 * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If
 * fails, it will be kept untouched.
 */
static int prepare_arg_info(struct btf *btf,
			    const char *st_ops_name,
			    const char *member_name,
			    const struct btf_type *func_proto,
			    struct bpf_struct_ops_arg_info *arg_info)
{ … }

/* Clean up the arg_info in a struct bpf_struct_ops_desc. */
void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
{ … }

int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
			     struct btf *btf,
			     struct bpf_verifier_log *log)
{ … }

static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
					   void *next_key)
{ … }

int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
				       void *value)
{ … }

static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
{ … }

static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
{ … }

static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data)
{ … }

static void bpf_struct_ops_link_release(struct bpf_link *link)
{ … }

static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
{ … }

const struct bpf_link_ops bpf_struct_ops_link_lops = …;

int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
				      struct bpf_tramp_link *link,
				      const struct btf_func_model *model,
				      void *stub_func,
				      void **_image, u32 *_image_off,
				      bool allow_alloc)
{ … }

static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
					   void *value, u64 flags)
{ … }

static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
{ … }

static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
					     struct seq_file *m)
{ … }

static void __bpf_struct_ops_map_free(struct bpf_map *map)
{ … }

static void bpf_struct_ops_map_free(struct bpf_map *map)
{ … }

static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
{ … }

static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
{ … }

static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map)
{ … }

BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
const struct bpf_map_ops bpf_struct_ops_map_ops = …;

/* "const void *" because some subsystem is
 * passing a const (e.g. const struct tcp_congestion_ops *)
 */
bool bpf_struct_ops_get(const void *kdata)
{ … }

void bpf_struct_ops_put(const void *kdata)
{ … }

static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
{ … }

static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
{ … }

static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
					    struct seq_file *seq)
{ … }

static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
					       struct bpf_link_info *info)
{ … }

static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map,
					  struct bpf_map *expected_old_map)
{ … }

static int bpf_struct_ops_map_link_detach(struct bpf_link *link)
{ … }

static __poll_t bpf_struct_ops_map_link_poll(struct file *file,
					     struct poll_table_struct *pts)
{ … }

static const struct bpf_link_ops bpf_struct_ops_map_lops = …;

int bpf_struct_ops_link_create(union bpf_attr *attr)
{ … }

void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
{ … }