arraymap.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 * Copyright (c) 2016,2017 Facebook
 */
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/filter.h>
#include <linux/perf_event.h>
#include <uapi/linux/btf.h>
#include <linux/rcupdate_trace.h>
#include <linux/btf_ids.h>

#include "map_in_map.h"

#define ARRAY_CREATE_FLAG_MASK …

static void bpf_array_free_percpu(struct bpf_array *array)
{ … }

static int bpf_array_alloc_percpu(struct bpf_array *array)
{ … }

/* Called from syscall */
int array_map_alloc_check(union bpf_attr *attr)
{ … }

static struct bpf_map *array_map_alloc(union bpf_attr *attr)
{ … }

static void *array_map_elem_ptr(struct bpf_array* array, u32 index)
{ … }

/* Called from syscall or from eBPF program */
static void *array_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,
				       u32 off)
{ … }

static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,
				       u32 *off)
{ … }

/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
{ … }

/* Called from eBPF program */
static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

/* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */
static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
{ … }

static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
{ … }

int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
{ … }

/* Called from syscall */
static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
{ … }

/* Called from syscall or from eBPF program */
static long array_map_update_elem(struct bpf_map *map, void *key, void *value,
				  u64 map_flags)
{ … }

int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
			    u64 map_flags)
{ … }

/* Called from syscall or from eBPF program */
static long array_map_delete_elem(struct bpf_map *map, void *key)
{ … }

static void *array_map_vmalloc_addr(struct bpf_array *array)
{ … }

static void array_map_free_timers_wq(struct bpf_map *map)
{ … }

/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
static void array_map_free(struct bpf_map *map)
{ … }

static void array_map_seq_show_elem(struct bpf_map *map, void *key,
				    struct seq_file *m)
{ … }

static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,
					   struct seq_file *m)
{ … }

static int array_map_check_btf(const struct bpf_map *map,
			       const struct btf *btf,
			       const struct btf_type *key_type,
			       const struct btf_type *value_type)
{ … }

static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
{ … }

static bool array_map_meta_equal(const struct bpf_map *meta0,
				 const struct bpf_map *meta1)
{ … }

struct bpf_iter_seq_array_map_info { … };

static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
{ … }

static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{ … }

static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
{ … }

static int bpf_array_map_seq_show(struct seq_file *seq, void *v)
{ … }

static void bpf_array_map_seq_stop(struct seq_file *seq, void *v)
{ … }

static int bpf_iter_init_array_map(void *priv_data,
				   struct bpf_iter_aux_info *aux)
{ … }

static void bpf_iter_fini_array_map(void *priv_data)
{ … }

static const struct seq_operations bpf_array_map_seq_ops = …;

static const struct bpf_iter_seq_info iter_seq_info = …;

static long bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn,
				    void *callback_ctx, u64 flags)
{ … }

static u64 array_map_mem_usage(const struct bpf_map *map)
{ … }

BTF_ID_LIST_SINGLE(array_map_btf_ids, struct, bpf_array)
const struct bpf_map_ops array_map_ops = …;

const struct bpf_map_ops percpu_array_map_ops = …;

static int fd_array_map_alloc_check(union bpf_attr *attr)
{ … }

static void fd_array_map_free(struct bpf_map *map)
{ … }

static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

/* only called from syscall */
int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
{ … }

/* only called from syscall */
int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
				 void *key, void *value, u64 map_flags)
{ … }

static long __fd_array_map_delete_elem(struct bpf_map *map, void *key, bool need_defer)
{ … }

static long fd_array_map_delete_elem(struct bpf_map *map, void *key)
{ … }

static void *prog_fd_array_get_ptr(struct bpf_map *map,
				   struct file *map_file, int fd)
{ … }

static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
{ … }

static u32 prog_fd_array_sys_lookup_elem(void *ptr)
{ … }

/* decrement refcnt of all bpf_progs that are stored in this map */
static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer)
{ … }

static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
					 struct seq_file *m)
{ … }

struct prog_poke_elem { … };

static int prog_array_map_poke_track(struct bpf_map *map,
				     struct bpf_prog_aux *prog_aux)
{ … }

static void prog_array_map_poke_untrack(struct bpf_map *map,
					struct bpf_prog_aux *prog_aux)
{ … }

void __weak bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
				      struct bpf_prog *new, struct bpf_prog *old)
{ … }

static void prog_array_map_poke_run(struct bpf_map *map, u32 key,
				    struct bpf_prog *old,
				    struct bpf_prog *new)
{ … }

static void prog_array_map_clear_deferred(struct work_struct *work)
{ … }

static void prog_array_map_clear(struct bpf_map *map)
{ … }

static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
{ … }

static void prog_array_map_free(struct bpf_map *map)
{ … }

/* prog_array->aux->{type,jited} is a runtime binding.
 * Doing static check alone in the verifier is not enough.
 * Thus, prog_array_map cannot be used as an inner_map
 * and map_meta_equal is not implemented.
 */
const struct bpf_map_ops prog_array_map_ops = …;

static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
						   struct file *map_file)
{ … }

static void __bpf_event_entry_free(struct rcu_head *rcu)
{ … }

static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
{ … }

static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
					 struct file *map_file, int fd)
{ … }

static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
{ … }

static void perf_event_fd_array_release(struct bpf_map *map,
					struct file *map_file)
{ … }

static void perf_event_fd_array_map_free(struct bpf_map *map)
{ … }

const struct bpf_map_ops perf_event_array_map_ops = …;

#ifdef CONFIG_CGROUPS
static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
				     struct file *map_file /* not used */,
				     int fd)
{ … }

static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
{ … }

static void cgroup_fd_array_free(struct bpf_map *map)
{ … }

const struct bpf_map_ops cgroup_array_map_ops = …;
#endif

static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
{ … }

static void array_of_map_free(struct bpf_map *map)
{ … }

static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

static int array_of_map_gen_lookup(struct bpf_map *map,
				   struct bpf_insn *insn_buf)
{ … }

const struct bpf_map_ops array_of_maps_map_ops = …;
linux/kernel/bpf/arraymap.c
