linux/kernel/bpf/bpf_iter.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 Facebook */

#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <linux/rcupdate_trace.h>

struct bpf_iter_target_info { … };

struct bpf_iter_link { … };

struct bpf_iter_priv_data { … };

static struct list_head targets = …;
static DEFINE_MUTEX(targets_mutex);

/* protect bpf_iter_link changes */
static DEFINE_MUTEX(link_mutex);

/* incremented on every opened seq_file */
static atomic64_t session_id;

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
			    const struct bpf_iter_seq_info *seq_info);

static void bpf_iter_inc_seq_num(struct seq_file *seq)
{ … }

static void bpf_iter_dec_seq_num(struct seq_file *seq)
{ … }

static void bpf_iter_done_stop(struct seq_file *seq)
{ … }

static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
{ … }

static bool bpf_iter_support_resched(struct seq_file *seq)
{ … }

/* maximum visited objects before bailing out */
#define MAX_ITER_OBJECTS …

/* bpf_seq_read, a customized and simpler version for bpf iterator.
 * The following are differences from seq_read():
 *  . fixed buffer size (PAGE_SIZE)
 *  . assuming NULL ->llseek()
 *  . stop() may call bpf program, handling potential overflow there
 */
static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
			    loff_t *ppos)
{ … }

static const struct bpf_iter_seq_info *
__get_seq_info(struct bpf_iter_link *link)
{ … }

static int iter_open(struct inode *inode, struct file *file)
{ … }

static int iter_release(struct inode *inode, struct file *file)
{ … }

const struct file_operations bpf_iter_fops = …;

/* The argument reg_info will be cached in bpf_iter_target_info.
 * The common practice is to declare target reg_info as
 * a const static variable and passed as an argument to
 * bpf_iter_reg_target().
 */
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
{ … }

void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
{ … }

static void cache_btf_id(struct bpf_iter_target_info *tinfo,
			 struct bpf_prog *prog)
{ … }

bool bpf_iter_prog_supported(struct bpf_prog *prog)
{ … }

const struct bpf_func_proto *
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{ … }

static void bpf_iter_link_release(struct bpf_link *link)
{ … }

static void bpf_iter_link_dealloc(struct bpf_link *link)
{ … }

static int bpf_iter_link_replace(struct bpf_link *link,
				 struct bpf_prog *new_prog,
				 struct bpf_prog *old_prog)
{ … }

static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
				      struct seq_file *seq)
{ … }

static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
					struct bpf_link_info *info)
{ … }

static const struct bpf_link_ops bpf_iter_link_lops = …;

bool bpf_link_is_iter(struct bpf_link *link)
{ … }

int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
			 struct bpf_prog *prog)
{ … }

static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
			  struct bpf_iter_target_info *tinfo,
			  const struct bpf_iter_seq_info *seq_info,
			  struct bpf_prog *prog)
{ … }

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
			    const struct bpf_iter_seq_info *seq_info)
{ … }

int bpf_iter_new_fd(struct bpf_link *link)
{ … }

struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
{ … }

int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
{ … }

BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
	   void *, callback_ctx, u64, flags)
{ … }

const struct bpf_func_proto bpf_for_each_map_elem_proto = …;

BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
	   u64, flags)
{ … }

const struct bpf_func_proto bpf_loop_proto = …;

struct bpf_iter_num_kern { … } __aligned(…);

__bpf_kfunc_start_defs();

__bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end)
{ … }

__bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it)
{ … }

__bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
{ … }

__bpf_kfunc_end_defs();