test_run.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Facebook
 */
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/vmalloc.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/rcupdate_trace.h>
#include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h>
#include <net/hotdata.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <net/net_namespace.h>
#include <net/page_pool/helpers.h>
#include <linux/error-injection.h>
#include <linux/smp.h>
#include <linux/sock_diag.h>
#include <linux/netfilter.h>
#include <net/netdev_rx_queue.h>
#include <net/xdp.h>
#include <net/netfilter/nf_bpf_link.h>

#define CREATE_TRACE_POINTS
#include <trace/events/bpf_test_run.h>

struct bpf_test_timer { … };

static void bpf_test_timer_enter(struct bpf_test_timer *t)
	__acquires(rcu)
{ … }

static void bpf_test_timer_leave(struct bpf_test_timer *t)
	__releases(rcu)
{ … }

static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
				    u32 repeat, int *err, u32 *duration)
	__must_hold(rcu)
{ … }

/* We put this struct at the head of each page with a context and frame
 * initialised when the page is allocated, so we don't have to do this on each
 * repetition of the test run.
 */
struct xdp_page_head { … };

struct xdp_test_data { … };

/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
 * must be updated accordingly this gets changed, otherwise BPF selftests
 * will fail.
 */
#define TEST_XDP_FRAME_SIZE …
#define TEST_XDP_MAX_BATCH …

static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
{ … }

static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
{ … }

static void xdp_test_run_teardown(struct xdp_test_data *xdp)
{ … }

static bool frame_was_changed(const struct xdp_page_head *head)
{ … }

static bool ctx_was_changed(struct xdp_page_head *head)
{ … }

static void reset_ctx(struct xdp_page_head *head)
{ … }

static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
			   struct sk_buff **skbs,
			   struct net_device *dev)
{ … }

static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
			      u32 repeat)
{ … }

static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
				 u32 repeat, u32 batch_size, u32 *time)

{ … }

static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
			u32 *retval, u32 *time, bool xdp)
{ … }

static int bpf_test_finish(const union bpf_attr *kattr,
			   union bpf_attr __user *uattr, const void *data,
			   struct skb_shared_info *sinfo, u32 size,
			   u32 retval, u32 duration)
{ … }

/* Integer types of various sizes and pointer combinations cover variety of
 * architecture dependent calling conventions. 7+ can be supported in the
 * future.
 */
__bpf_kfunc_start_defs();

__bpf_kfunc int bpf_fentry_test1(int a)
{ … }
EXPORT_SYMBOL_GPL(…);

int noinline bpf_fentry_test2(int a, u64 b)
{ … }

int noinline bpf_fentry_test3(char a, int b, u64 c)
{ … }

int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
{ … }

int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
{ … }

int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
{ … }

struct bpf_fentry_test_t { … };

int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
{ … }

int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
{ … }

__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
{ … }

void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
{ … }

__bpf_kfunc int bpf_modify_return_test(int a, int *b)
{ … }

__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d,
					void *e, char f, int g)
{ … }

__bpf_kfunc int bpf_modify_return_test_tp(int nonce)
{ … }

int noinline bpf_fentry_shadow_test(int a)
{ … }

struct prog_test_member1 { … };

struct prog_test_member { … };

struct prog_test_ref_kfunc { … };

__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
{ … }

__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p)
{ … }
CFI_NOSEAL(…);

__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
{ … }

__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p)
{ … }
CFI_NOSEAL(…);

__bpf_kfunc_end_defs();

BTF_KFUNCS_START(bpf_test_modify_return_ids)
BTF_ID_FLAGS(…)
BTF_ID_FLAGS(…)
BTF_ID_FLAGS(…)
BTF_ID_FLAGS(…)
BTF_KFUNCS_END(…)

static const struct btf_kfunc_id_set bpf_test_modify_return_set = …;

BTF_KFUNCS_START(test_sk_check_kfunc_ids)
BTF_ID_FLAGS(…)
BTF_ID_FLAGS(…)
BTF_KFUNCS_END(…)

static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
			   u32 size, u32 headroom, u32 tailroom)
{ … }

int bpf_prog_test_run_tracing(struct bpf_prog *prog,
			      const union bpf_attr *kattr,
			      union bpf_attr __user *uattr)
{ … }

struct bpf_raw_tp_test_run_info { … };

static void
__bpf_prog_test_run_raw_tp(void *data)
{ … }

int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
			     const union bpf_attr *kattr,
			     union bpf_attr __user *uattr)
{ … }

static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
{ … }

static int bpf_ctx_finish(const union bpf_attr *kattr,
			  union bpf_attr __user *uattr, const void *data,
			  u32 size)
{ … }

/**
 * range_is_zero - test whether buffer is initialized
 * @buf: buffer to check
 * @from: check from this position
 * @to: check up until (excluding) this position
 *
 * This function returns true if the there is a non-zero byte
 * in the buf in the range [from,to).
 */
static inline bool range_is_zero(void *buf, size_t from, size_t to)
{ … }

static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
{ … }

static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
{ … }

static struct proto bpf_dummy_proto = …;

int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
			  union bpf_attr __user *uattr)
{ … }

static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
{ … }

static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
{ … }

int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
			  union bpf_attr __user *uattr)
{ … }

static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
{ … }

int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
				     const union bpf_attr *kattr,
				     union bpf_attr __user *uattr)
{ … }

int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
				union bpf_attr __user *uattr)
{ … }

int bpf_prog_test_run_syscall(struct bpf_prog *prog,
			      const union bpf_attr *kattr,
			      union bpf_attr __user *uattr)
{ … }

static int verify_and_copy_hook_state(struct nf_hook_state *state,
				      const struct nf_hook_state *user,
				      struct net_device *dev)
{ … }

static __be16 nfproto_eth(int nfproto)
{ … }

int bpf_prog_test_run_nf(struct bpf_prog *prog,
			 const union bpf_attr *kattr,
			 union bpf_attr __user *uattr)
{ … }

static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = …;

BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
BTF_ID(…)
BTF_ID(…)
BTF_ID(…)
BTF_ID(…)

static int __init bpf_prog_test_run_init(void)
{ … }
late_initcall(bpf_prog_test_run_init);
linux/net/bpf/test_run.c
