// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#include <linux/capability.h>
#include <stdlib.h>
#include <regex.h>
#include <test_progs.h>
#include <bpf/btf.h>
#include "autoconf_helper.h"
#include "disasm_helpers.h"
#include "unpriv_helpers.h"
#include "cap_helpers.h"
#include "jit_disasm_helpers.h"
#define str_has_pfx(str, pfx) \
(strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
#define TEST_LOADER_LOG_BUF_SZ 2097152
#define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"
#define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
#define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
#define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="
#define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"
#define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"
#define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="
#define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="
#define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
#define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
#define TEST_TAG_DESCRIPTION_PFX "comment:test_description="
#define TEST_TAG_RETVAL_PFX "comment:test_retval="
#define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="
#define TEST_TAG_AUXILIARY "comment:test_auxiliary"
#define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"
#define TEST_BTF_PATH "comment:test_btf_path="
#define TEST_TAG_ARCH "comment:test_arch="
#define TEST_TAG_JITED_PFX "comment:test_jited="
#define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
/* Warning: duplicated in bpf_misc.h */
#define POINTER_VALUE 0xcafe4all
#define TEST_DATA_LEN 64
#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
#define EFFICIENT_UNALIGNED_ACCESS 1
#else
#define EFFICIENT_UNALIGNED_ACCESS 0
#endif
static int sysctl_unpriv_disabled = -1;
enum mode {
PRIV = 1,
UNPRIV = 2
};
struct expect_msg {
const char *substr; /* substring match */
regex_t regex;
bool is_regex;
bool on_next_line;
};
struct expected_msgs {
struct expect_msg *patterns;
size_t cnt;
};
struct test_subspec {
char *name;
bool expect_failure;
struct expected_msgs expect_msgs;
struct expected_msgs expect_xlated;
struct expected_msgs jited;
int retval;
bool execute;
};
struct test_spec {
const char *prog_name;
struct test_subspec priv;
struct test_subspec unpriv;
const char *btf_custom_path;
int log_level;
int prog_flags;
int mode_mask;
int arch_mask;
bool auxiliary;
bool valid;
};
static int tester_init(struct test_loader *tester)
{
if (!tester->log_buf) {
tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
tester->log_buf = calloc(tester->log_buf_sz, 1);
if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
return -ENOMEM;
}
return 0;
}
void test_loader_fini(struct test_loader *tester)
{
if (!tester)
return;
free(tester->log_buf);
}
static void free_msgs(struct expected_msgs *msgs)
{
int i;
for (i = 0; i < msgs->cnt; i++)
if (msgs->patterns[i].is_regex)
regfree(&msgs->patterns[i].regex);
free(msgs->patterns);
msgs->patterns = NULL;
msgs->cnt = 0;
}
static void free_test_spec(struct test_spec *spec)
{
/* Deallocate expect_msgs arrays. */
free_msgs(&spec->priv.expect_msgs);
free_msgs(&spec->unpriv.expect_msgs);
free_msgs(&spec->priv.expect_xlated);
free_msgs(&spec->unpriv.expect_xlated);
free_msgs(&spec->priv.jited);
free_msgs(&spec->unpriv.jited);
free(spec->priv.name);
free(spec->unpriv.name);
spec->priv.name = NULL;
spec->unpriv.name = NULL;
}
/* Compiles regular expression matching pattern.
* Pattern has a special syntax:
*
* pattern := (<verbatim text> | regex)*
* regex := "{{" <posix extended regular expression> "}}"
*
* In other words, pattern is a verbatim text with inclusion
* of regular expressions enclosed in "{{" "}}" pairs.
* For example, pattern "foo{{[0-9]+}}" matches strings like
* "foo0", "foo007", etc.
*/
static int compile_regex(const char *pattern, regex_t *regex)
{
char err_buf[256], buf[256] = {}, *ptr, *buf_end;
const char *original_pattern = pattern;
bool in_regex = false;
int err;
buf_end = buf + sizeof(buf);
ptr = buf;
while (*pattern && ptr < buf_end - 2) {
if (!in_regex && str_has_pfx(pattern, "{{")) {
in_regex = true;
pattern += 2;
continue;
}
if (in_regex && str_has_pfx(pattern, "}}")) {
in_regex = false;
pattern += 2;
continue;
}
if (in_regex) {
*ptr++ = *pattern++;
continue;
}
/* list of characters that need escaping for extended posix regex */
if (strchr(".[]\\()*+?{}|^$", *pattern)) {
*ptr++ = '\\';
*ptr++ = *pattern++;
continue;
}
*ptr++ = *pattern++;
}
if (*pattern) {
PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
return -EINVAL;
}
if (in_regex) {
PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
return -EINVAL;
}
err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
if (err != 0) {
regerror(err, regex, err_buf, sizeof(err_buf));
PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
return -EINVAL;
}
return 0;
}
static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)
{
struct expect_msg *msg;
void *tmp;
int err;
tmp = realloc(msgs->patterns,
(1 + msgs->cnt) * sizeof(struct expect_msg));
if (!tmp) {
ASSERT_FAIL("failed to realloc memory for messages\n");
return -ENOMEM;
}
msgs->patterns = tmp;
msg = &msgs->patterns[msgs->cnt];
msg->on_next_line = on_next_line;
msg->substr = pattern;
msg->is_regex = false;
if (strstr(pattern, "{{")) {
err = compile_regex(pattern, &msg->regex);
if (err)
return err;
msg->is_regex = true;
}
msgs->cnt += 1;
return 0;
}
static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
{
struct expect_msg *msg;
int i, err;
for (i = 0; i < from->cnt; i++) {
msg = &from->patterns[i];
err = __push_msg(msg->substr, msg->on_next_line, to);
if (err)
return err;
}
return 0;
}
static int push_msg(const char *substr, struct expected_msgs *msgs)
{
return __push_msg(substr, false, msgs);
}
static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
{
int err;
if (strcmp(regex_str, "...") == 0) {
*on_next_line = false;
return 0;
}
err = __push_msg(regex_str, *on_next_line, msgs);
if (err)
return err;
*on_next_line = true;
return 0;
}
static int parse_int(const char *str, int *val, const char *name)
{
char *end;
long tmp;
errno = 0;
if (str_has_pfx(str, "0x"))
tmp = strtol(str + 2, &end, 16);
else
tmp = strtol(str, &end, 10);
if (errno || end[0] != '\0') {
PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
return -EINVAL;
}
*val = tmp;
return 0;
}
static int parse_retval(const char *str, int *val, const char *name)
{
struct {
char *name;
int val;
} named_values[] = {
{ "INT_MIN" , INT_MIN },
{ "POINTER_VALUE", POINTER_VALUE },
{ "TEST_DATA_LEN", TEST_DATA_LEN },
};
int i;
for (i = 0; i < ARRAY_SIZE(named_values); ++i) {
if (strcmp(str, named_values[i].name) != 0)
continue;
*val = named_values[i].val;
return 0;
}
return parse_int(str, val, name);
}
static void update_flags(int *flags, int flag, bool clear)
{
if (clear)
*flags &= ~flag;
else
*flags |= flag;
}
/* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.
* Used to parse btf_decl_tag values.
* Such values require unique prefix because compiler does not add
* same __attribute__((btf_decl_tag(...))) twice.
* Test suite uses two-component tags for such cases:
*
* <pfx> __COUNTER__ '='
*
* For example, two consecutive __msg tags '__msg("foo") __msg("foo")'
* would be encoded as:
*
* [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1
* [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1
*
* And the purpose of this function is to extract 'foo' from the above.
*/
static const char *skip_dynamic_pfx(const char *s, const char *pfx)
{
const char *msg;
if (strncmp(s, pfx, strlen(pfx)) != 0)
return NULL;
msg = s + strlen(pfx);
msg = strchr(msg, '=');
if (!msg)
return NULL;
return msg + 1;
}
enum arch {
ARCH_UNKNOWN = 0x1,
ARCH_X86_64 = 0x2,
ARCH_ARM64 = 0x4,
ARCH_RISCV64 = 0x8,
};
static int get_current_arch(void)
{
#if defined(__x86_64__)
return ARCH_X86_64;
#elif defined(__aarch64__)
return ARCH_ARM64;
#elif defined(__riscv) && __riscv_xlen == 64
return ARCH_RISCV64;
#endif
return ARCH_UNKNOWN;
}
/* Uses btf_decl_tag attributes to describe the expected test
* behavior, see bpf_misc.h for detailed description of each attribute
* and attribute combinations.
*/
static int parse_test_spec(struct test_loader *tester,
struct bpf_object *obj,
struct bpf_program *prog,
struct test_spec *spec)
{
const char *description = NULL;
bool has_unpriv_result = false;
bool has_unpriv_retval = false;
bool unpriv_xlated_on_next_line = true;
bool xlated_on_next_line = true;
bool unpriv_jit_on_next_line;
bool jit_on_next_line;
bool collect_jit = false;
int func_id, i, err = 0;
u32 arch_mask = 0;
struct btf *btf;
enum arch arch;
memset(spec, 0, sizeof(*spec));
spec->prog_name = bpf_program__name(prog);
spec->prog_flags = testing_prog_flags();
btf = bpf_object__btf(obj);
if (!btf) {
ASSERT_FAIL("BPF object has no BTF");
return -EINVAL;
}
func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
if (func_id < 0) {
ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
return -EINVAL;
}
for (i = 1; i < btf__type_cnt(btf); i++) {
const char *s, *val, *msg;
const struct btf_type *t;
bool clear;
int flags;
t = btf__type_by_id(btf, i);
if (!btf_is_decl_tag(t))
continue;
if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)
continue;
s = btf__str_by_offset(btf, t->name_off);
if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {
description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;
} else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {
spec->priv.expect_failure = true;
spec->mode_mask |= PRIV;
} else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
spec->priv.expect_failure = false;
spec->mode_mask |= PRIV;
} else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {
spec->unpriv.expect_failure = true;
spec->mode_mask |= UNPRIV;
has_unpriv_result = true;
} else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {
spec->unpriv.expect_failure = false;
spec->mode_mask |= UNPRIV;
has_unpriv_result = true;
} else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {
spec->auxiliary = true;
spec->mode_mask |= PRIV;
} else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {
spec->auxiliary = true;
spec->mode_mask |= UNPRIV;
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {
err = push_msg(msg, &spec->priv.expect_msgs);
if (err)
goto cleanup;
spec->mode_mask |= PRIV;
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {
err = push_msg(msg, &spec->unpriv.expect_msgs);
if (err)
goto cleanup;
spec->mode_mask |= UNPRIV;
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {
if (arch_mask == 0) {
PRINT_FAIL("__jited used before __arch_*");
goto cleanup;
}
if (collect_jit) {
err = push_disasm_msg(msg, &jit_on_next_line,
&spec->priv.jited);
if (err)
goto cleanup;
spec->mode_mask |= PRIV;
}
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {
if (arch_mask == 0) {
PRINT_FAIL("__unpriv_jited used before __arch_*");
goto cleanup;
}
if (collect_jit) {
err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
&spec->unpriv.jited);
if (err)
goto cleanup;
spec->mode_mask |= UNPRIV;
}
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {
err = push_disasm_msg(msg, &xlated_on_next_line,
&spec->priv.expect_xlated);
if (err)
goto cleanup;
spec->mode_mask |= PRIV;
} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {
err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
&spec->unpriv.expect_xlated);
if (err)
goto cleanup;
spec->mode_mask |= UNPRIV;
} else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {
val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;
err = parse_retval(val, &spec->priv.retval, "__retval");
if (err)
goto cleanup;
spec->priv.execute = true;
spec->mode_mask |= PRIV;
} else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {
val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;
err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
if (err)
goto cleanup;
spec->mode_mask |= UNPRIV;
spec->unpriv.execute = true;
has_unpriv_retval = true;
} else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
err = parse_int(val, &spec->log_level, "test log level");
if (err)
goto cleanup;
} else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
clear = val[0] == '!';
if (clear)
val++;
if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
} else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
} else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
} else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
} else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
} else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
} else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
} else /* assume numeric value */ {
err = parse_int(val, &flags, "test prog flags");
if (err)
goto cleanup;
update_flags(&spec->prog_flags, flags, clear);
}
} else if (str_has_pfx(s, TEST_TAG_ARCH)) {
val = s + sizeof(TEST_TAG_ARCH) - 1;
if (strcmp(val, "X86_64") == 0) {
arch = ARCH_X86_64;
} else if (strcmp(val, "ARM64") == 0) {
arch = ARCH_ARM64;
} else if (strcmp(val, "RISCV64") == 0) {
arch = ARCH_RISCV64;
} else {
PRINT_FAIL("bad arch spec: '%s'", val);
err = -EINVAL;
goto cleanup;
}
arch_mask |= arch;
collect_jit = get_current_arch() == arch;
unpriv_jit_on_next_line = true;
jit_on_next_line = true;
} else if (str_has_pfx(s, TEST_BTF_PATH)) {
spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;
}
}
spec->arch_mask = arch_mask ?: -1;
if (spec->mode_mask == 0)
spec->mode_mask = PRIV;
if (!description)
description = spec->prog_name;
if (spec->mode_mask & PRIV) {
spec->priv.name = strdup(description);
if (!spec->priv.name) {
PRINT_FAIL("failed to allocate memory for priv.name\n");
err = -ENOMEM;
goto cleanup;
}
}
if (spec->mode_mask & UNPRIV) {
int descr_len = strlen(description);
const char *suffix = " @unpriv";
char *name;
name = malloc(descr_len + strlen(suffix) + 1);
if (!name) {
PRINT_FAIL("failed to allocate memory for unpriv.name\n");
err = -ENOMEM;
goto cleanup;
}
strcpy(name, description);
strcpy(&name[descr_len], suffix);
spec->unpriv.name = name;
}
if (spec->mode_mask & (PRIV | UNPRIV)) {
if (!has_unpriv_result)
spec->unpriv.expect_failure = spec->priv.expect_failure;
if (!has_unpriv_retval) {
spec->unpriv.retval = spec->priv.retval;
spec->unpriv.execute = spec->priv.execute;
}
if (spec->unpriv.expect_msgs.cnt == 0)
clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
if (spec->unpriv.expect_xlated.cnt == 0)
clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
if (spec->unpriv.jited.cnt == 0)
clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
}
spec->valid = true;
return 0;
cleanup:
free_test_spec(spec);
return err;
}
static void prepare_case(struct test_loader *tester,
struct test_spec *spec,
struct bpf_object *obj,
struct bpf_program *prog)
{
int min_log_level = 0, prog_flags;
if (env.verbosity > VERBOSE_NONE)
min_log_level = 1;
if (env.verbosity > VERBOSE_VERY)
min_log_level = 2;
bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
/* Make sure we set at least minimal log level, unless test requires
* even higher level already. Make sure to preserve independent log
* level 4 (verifier stats), though.
*/
if ((spec->log_level & 3) < min_log_level)
bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
else
bpf_program__set_log_level(prog, spec->log_level);
prog_flags = bpf_program__flags(prog);
bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
tester->log_buf[0] = '\0';
}
static void emit_verifier_log(const char *log_buf, bool force)
{
if (!force && env.verbosity == VERBOSE_NONE)
return;
fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
}
static void emit_xlated(const char *xlated, bool force)
{
if (!force && env.verbosity == VERBOSE_NONE)
return;
fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
}
static void emit_jited(const char *jited, bool force)
{
if (!force && env.verbosity == VERBOSE_NONE)
return;
fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
}
static void validate_msgs(char *log_buf, struct expected_msgs *msgs,
void (*emit_fn)(const char *buf, bool force))
{
const char *log = log_buf, *prev_match;
regmatch_t reg_match[1];
int prev_match_line;
int match_line;
int i, j, err;
prev_match_line = -1;
match_line = 0;
prev_match = log;
for (i = 0; i < msgs->cnt; i++) {
struct expect_msg *msg = &msgs->patterns[i];
const char *match = NULL, *pat_status;
bool wrong_line = false;
if (!msg->is_regex) {
match = strstr(log, msg->substr);
if (match)
log = match + strlen(msg->substr);
} else {
err = regexec(&msg->regex, log, 1, reg_match, 0);
if (err == 0) {
match = log + reg_match[0].rm_so;
log += reg_match[0].rm_eo;
}
}
if (match) {
for (; prev_match < match; ++prev_match)
if (*prev_match == '\n')
++match_line;
wrong_line = msg->on_next_line && prev_match_line >= 0 &&
prev_match_line + 1 != match_line;
}
if (!match || wrong_line) {
PRINT_FAIL("expect_msg\n");
if (env.verbosity == VERBOSE_NONE)
emit_fn(log_buf, true /*force*/);
for (j = 0; j <= i; j++) {
msg = &msgs->patterns[j];
if (j < i)
pat_status = "MATCHED ";
else if (wrong_line)
pat_status = "WRONG LINE";
else
pat_status = "EXPECTED ";
msg = &msgs->patterns[j];
fprintf(stderr, "%s %s: '%s'\n",
pat_status,
msg->is_regex ? " REGEX" : "SUBSTR",
msg->substr);
}
if (wrong_line) {
fprintf(stderr,
"expecting match at line %d, actual match is at line %d\n",
prev_match_line + 1, match_line);
}
break;
}
prev_match_line = match_line;
}
}
struct cap_state {
__u64 old_caps;
bool initialized;
};
static int drop_capabilities(struct cap_state *caps)
{
const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
1ULL << CAP_PERFMON | 1ULL << CAP_BPF);
int err;
err = cap_disable_effective(caps_to_drop, &caps->old_caps);
if (err) {
PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
return err;
}
caps->initialized = true;
return 0;
}
static int restore_capabilities(struct cap_state *caps)
{
int err;
if (!caps->initialized)
return 0;
err = cap_enable_effective(caps->old_caps, NULL);
if (err)
PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
caps->initialized = false;
return err;
}
static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
{
if (sysctl_unpriv_disabled < 0)
sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
if (sysctl_unpriv_disabled)
return false;
if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
return false;
return true;
}
static bool is_unpriv_capable_map(struct bpf_map *map)
{
enum bpf_map_type type;
__u32 flags;
type = bpf_map__type(map);
switch (type) {
case BPF_MAP_TYPE_HASH:
case BPF_MAP_TYPE_PERCPU_HASH:
case BPF_MAP_TYPE_HASH_OF_MAPS:
flags = bpf_map__map_flags(map);
return !(flags & BPF_F_ZERO_SEED);
case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
case BPF_MAP_TYPE_ARRAY:
case BPF_MAP_TYPE_RINGBUF:
case BPF_MAP_TYPE_PROG_ARRAY:
case BPF_MAP_TYPE_CGROUP_ARRAY:
case BPF_MAP_TYPE_PERCPU_ARRAY:
case BPF_MAP_TYPE_USER_RINGBUF:
case BPF_MAP_TYPE_ARRAY_OF_MAPS:
case BPF_MAP_TYPE_CGROUP_STORAGE:
case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
return true;
default:
return false;
}
}
static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)
{
__u8 tmp_out[TEST_DATA_LEN << 2] = {};
__u8 tmp_in[TEST_DATA_LEN] = {};
int err, saved_errno;
LIBBPF_OPTS(bpf_test_run_opts, topts,
.data_in = tmp_in,
.data_size_in = sizeof(tmp_in),
.data_out = tmp_out,
.data_size_out = sizeof(tmp_out),
.repeat = 1,
);
if (empty_opts) {
memset(&topts, 0, sizeof(struct bpf_test_run_opts));
topts.sz = sizeof(struct bpf_test_run_opts);
}
err = bpf_prog_test_run_opts(fd_prog, &topts);
saved_errno = errno;
if (err) {
PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
saved_errno, strerror(saved_errno));
return err;
}
ASSERT_OK(0, "bpf_prog_test_run");
*retval = topts.retval;
return 0;
}
static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
{
if (!subspec->execute)
return false;
if (subspec->expect_failure)
return false;
if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
if (env.verbosity != VERBOSE_NONE)
printf("alignment prevents execution\n");
return false;
}
return true;
}
/* Get a disassembly of BPF program after verifier applies all rewrites */
static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
{
struct bpf_insn *insn_start = NULL, *insn, *insn_end;
__u32 insns_cnt = 0, i;
char buf[64];
FILE *out = NULL;
int err;
err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
if (!ASSERT_OK(err, "get_xlated_program"))
goto out;
out = fmemopen(text, text_sz, "w");
if (!ASSERT_OK_PTR(out, "open_memstream"))
goto out;
insn_end = insn_start + insns_cnt;
insn = insn_start;
while (insn < insn_end) {
i = insn - insn_start;
insn = disasm_insn(insn, buf, sizeof(buf));
fprintf(out, "%d: %s\n", i, buf);
}
fflush(out);
out:
free(insn_start);
if (out)
fclose(out);
return err;
}
/* this function is forced noinline and has short generic name to look better
* in test_progs output (in case of a failure)
*/
static noinline
void run_subtest(struct test_loader *tester,
struct bpf_object_open_opts *open_opts,
const void *obj_bytes,
size_t obj_byte_cnt,
struct test_spec *specs,
struct test_spec *spec,
bool unpriv)
{
struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
struct bpf_program *tprog = NULL, *tprog_iter;
struct bpf_link *link, *links[32] = {};
struct test_spec *spec_iter;
struct cap_state caps = {};
struct bpf_object *tobj;
struct bpf_map *map;
int retval, err, i;
int links_cnt = 0;
bool should_load;
if (!test__start_subtest(subspec->name))
return;
if ((get_current_arch() & spec->arch_mask) == 0) {
test__skip();
return;
}
if (unpriv) {
if (!can_execute_unpriv(tester, spec)) {
test__skip();
test__end_subtest();
return;
}
if (drop_capabilities(&caps)) {
test__end_subtest();
return;
}
}
/* Implicitly reset to NULL if next test case doesn't specify */
open_opts->btf_custom_path = spec->btf_custom_path;
tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
goto subtest_cleanup;
i = 0;
bpf_object__for_each_program(tprog_iter, tobj) {
spec_iter = &specs[i++];
should_load = false;
if (spec_iter->valid) {
if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
tprog = tprog_iter;
should_load = true;
}
if (spec_iter->auxiliary &&
spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
should_load = true;
}
bpf_program__set_autoload(tprog_iter, should_load);
}
prepare_case(tester, spec, tobj, tprog);
/* By default bpf_object__load() automatically creates all
* maps declared in the skeleton. Some map types are only
* allowed in priv mode. Disable autoload for such maps in
* unpriv mode.
*/
bpf_object__for_each_map(map, tobj)
bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
err = bpf_object__load(tobj);
if (subspec->expect_failure) {
if (!ASSERT_ERR(err, "unexpected_load_success")) {
emit_verifier_log(tester->log_buf, false /*force*/);
goto tobj_cleanup;
}
} else {
if (!ASSERT_OK(err, "unexpected_load_failure")) {
emit_verifier_log(tester->log_buf, true /*force*/);
goto tobj_cleanup;
}
}
emit_verifier_log(tester->log_buf, false /*force*/);
validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
if (subspec->expect_xlated.cnt) {
err = get_xlated_program_text(bpf_program__fd(tprog),
tester->log_buf, tester->log_buf_sz);
if (err)
goto tobj_cleanup;
emit_xlated(tester->log_buf, false /*force*/);
validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
}
if (subspec->jited.cnt) {
err = get_jited_program_text(bpf_program__fd(tprog),
tester->log_buf, tester->log_buf_sz);
if (err == -EOPNOTSUPP) {
printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
test__skip();
goto tobj_cleanup;
}
if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
goto tobj_cleanup;
emit_jited(tester->log_buf, false /*force*/);
validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
}
if (should_do_test_run(spec, subspec)) {
/* For some reason test_verifier executes programs
* with all capabilities restored. Do the same here.
*/
if (restore_capabilities(&caps))
goto tobj_cleanup;
/* Do bpf_map__attach_struct_ops() for each struct_ops map.
* This should trigger bpf_struct_ops->reg callback on kernel side.
*/
bpf_object__for_each_map(map, tobj) {
if (!bpf_map__autocreate(map) ||
bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
continue;
if (links_cnt >= ARRAY_SIZE(links)) {
PRINT_FAIL("too many struct_ops maps");
goto tobj_cleanup;
}
link = bpf_map__attach_struct_ops(map);
if (!link) {
PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
bpf_map__name(map), err);
goto tobj_cleanup;
}
links[links_cnt++] = link;
}
if (tester->pre_execution_cb) {
err = tester->pre_execution_cb(tobj);
if (err) {
PRINT_FAIL("pre_execution_cb failed: %d\n", err);
goto tobj_cleanup;
}
}
do_prog_test_run(bpf_program__fd(tprog), &retval,
bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);
if (retval != subspec->retval && subspec->retval != POINTER_VALUE) {
PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
goto tobj_cleanup;
}
/* redo bpf_map__attach_struct_ops for each test */
while (links_cnt > 0)
bpf_link__destroy(links[--links_cnt]);
}
tobj_cleanup:
while (links_cnt > 0)
bpf_link__destroy(links[--links_cnt]);
bpf_object__close(tobj);
subtest_cleanup:
test__end_subtest();
restore_capabilities(&caps);
}
static void process_subtest(struct test_loader *tester,
const char *skel_name,
skel_elf_bytes_fn elf_bytes_factory)
{
LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
struct test_spec *specs = NULL;
struct bpf_object *obj = NULL;
struct bpf_program *prog;
const void *obj_bytes;
int err, i, nr_progs;
size_t obj_byte_cnt;
if (tester_init(tester) < 0)
return; /* failed to initialize tester */
obj_bytes = elf_bytes_factory(&obj_byte_cnt);
obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
return;
nr_progs = 0;
bpf_object__for_each_program(prog, obj)
++nr_progs;
specs = calloc(nr_progs, sizeof(struct test_spec));
if (!ASSERT_OK_PTR(specs, "specs_alloc"))
return;
i = 0;
bpf_object__for_each_program(prog, obj) {
/* ignore tests for which we can't derive test specification */
err = parse_test_spec(tester, obj, prog, &specs[i++]);
if (err)
PRINT_FAIL("Can't parse test spec for program '%s'\n",
bpf_program__name(prog));
}
i = 0;
bpf_object__for_each_program(prog, obj) {
struct test_spec *spec = &specs[i++];
if (!spec->valid || spec->auxiliary)
continue;
if (spec->mode_mask & PRIV)
run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
specs, spec, false);
if (spec->mode_mask & UNPRIV)
run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
specs, spec, true);
}
for (i = 0; i < nr_progs; ++i)
free_test_spec(&specs[i]);
free(specs);
bpf_object__close(obj);
}
void test_loader__run_subtests(struct test_loader *tester,
const char *skel_name,
skel_elf_bytes_fn elf_bytes_factory)
{
/* see comment in run_subtest() for why we do this function nesting */
process_subtest(tester, skel_name, elf_bytes_factory);
}
Codebase Browser
linux