// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* Copyright (c) 2019 Facebook */ #ifdef __KERNEL__ #include <linux/bpf.h> #include <linux/btf.h> #include <linux/string.h> #include <linux/bpf_verifier.h> #include "relo_core.h" static const char *btf_kind_str(const struct btf_type *t) { … } static bool is_ldimm64_insn(struct bpf_insn *insn) { … } static const struct btf_type * skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id) { … } static const char *btf__name_by_offset(const struct btf *btf, u32 offset) { … } static s64 btf__resolve_size(const struct btf *btf, u32 type_id) { … } enum libbpf_print_level { … }; #undef pr_warn #undef pr_info #undef pr_debug #define pr_warn(fmt, log, ...) … #define pr_info(fmt, log, ...) … #define pr_debug(fmt, log, ...) … #define libbpf_print(level, fmt, ...) … #else #include <stdio.h> #include <string.h> #include <errno.h> #include <ctype.h> #include <linux/err.h> #include "libbpf.h" #include "bpf.h" #include "btf.h" #include "str_error.h" #include "libbpf_internal.h" #endif static bool is_flex_arr(const struct btf *btf, const struct bpf_core_accessor *acc, const struct btf_array *arr) { … } static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) { … } static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) { … } static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) { … } static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) { … } int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id, int level) { … } /* * Turn bpf_core_relo into a low- and high-level spec representation, * validating correctness along the way, as well as calculating resulting * field bit offset, specified by accessor string. Low-level spec captures * every single level of nestedness, including traversing anonymous * struct/union members. High-level one only captures semantically meaningful * "turning points": named fields and array indicies. * E.g., for this case: * * struct sample { * int __unimportant; * struct { * int __1; * int __2; * int a[7]; * }; * }; * * struct sample *s = ...; * * int x = &s->a[3]; // access string = '0:1:2:3' * * Low-level spec has 1:1 mapping with each element of access string (it's * just a parsed access string representation): [0, 1, 2, 3]. * * High-level spec will capture only 3 points: * - initial zero-index access by pointer (&s->... is the same as &s[0]...); * - field 'a' access (corresponds to '2' in low-level spec); * - array element #3 access (corresponds to '3' in low-level spec). * * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE, * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their * spec and raw_spec are kept empty. * * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access * string to specify enumerator's value index that need to be relocated. */ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, const struct bpf_core_relo *relo, struct bpf_core_spec *spec) { … } /* Check two types for compatibility for the purpose of field access * relocation. const/volatile/restrict and typedefs are skipped to ensure we * are relocating semantically compatible entities: * - any two STRUCTs/UNIONs are compatible and can be mixed; * - any two FWDs are compatible, if their names match (modulo flavor suffix); * - any two PTRs are always compatible; * - for ENUMs, names should be the same (ignoring flavor suffix) or at * least one of enums should be anonymous; * - for ENUMs, check sizes, names are ignored; * - for INT, size and signedness are ignored; * - any two FLOATs are always compatible; * - for ARRAY, dimensionality is ignored, element types are checked for * compatibility recursively; * - everything else shouldn't be ever a target of relocation. * These rules are not set in stone and probably will be adjusted as we get * more experience with using BPF CO-RE relocations. */ static int bpf_core_fields_are_compat(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id) { … } /* * Given single high-level named field accessor in local type, find * corresponding high-level accessor for a target type. Along the way, * maintain low-level spec for target as well. Also keep updating target * bit offset. * * Searching is performed through recursive exhaustive enumeration of all * fields of a struct/union. If there are any anonymous (embedded) * structs/unions, they are recursively searched as well. If field with * desired name is found, check compatibility between local and target types, * before returning result. * * 1 is returned, if field is found. * 0 is returned if no compatible field is found. * <0 is returned on error. */ static int bpf_core_match_member(const struct btf *local_btf, const struct bpf_core_accessor *local_acc, const struct btf *targ_btf, __u32 targ_id, struct bpf_core_spec *spec, __u32 *next_targ_id) { … } /* * Try to match local spec to a target type and, if successful, produce full * target spec (high-level, low-level + bit offset). */ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, const struct btf *targ_btf, __u32 targ_id, struct bpf_core_spec *targ_spec) { … } static int bpf_core_calc_field_relo(const char *prog_name, const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, __u64 *val, __u32 *field_sz, __u32 *type_id, bool *validate) { … } static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, __u64 *val, bool *validate) { … } static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, __u64 *val) { … } /* Calculate original and target relocation values, given local and target * specs and relocation kind. These values are calculated for each candidate. * If there are multiple candidates, resulting values should all be consistent * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity. * If instruction has to be poisoned, *poison will be set to true. */ static int bpf_core_calc_relo(const char *prog_name, const struct bpf_core_relo *relo, int relo_idx, const struct bpf_core_spec *local_spec, const struct bpf_core_spec *targ_spec, struct bpf_core_relo_res *res) { … } /* * Turn instruction for which CO_RE relocation failed into invalid one with * distinct signature. */ static void bpf_core_poison_insn(const char *prog_name, int relo_idx, int insn_idx, struct bpf_insn *insn) { … } static int insn_bpf_size_to_bytes(struct bpf_insn *insn) { … } static int insn_bytes_to_bpf_size(__u32 sz) { … } /* * Patch relocatable BPF instruction. * * Patched value is determined by relocation kind and target specification. * For existence relocations target spec will be NULL if field/type is not found. * Expected insn->imm value is determined using relocation kind and local * spec, and is checked before patching instruction. If actual insn->imm value * is wrong, bail out with error. * * Currently supported classes of BPF instruction are: * 1. rX = <imm> (assignment with immediate operand); * 2. rX += <imm> (arithmetic operations with immediate operand); * 3. rX = <imm64> (load with 64-bit immediate value); * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64}; * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64}; * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}. */ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, int insn_idx, const struct bpf_core_relo *relo, int relo_idx, const struct bpf_core_relo_res *res) { … } /* Output spec definition in the format: * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b */ int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec) { … } /* * Calculate CO-RE relocation target result. * * The outline and important points of the algorithm: * 1. For given local type, find corresponding candidate target types. * Candidate type is a type with the same "essential" name, ignoring * everything after last triple underscore (___). E.g., `sample`, * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates * for each other. Names with triple underscore are referred to as * "flavors" and are useful, among other things, to allow to * specify/support incompatible variations of the same kernel struct, which * might differ between different kernel versions and/or build * configurations. * * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C * converter, when deduplicated BTF of a kernel still contains more than * one different types with the same name. In that case, ___2, ___3, etc * are appended starting from second name conflict. But start flavors are * also useful to be defined "locally", in BPF program, to extract same * data from incompatible changes between different kernel * versions/configurations. For instance, to handle field renames between * kernel versions, one can use two flavors of the struct name with the * same common name and use conditional relocations to extract that field, * depending on target kernel version. * 2. For each candidate type, try to match local specification to this * candidate target type. Matching involves finding corresponding * high-level spec accessors, meaning that all named fields should match, * as well as all array accesses should be within the actual bounds. Also, * types should be compatible (see bpf_core_fields_are_compat for details). * 3. It is supported and expected that there might be multiple flavors * matching the spec. As long as all the specs resolve to the same set of * offsets across all candidates, there is no error. If there is any * ambiguity, CO-RE relocation will fail. This is necessary to accommodate * imperfection of BTF deduplication, which can cause slight duplication of * the same BTF type, if some directly or indirectly referenced (by * pointer) type gets resolved to different actual types in different * object files. If such a situation occurs, deduplicated BTF will end up * with two (or more) structurally identical types, which differ only in * types they refer to through pointer. This should be OK in most cases and * is not an error. * 4. Candidate types search is performed by linearly scanning through all * types in target BTF. It is anticipated that this is overall more * efficient memory-wise and not significantly worse (if not better) * CPU-wise compared to prebuilding a map from all local type names to * a list of candidate type names. It's also sped up by caching resolved * list of matching candidates per each local "root" type ID, that has at * least one bpf_core_relo associated with it. This list is shared * between multiple relocations for the same type ID and is updated as some * of the candidates are pruned due to structural incompatibility. */ int bpf_core_calc_relo_insn(const char *prog_name, const struct bpf_core_relo *relo, int relo_idx, const struct btf *local_btf, struct bpf_core_cand_list *cands, struct bpf_core_spec *specs_scratch, struct bpf_core_relo_res *targ_res) { … } static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off, const struct btf *targ_btf, size_t targ_name_off) { … } static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t, const struct btf *targ_btf, const struct btf_type *targ_t) { … } static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t, const struct btf *targ_btf, const struct btf_type *targ_t, bool behind_ptr, int level) { … } /* Check that two types "match". This function assumes that root types were * already checked for name match. * * The matching relation is defined as follows: * - modifiers and typedefs are stripped (and, hence, effectively ignored) * - generally speaking types need to be of same kind (struct vs. struct, union * vs. union, etc.) * - exceptions are struct/union behind a pointer which could also match a * forward declaration of a struct or union, respectively, and enum vs. * enum64 (see below) * Then, depending on type: * - integers: * - match if size and signedness match * - arrays & pointers: * - target types are recursively matched * - structs & unions: * - local members need to exist in target with the same name * - for each member we recursively check match unless it is already behind a * pointer, in which case we only check matching names and compatible kind * - enums: * - local variants have to have a match in target by symbolic name (but not * numeric value) * - size has to match (but enum may match enum64 and vice versa) * - function pointers: * - number and position of arguments in local type has to match target * - for each argument and the return value we recursively check match */ int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id, bool behind_ptr, int level) { … }
Codebase Browser
linux