package ebpf
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"math"
"path/filepath"
"runtime"
"strings"
"time"
"unsafe"
"github.com/cilium/ebpf/asm"
"github.com/cilium/ebpf/btf"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/unix"
)
// ErrNotSupported is returned whenever the kernel doesn't support a feature.
var ErrNotSupported = internal.ErrNotSupported
// ProgramID represents the unique ID of an eBPF program.
type ProgramID uint32
const (
// Number of bytes to pad the output buffer for BPF_PROG_TEST_RUN.
// This is currently the maximum of spare space allocated for SKB
// and XDP programs, and equal to XDP_PACKET_HEADROOM + NET_IP_ALIGN.
outputPad = 256 + 2
)
// DefaultVerifierLogSize is the default number of bytes allocated for the
// verifier log.
const DefaultVerifierLogSize = 64 * 1024
// maxVerifierLogSize is the maximum size of verifier log buffer the kernel
// will accept before returning EINVAL.
const maxVerifierLogSize = math.MaxUint32 >> 2
// ProgramOptions control loading a program into the kernel.
type ProgramOptions struct {
// Bitmap controlling the detail emitted by the kernel's eBPF verifier log.
// LogLevel-type values can be ORed together to request specific kinds of
// verifier output. See the documentation on [ebpf.LogLevel] for details.
//
// opts.LogLevel = (ebpf.LogLevelBranch | ebpf.LogLevelStats)
//
// If left to its default value, the program will first be loaded without
// verifier output enabled. Upon error, the program load will be repeated
// with LogLevelBranch and the given (or default) LogSize value.
//
// Setting this to a non-zero value will unconditionally enable the verifier
// log, populating the [ebpf.Program.VerifierLog] field on successful loads
// and including detailed verifier errors if the program is rejected. This
// will always allocate an output buffer, but will result in only a single
// attempt at loading the program.
LogLevel LogLevel
// Controls the output buffer size for the verifier log, in bytes. See the
// documentation on ProgramOptions.LogLevel for details about how this value
// is used.
//
// If this value is set too low to fit the verifier log, the resulting
// [ebpf.VerifierError]'s Truncated flag will be true, and the error string
// will also contain a hint to that effect.
//
// Defaults to DefaultVerifierLogSize.
LogSize int
// Disables the verifier log completely, regardless of other options.
LogDisabled bool
// Type information used for CO-RE relocations.
//
// This is useful in environments where the kernel BTF is not available
// (containers) or where it is in a non-standard location. Defaults to
// use the kernel BTF from a well-known location if nil.
KernelTypes *btf.Spec
}
// ProgramSpec defines a Program.
type ProgramSpec struct {
// Name is passed to the kernel as a debug aid. Must only contain
// alpha numeric and '_' characters.
Name string
// Type determines at which hook in the kernel a program will run.
Type ProgramType
// AttachType of the program, needed to differentiate allowed context
// accesses in some newer program types like CGroupSockAddr.
//
// Available on kernels 4.17 and later.
AttachType AttachType
// Name of a kernel data structure or function to attach to. Its
// interpretation depends on Type and AttachType.
AttachTo string
// The program to attach to. Must be provided manually.
AttachTarget *Program
// The name of the ELF section this program originated from.
SectionName string
Instructions asm.Instructions
// Flags is passed to the kernel and specifies additional program
// load attributes.
Flags uint32
// License of the program. Some helpers are only available if
// the license is deemed compatible with the GPL.
//
// See https://www.kernel.org/doc/html/latest/process/license-rules.html#id1
License string
// Version used by Kprobe programs.
//
// Deprecated on kernels 5.0 and later. Leave empty to let the library
// detect this value automatically.
KernelVersion uint32
// The byte order this program was compiled for, may be nil.
ByteOrder binary.ByteOrder
}
// Copy returns a copy of the spec.
func (ps *ProgramSpec) Copy() *ProgramSpec {
if ps == nil {
return nil
}
cpy := *ps
cpy.Instructions = make(asm.Instructions, len(ps.Instructions))
copy(cpy.Instructions, ps.Instructions)
return &cpy
}
// Tag calculates the kernel tag for a series of instructions.
//
// Use asm.Instructions.Tag if you need to calculate for non-native endianness.
func (ps *ProgramSpec) Tag() (string, error) {
return ps.Instructions.Tag(internal.NativeEndian)
}
// VerifierError is returned by [NewProgram] and [NewProgramWithOptions] if a
// program is rejected by the verifier.
//
// Use [errors.As] to access the error.
type VerifierError = internal.VerifierError
// Program represents BPF program loaded into the kernel.
//
// It is not safe to close a Program which is used by other goroutines.
type Program struct {
// Contains the output of the kernel verifier if enabled,
// otherwise it is empty.
VerifierLog string
fd *sys.FD
name string
pinnedPath string
typ ProgramType
}
// NewProgram creates a new Program.
//
// See [NewProgramWithOptions] for details.
//
// Returns a [VerifierError] containing the full verifier log if the program is
// rejected by the kernel.
func NewProgram(spec *ProgramSpec) (*Program, error) {
return NewProgramWithOptions(spec, ProgramOptions{})
}
// NewProgramWithOptions creates a new Program.
//
// Loading a program for the first time will perform
// feature detection by loading small, temporary programs.
//
// Returns a [VerifierError] containing the full verifier log if the program is
// rejected by the kernel.
func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
if spec == nil {
return nil, errors.New("can't load a program from a nil spec")
}
prog, err := newProgramWithOptions(spec, opts)
if errors.Is(err, asm.ErrUnsatisfiedMapReference) {
return nil, fmt.Errorf("cannot load program without loading its whole collection: %w", err)
}
return prog, err
}
func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
if len(spec.Instructions) == 0 {
return nil, errors.New("instructions cannot be empty")
}
if spec.Type == UnspecifiedProgram {
return nil, errors.New("can't load program of unspecified type")
}
if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian {
return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian)
}
if opts.LogSize < 0 {
return nil, errors.New("ProgramOptions.LogSize must be a positive value; disable verifier logs using ProgramOptions.LogDisabled")
}
// Kernels before 5.0 (6c4fc209fcf9 "bpf: remove useless version check for prog load")
// require the version field to be set to the value of the KERNEL_VERSION
// macro for kprobe-type programs.
// Overwrite Kprobe program version if set to zero or the magic version constant.
kv := spec.KernelVersion
if spec.Type == Kprobe && (kv == 0 || kv == internal.MagicKernelVersion) {
v, err := internal.KernelVersion()
if err != nil {
return nil, fmt.Errorf("detecting kernel version: %w", err)
}
kv = v.Kernel()
}
attr := &sys.ProgLoadAttr{
ProgType: sys.ProgType(spec.Type),
ProgFlags: spec.Flags,
ExpectedAttachType: sys.AttachType(spec.AttachType),
License: sys.NewStringPointer(spec.License),
KernVersion: kv,
}
if haveObjName() == nil {
attr.ProgName = sys.NewObjName(spec.Name)
}
insns := make(asm.Instructions, len(spec.Instructions))
copy(insns, spec.Instructions)
handle, fib, lib, err := btf.MarshalExtInfos(insns)
if err != nil && !errors.Is(err, btf.ErrNotSupported) {
return nil, fmt.Errorf("load ext_infos: %w", err)
}
if handle != nil {
defer handle.Close()
attr.ProgBtfFd = uint32(handle.FD())
attr.FuncInfoRecSize = btf.FuncInfoSize
attr.FuncInfoCnt = uint32(len(fib)) / btf.FuncInfoSize
attr.FuncInfo = sys.NewSlicePointer(fib)
attr.LineInfoRecSize = btf.LineInfoSize
attr.LineInfoCnt = uint32(len(lib)) / btf.LineInfoSize
attr.LineInfo = sys.NewSlicePointer(lib)
}
if err := applyRelocations(insns, opts.KernelTypes, spec.ByteOrder); err != nil {
return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
}
kconfig, err := resolveKconfigReferences(insns)
if err != nil {
return nil, fmt.Errorf("resolve .kconfig: %w", err)
}
defer kconfig.Close()
if err := fixupAndValidate(insns); err != nil {
return nil, err
}
handles, err := fixupKfuncs(insns)
if err != nil {
return nil, fmt.Errorf("fixing up kfuncs: %w", err)
}
defer handles.close()
if len(handles) > 0 {
fdArray := handles.fdArray()
attr.FdArray = sys.NewPointer(unsafe.Pointer(&fdArray[0]))
}
buf := bytes.NewBuffer(make([]byte, 0, insns.Size()))
err = insns.Marshal(buf, internal.NativeEndian)
if err != nil {
return nil, err
}
bytecode := buf.Bytes()
attr.Insns = sys.NewSlicePointer(bytecode)
attr.InsnCnt = uint32(len(bytecode) / asm.InstructionSize)
if spec.AttachTarget != nil {
targetID, err := findTargetInProgram(spec.AttachTarget, spec.AttachTo, spec.Type, spec.AttachType)
if err != nil {
return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
}
attr.AttachBtfId = targetID
attr.AttachBtfObjFd = uint32(spec.AttachTarget.FD())
defer runtime.KeepAlive(spec.AttachTarget)
} else if spec.AttachTo != "" {
module, targetID, err := findProgramTargetInKernel(spec.AttachTo, spec.Type, spec.AttachType)
if err != nil && !errors.Is(err, errUnrecognizedAttachType) {
// We ignore errUnrecognizedAttachType since AttachTo may be non-empty
// for programs that don't attach anywhere.
return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
}
attr.AttachBtfId = targetID
if module != nil {
attr.AttachBtfObjFd = uint32(module.FD())
defer module.Close()
}
}
if opts.LogSize == 0 {
opts.LogSize = DefaultVerifierLogSize
}
// The caller requested a specific verifier log level. Set up the log buffer.
var logBuf []byte
if !opts.LogDisabled && opts.LogLevel != 0 {
logBuf = make([]byte, opts.LogSize)
attr.LogLevel = opts.LogLevel
attr.LogSize = uint32(len(logBuf))
attr.LogBuf = sys.NewSlicePointer(logBuf)
}
fd, err := sys.ProgLoad(attr)
if err == nil {
return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil
}
// An error occurred loading the program, but the caller did not explicitly
// enable the verifier log. Re-run with branch-level verifier logs enabled to
// obtain more info. Preserve the original error to return it to the caller.
// An undersized log buffer will result in ENOSPC regardless of the underlying
// cause.
var err2 error
if !opts.LogDisabled && opts.LogLevel == 0 {
logBuf = make([]byte, opts.LogSize)
attr.LogLevel = LogLevelBranch
attr.LogSize = uint32(len(logBuf))
attr.LogBuf = sys.NewSlicePointer(logBuf)
_, err2 = sys.ProgLoad(attr)
}
switch {
case errors.Is(err, unix.EPERM):
if len(logBuf) > 0 && logBuf[0] == 0 {
// EPERM due to RLIMIT_MEMLOCK happens before the verifier, so we can
// check that the log is empty to reduce false positives.
return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
}
fallthrough
case errors.Is(err, unix.EINVAL):
if hasFunctionReferences(spec.Instructions) {
if err := haveBPFToBPFCalls(); err != nil {
return nil, fmt.Errorf("load program: %w", err)
}
}
if opts.LogSize > maxVerifierLogSize {
return nil, fmt.Errorf("load program: %w (ProgramOptions.LogSize exceeds maximum value of %d)", err, maxVerifierLogSize)
}
}
truncated := errors.Is(err, unix.ENOSPC) || errors.Is(err2, unix.ENOSPC)
return nil, internal.ErrorWithLog("load program", err, logBuf, truncated)
}
// NewProgramFromFD creates a program from a raw fd.
//
// You should not use fd after calling this function.
//
// Requires at least Linux 4.10.
func NewProgramFromFD(fd int) (*Program, error) {
f, err := sys.NewFD(fd)
if err != nil {
return nil, err
}
return newProgramFromFD(f)
}
// NewProgramFromID returns the program for a given id.
//
// Returns ErrNotExist, if there is no eBPF program with the given id.
func NewProgramFromID(id ProgramID) (*Program, error) {
fd, err := sys.ProgGetFdById(&sys.ProgGetFdByIdAttr{
Id: uint32(id),
})
if err != nil {
return nil, fmt.Errorf("get program by id: %w", err)
}
return newProgramFromFD(fd)
}
func newProgramFromFD(fd *sys.FD) (*Program, error) {
info, err := newProgramInfoFromFd(fd)
if err != nil {
fd.Close()
return nil, fmt.Errorf("discover program type: %w", err)
}
return &Program{"", fd, info.Name, "", info.Type}, nil
}
func (p *Program) String() string {
if p.name != "" {
return fmt.Sprintf("%s(%s)#%v", p.typ, p.name, p.fd)
}
return fmt.Sprintf("%s(%v)", p.typ, p.fd)
}
// Type returns the underlying type of the program.
func (p *Program) Type() ProgramType {
return p.typ
}
// Info returns metadata about the program.
//
// Requires at least 4.10.
func (p *Program) Info() (*ProgramInfo, error) {
return newProgramInfoFromFd(p.fd)
}
// Handle returns a reference to the program's type information in the kernel.
//
// Returns ErrNotSupported if the kernel has no BTF support, or if there is no
// BTF associated with the program.
func (p *Program) Handle() (*btf.Handle, error) {
info, err := p.Info()
if err != nil {
return nil, err
}
id, ok := info.BTFID()
if !ok {
return nil, fmt.Errorf("program %s: retrieve BTF ID: %w", p, ErrNotSupported)
}
return btf.NewHandleFromID(id)
}
// FD gets the file descriptor of the Program.
//
// It is invalid to call this function after Close has been called.
func (p *Program) FD() int {
return p.fd.Int()
}
// Clone creates a duplicate of the Program.
//
// Closing the duplicate does not affect the original, and vice versa.
//
// Cloning a nil Program returns nil.
func (p *Program) Clone() (*Program, error) {
if p == nil {
return nil, nil
}
dup, err := p.fd.Dup()
if err != nil {
return nil, fmt.Errorf("can't clone program: %w", err)
}
return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil
}
// Pin persists the Program on the BPF virtual file system past the lifetime of
// the process that created it
//
// Calling Pin on a previously pinned program will overwrite the path, except when
// the new path already exists. Re-pinning across filesystems is not supported.
//
// This requires bpffs to be mounted above fileName.
// See https://docs.cilium.io/en/stable/network/kubernetes/configuration/#mounting-bpffs-with-systemd
func (p *Program) Pin(fileName string) error {
if err := internal.Pin(p.pinnedPath, fileName, p.fd); err != nil {
return err
}
p.pinnedPath = fileName
return nil
}
// Unpin removes the persisted state for the Program from the BPF virtual filesystem.
//
// Failed calls to Unpin will not alter the state returned by IsPinned.
//
// Unpinning an unpinned Program returns nil.
func (p *Program) Unpin() error {
if err := internal.Unpin(p.pinnedPath); err != nil {
return err
}
p.pinnedPath = ""
return nil
}
// IsPinned returns true if the Program has a non-empty pinned path.
func (p *Program) IsPinned() bool {
return p.pinnedPath != ""
}
// Close the Program's underlying file descriptor, which could unload
// the program from the kernel if it is not pinned or attached to a
// kernel hook.
func (p *Program) Close() error {
if p == nil {
return nil
}
return p.fd.Close()
}
// Various options for Run'ing a Program
type RunOptions struct {
// Program's data input. Required field.
//
// The kernel expects at least 14 bytes input for an ethernet header for
// XDP and SKB programs.
Data []byte
// Program's data after Program has run. Caller must allocate. Optional field.
DataOut []byte
// Program's context input. Optional field.
Context interface{}
// Program's context after Program has run. Must be a pointer or slice. Optional field.
ContextOut interface{}
// Minimum number of times to run Program. Optional field. Defaults to 1.
//
// The program may be executed more often than this due to interruptions, e.g.
// when runtime.AllThreadsSyscall is invoked.
Repeat uint32
// Optional flags.
Flags uint32
// CPU to run Program on. Optional field.
// Note not all program types support this field.
CPU uint32
// Called whenever the syscall is interrupted, and should be set to testing.B.ResetTimer
// or similar. Typically used during benchmarking. Optional field.
//
// Deprecated: use [testing.B.ReportMetric] with unit "ns/op" instead.
Reset func()
}
// Test runs the Program in the kernel with the given input and returns the
// value returned by the eBPF program. outLen may be zero.
//
// Note: the kernel expects at least 14 bytes input for an ethernet header for
// XDP and SKB programs.
//
// This function requires at least Linux 4.12.
func (p *Program) Test(in []byte) (uint32, []byte, error) {
// Older kernels ignore the dataSizeOut argument when copying to user space.
// Combined with things like bpf_xdp_adjust_head() we don't really know what the final
// size will be. Hence we allocate an output buffer which we hope will always be large
// enough, and panic if the kernel wrote past the end of the allocation.
// See https://patchwork.ozlabs.org/cover/1006822/
var out []byte
if len(in) > 0 {
out = make([]byte, len(in)+outputPad)
}
opts := RunOptions{
Data: in,
DataOut: out,
Repeat: 1,
}
ret, _, err := p.run(&opts)
if err != nil {
return ret, nil, fmt.Errorf("test program: %w", err)
}
return ret, opts.DataOut, nil
}
// Run runs the Program in kernel with given RunOptions.
//
// Note: the same restrictions from Test apply.
func (p *Program) Run(opts *RunOptions) (uint32, error) {
ret, _, err := p.run(opts)
if err != nil {
return ret, fmt.Errorf("run program: %w", err)
}
return ret, nil
}
// Benchmark runs the Program with the given input for a number of times
// and returns the time taken per iteration.
//
// Returns the result of the last execution of the program and the time per
// run or an error. reset is called whenever the benchmark syscall is
// interrupted, and should be set to testing.B.ResetTimer or similar.
//
// This function requires at least Linux 4.12.
func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.Duration, error) {
if uint(repeat) > math.MaxUint32 {
return 0, 0, fmt.Errorf("repeat is too high")
}
opts := RunOptions{
Data: in,
Repeat: uint32(repeat),
Reset: reset,
}
ret, total, err := p.run(&opts)
if err != nil {
return ret, total, fmt.Errorf("benchmark program: %w", err)
}
return ret, total, nil
}
var haveProgRun = internal.NewFeatureTest("BPF_PROG_RUN", "4.12", func() error {
prog, err := NewProgram(&ProgramSpec{
// SocketFilter does not require privileges on newer kernels.
Type: SocketFilter,
Instructions: asm.Instructions{
asm.LoadImm(asm.R0, 0, asm.DWord),
asm.Return(),
},
License: "MIT",
})
if err != nil {
// This may be because we lack sufficient permissions, etc.
return err
}
defer prog.Close()
in := internal.EmptyBPFContext
attr := sys.ProgRunAttr{
ProgFd: uint32(prog.FD()),
DataSizeIn: uint32(len(in)),
DataIn: sys.NewSlicePointer(in),
}
err = sys.ProgRun(&attr)
switch {
case errors.Is(err, unix.EINVAL):
// Check for EINVAL specifically, rather than err != nil since we
// otherwise misdetect due to insufficient permissions.
return internal.ErrNotSupported
case errors.Is(err, unix.EINTR):
// We know that PROG_TEST_RUN is supported if we get EINTR.
return nil
case errors.Is(err, sys.ENOTSUPP):
// The first PROG_TEST_RUN patches shipped in 4.12 didn't include
// a test runner for SocketFilter. ENOTSUPP means PROG_TEST_RUN is
// supported, but not for the program type used in the probe.
return nil
}
return err
})
func (p *Program) run(opts *RunOptions) (uint32, time.Duration, error) {
if uint(len(opts.Data)) > math.MaxUint32 {
return 0, 0, fmt.Errorf("input is too long")
}
if err := haveProgRun(); err != nil {
return 0, 0, err
}
var ctxBytes []byte
if opts.Context != nil {
ctx := new(bytes.Buffer)
if err := binary.Write(ctx, internal.NativeEndian, opts.Context); err != nil {
return 0, 0, fmt.Errorf("cannot serialize context: %v", err)
}
ctxBytes = ctx.Bytes()
}
var ctxOut []byte
if opts.ContextOut != nil {
ctxOut = make([]byte, binary.Size(opts.ContextOut))
}
attr := sys.ProgRunAttr{
ProgFd: p.fd.Uint(),
DataSizeIn: uint32(len(opts.Data)),
DataSizeOut: uint32(len(opts.DataOut)),
DataIn: sys.NewSlicePointer(opts.Data),
DataOut: sys.NewSlicePointer(opts.DataOut),
Repeat: uint32(opts.Repeat),
CtxSizeIn: uint32(len(ctxBytes)),
CtxSizeOut: uint32(len(ctxOut)),
CtxIn: sys.NewSlicePointer(ctxBytes),
CtxOut: sys.NewSlicePointer(ctxOut),
Flags: opts.Flags,
Cpu: opts.CPU,
}
if attr.Repeat == 0 {
attr.Repeat = 1
}
retry:
for {
err := sys.ProgRun(&attr)
if err == nil {
break retry
}
if errors.Is(err, unix.EINTR) {
if attr.Repeat == 1 {
// Older kernels check whether enough repetitions have been
// executed only after checking for pending signals.
//
// run signal? done? run ...
//
// As a result we can get EINTR for repeat==1 even though
// the program was run exactly once. Treat this as a
// successful run instead.
//
// Since commit 607b9cc92bd7 ("bpf: Consolidate shared test timing code")
// the conditions are reversed:
// run done? signal? ...
break retry
}
if opts.Reset != nil {
opts.Reset()
}
continue retry
}
if errors.Is(err, sys.ENOTSUPP) {
return 0, 0, fmt.Errorf("kernel doesn't support running %s: %w", p.Type(), ErrNotSupported)
}
return 0, 0, err
}
if opts.DataOut != nil {
if int(attr.DataSizeOut) > cap(opts.DataOut) {
// Houston, we have a problem. The program created more data than we allocated,
// and the kernel wrote past the end of our buffer.
panic("kernel wrote past end of output buffer")
}
opts.DataOut = opts.DataOut[:int(attr.DataSizeOut)]
}
if len(ctxOut) != 0 {
b := bytes.NewReader(ctxOut)
if err := binary.Read(b, internal.NativeEndian, opts.ContextOut); err != nil {
return 0, 0, fmt.Errorf("failed to decode ContextOut: %v", err)
}
}
total := time.Duration(attr.Duration) * time.Nanosecond
return attr.Retval, total, nil
}
func unmarshalProgram(buf []byte) (*Program, error) {
if len(buf) != 4 {
return nil, errors.New("program id requires 4 byte value")
}
// Looking up an entry in a nested map or prog array returns an id,
// not an fd.
id := internal.NativeEndian.Uint32(buf)
return NewProgramFromID(ProgramID(id))
}
func marshalProgram(p *Program, length int) ([]byte, error) {
if length != 4 {
return nil, fmt.Errorf("can't marshal program to %d bytes", length)
}
buf := make([]byte, 4)
internal.NativeEndian.PutUint32(buf, p.fd.Uint())
return buf, nil
}
// LoadPinnedProgram loads a Program from a BPF file.
//
// Requires at least Linux 4.11.
func LoadPinnedProgram(fileName string, opts *LoadPinOptions) (*Program, error) {
fd, err := sys.ObjGet(&sys.ObjGetAttr{
Pathname: sys.NewStringPointer(fileName),
FileFlags: opts.Marshal(),
})
if err != nil {
return nil, err
}
info, err := newProgramInfoFromFd(fd)
if err != nil {
_ = fd.Close()
return nil, fmt.Errorf("info for %s: %w", fileName, err)
}
var progName string
if haveObjName() == nil {
progName = info.Name
} else {
progName = filepath.Base(fileName)
}
return &Program{"", fd, progName, fileName, info.Type}, nil
}
// SanitizeName replaces all invalid characters in name with replacement.
// Passing a negative value for replacement will delete characters instead
// of replacing them. Use this to automatically generate valid names for maps
// and programs at runtime.
//
// The set of allowed characters depends on the running kernel version.
// Dots are only allowed as of kernel 5.2.
func SanitizeName(name string, replacement rune) string {
return strings.Map(func(char rune) rune {
if invalidBPFObjNameChar(char) {
return replacement
}
return char
}, name)
}
// ProgramGetNextID returns the ID of the next eBPF program.
//
// Returns ErrNotExist, if there is no next eBPF program.
func ProgramGetNextID(startID ProgramID) (ProgramID, error) {
attr := &sys.ProgGetNextIdAttr{Id: uint32(startID)}
return ProgramID(attr.NextId), sys.ProgGetNextId(attr)
}
// BindMap binds map to the program and is only released once program is released.
//
// This may be used in cases where metadata should be associated with the program
// which otherwise does not contain any references to the map.
func (p *Program) BindMap(m *Map) error {
attr := &sys.ProgBindMapAttr{
ProgFd: uint32(p.FD()),
MapFd: uint32(m.FD()),
}
return sys.ProgBindMap(attr)
}
var errUnrecognizedAttachType = errors.New("unrecognized attach type")
// find an attach target type in the kernel.
//
// name, progType and attachType determine which type we need to attach to.
//
// The attach target may be in a loaded kernel module.
// In that case the returned handle will be non-nil.
// The caller is responsible for closing the handle.
//
// Returns errUnrecognizedAttachType if the combination of progType and attachType
// is not recognised.
func findProgramTargetInKernel(name string, progType ProgramType, attachType AttachType) (*btf.Handle, btf.TypeID, error) {
type match struct {
p ProgramType
a AttachType
}
var (
typeName, featureName string
target btf.Type
)
switch (match{progType, attachType}) {
case match{LSM, AttachLSMMac}:
typeName = "bpf_lsm_" + name
featureName = name + " LSM hook"
target = (*btf.Func)(nil)
case match{Tracing, AttachTraceIter}:
typeName = "bpf_iter_" + name
featureName = name + " iterator"
target = (*btf.Func)(nil)
case match{Tracing, AttachTraceFEntry}:
typeName = name
featureName = fmt.Sprintf("fentry %s", name)
target = (*btf.Func)(nil)
case match{Tracing, AttachTraceFExit}:
typeName = name
featureName = fmt.Sprintf("fexit %s", name)
target = (*btf.Func)(nil)
case match{Tracing, AttachModifyReturn}:
typeName = name
featureName = fmt.Sprintf("fmod_ret %s", name)
target = (*btf.Func)(nil)
case match{Tracing, AttachTraceRawTp}:
typeName = fmt.Sprintf("btf_trace_%s", name)
featureName = fmt.Sprintf("raw_tp %s", name)
target = (*btf.Typedef)(nil)
default:
return nil, 0, errUnrecognizedAttachType
}
spec, err := btf.LoadKernelSpec()
if err != nil {
return nil, 0, fmt.Errorf("load kernel spec: %w", err)
}
spec, module, err := findTargetInKernel(spec, typeName, &target)
if errors.Is(err, btf.ErrNotFound) {
return nil, 0, &internal.UnsupportedFeatureError{Name: featureName}
}
// See cilium/ebpf#894. Until we can disambiguate between equally-named kernel
// symbols, we should explicitly refuse program loads. They will not reliably
// do what the caller intended.
if errors.Is(err, btf.ErrMultipleMatches) {
return nil, 0, fmt.Errorf("attaching to ambiguous kernel symbol is not supported: %w", err)
}
if err != nil {
return nil, 0, fmt.Errorf("find target for %s: %w", featureName, err)
}
id, err := spec.TypeID(target)
return module, id, err
}
// findTargetInKernel attempts to find a named type in the current kernel.
//
// target will point at the found type after a successful call. Searches both
// vmlinux and any loaded modules.
//
// Returns a non-nil handle if the type was found in a module, or btf.ErrNotFound
// if the type wasn't found at all.
func findTargetInKernel(kernelSpec *btf.Spec, typeName string, target *btf.Type) (*btf.Spec, *btf.Handle, error) {
err := kernelSpec.TypeByName(typeName, target)
if errors.Is(err, btf.ErrNotFound) {
spec, module, err := findTargetInModule(kernelSpec, typeName, target)
if err != nil {
return nil, nil, fmt.Errorf("find target in modules: %w", err)
}
return spec, module, nil
}
if err != nil {
return nil, nil, fmt.Errorf("find target in vmlinux: %w", err)
}
return kernelSpec, nil, err
}
// findTargetInModule attempts to find a named type in any loaded module.
//
// base must contain the kernel's types and is used to parse kmod BTF. Modules
// are searched in the order they were loaded.
//
// Returns btf.ErrNotFound if the target can't be found in any module.
func findTargetInModule(base *btf.Spec, typeName string, target *btf.Type) (*btf.Spec, *btf.Handle, error) {
it := new(btf.HandleIterator)
defer it.Handle.Close()
for it.Next() {
info, err := it.Handle.Info()
if err != nil {
return nil, nil, fmt.Errorf("get info for BTF ID %d: %w", it.ID, err)
}
if !info.IsModule() {
continue
}
spec, err := it.Handle.Spec(base)
if err != nil {
return nil, nil, fmt.Errorf("parse types for module %s: %w", info.Name, err)
}
err = spec.TypeByName(typeName, target)
if errors.Is(err, btf.ErrNotFound) {
continue
}
if err != nil {
return nil, nil, fmt.Errorf("lookup type in module %s: %w", info.Name, err)
}
return spec, it.Take(), nil
}
if err := it.Err(); err != nil {
return nil, nil, fmt.Errorf("iterate modules: %w", err)
}
return nil, nil, btf.ErrNotFound
}
// find an attach target type in a program.
//
// Returns errUnrecognizedAttachType.
func findTargetInProgram(prog *Program, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
type match struct {
p ProgramType
a AttachType
}
var typeName string
switch (match{progType, attachType}) {
case match{Extension, AttachNone}:
typeName = name
default:
return 0, errUnrecognizedAttachType
}
btfHandle, err := prog.Handle()
if err != nil {
return 0, fmt.Errorf("load target BTF: %w", err)
}
defer btfHandle.Close()
spec, err := btfHandle.Spec(nil)
if err != nil {
return 0, err
}
var targetFunc *btf.Func
err = spec.TypeByName(typeName, &targetFunc)
if err != nil {
return 0, fmt.Errorf("find target %s: %w", typeName, err)
}
return spec.TypeID(targetFunc)
}
Codebase Browser
kubernetes