// SPDX-License-Identifier: GPL-2.0-only
/*
* eBPF JIT compiler
*
* Copyright 2016 Naveen N. Rao <[email protected]>
* IBM Corporation
*
* Based on the powerpc classic BPF JIT compiler by Matt Evans
*/
#include <linux/moduleloader.h>
#include <asm/cacheflush.h>
#include <asm/asm-compat.h>
#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/bpf.h>
#include <asm/kprobes.h>
#include <asm/code-patching.h>
#include "bpf_jit.h"
static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
{
memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
}
int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
{
if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
PPC_JMP(exit_addr);
} else if (ctx->alt_exit_addr) {
if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
return -1;
PPC_JMP(ctx->alt_exit_addr);
} else {
ctx->alt_exit_addr = ctx->idx * 4;
bpf_jit_build_epilogue(image, ctx);
}
return 0;
}
struct powerpc_jit_data {
/* address of rw header */
struct bpf_binary_header *hdr;
/* address of ro final header */
struct bpf_binary_header *fhdr;
u32 *addrs;
u8 *fimage;
u32 proglen;
struct codegen_context ctx;
};
bool bpf_jit_needs_zext(void)
{
return true;
}
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
u32 alloclen;
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
struct powerpc_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
struct bpf_binary_header *fhdr = NULL;
struct bpf_binary_header *hdr = NULL;
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
bool extra_pass = false;
u8 *fimage = NULL;
u32 *fcode_base;
u32 extable_len;
u32 fixup_len;
if (!fp->jit_requested)
return org_fp;
tmp_fp = bpf_jit_blind_constants(org_fp);
if (IS_ERR(tmp_fp))
return org_fp;
if (tmp_fp != org_fp) {
bpf_blinded = true;
fp = tmp_fp;
}
jit_data = fp->aux->jit_data;
if (!jit_data) {
jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
if (!jit_data) {
fp = org_fp;
goto out;
}
fp->aux->jit_data = jit_data;
}
flen = fp->len;
addrs = jit_data->addrs;
if (addrs) {
cgctx = jit_data->ctx;
/*
* JIT compiled to a writable location (image/code_base) first.
* It is then moved to the readonly final location (fimage/fcode_base)
* using instruction patching.
*/
fimage = jit_data->fimage;
fhdr = jit_data->fhdr;
proglen = jit_data->proglen;
hdr = jit_data->hdr;
image = (void *)hdr + ((void *)fimage - (void *)fhdr);
extra_pass = true;
/* During extra pass, ensure index is reset before repopulating extable entries */
cgctx.exentry_idx = 0;
goto skip_init_ctx;
}
addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
fp = org_fp;
goto out_addrs;
}
memset(&cgctx, 0, sizeof(struct codegen_context));
bpf_jit_init_reg_mapping(&cgctx);
/* Make sure that the stack is quadword aligned. */
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
/* Scouting faux-generate pass 0 */
if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
/* We hit something illegal or unsupported. */
fp = org_fp;
goto out_addrs;
}
/*
* If we have seen a tail call, we need a second pass.
* This is because bpf_jit_emit_common_epilogue() is called
* from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
* We also need a second pass if we ended up with too large
* a program so as to ensure BPF_EXIT branches are in range.
*/
if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
cgctx.idx = 0;
if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
fp = org_fp;
goto out_addrs;
}
}
bpf_jit_realloc_regs(&cgctx);
/*
* Pretend to build prologue, given the features we've seen. This will
* update ctgtx.idx as it pretends to output instructions, then we can
* calculate total size from idx.
*/
bpf_jit_build_prologue(NULL, &cgctx);
addrs[fp->len] = cgctx.idx * 4;
bpf_jit_build_epilogue(NULL, &cgctx);
fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
proglen = cgctx.idx * 4;
alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
bpf_jit_fill_ill_insns);
if (!fhdr) {
fp = org_fp;
goto out_addrs;
}
if (extable_len)
fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
skip_init_ctx:
code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
/* Code generation passes 1-2 */
for (pass = 1; pass < 3; pass++) {
/* Now build the prologue, body code & epilogue for real. */
cgctx.idx = 0;
cgctx.alt_exit_addr = 0;
bpf_jit_build_prologue(code_base, &cgctx);
if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
extra_pass)) {
bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
bpf_jit_binary_pack_free(fhdr, hdr);
fp = org_fp;
goto out_addrs;
}
bpf_jit_build_epilogue(code_base, &cgctx);
if (bpf_jit_enable > 1)
pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
proglen - (cgctx.idx * 4), cgctx.seen);
}
if (bpf_jit_enable > 1)
/*
* Note that we output the base address of the code_base
* rather than image, since opcodes are in code_base.
*/
bpf_jit_dump(flen, proglen, pass, code_base);
#ifdef CONFIG_PPC64_ELF_ABI_V1
/* Function descriptor nastiness: Address + TOC */
((u64 *)image)[0] = (u64)fcode_base;
((u64 *)image)[1] = local_paca->kernel_toc;
#endif
fp->bpf_func = (void *)fimage;
fp->jited = 1;
fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
if (!fp->is_func || extra_pass) {
if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
fp = org_fp;
goto out_addrs;
}
bpf_prog_fill_jited_linfo(fp, addrs);
out_addrs:
kfree(addrs);
kfree(jit_data);
fp->aux->jit_data = NULL;
} else {
jit_data->addrs = addrs;
jit_data->ctx = cgctx;
jit_data->proglen = proglen;
jit_data->fimage = fimage;
jit_data->fhdr = fhdr;
jit_data->hdr = hdr;
}
out:
if (bpf_blinded)
bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
return fp;
}
/*
* The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
* this function, as this only applies to BPF_PROBE_MEM, for now.
*/
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
struct codegen_context *ctx, int insn_idx, int jmp_off,
int dst_reg)
{
off_t offset;
unsigned long pc;
struct exception_table_entry *ex, *ex_entry;
u32 *fixup;
/* Populate extable entries only in the last pass */
if (pass != 2)
return 0;
if (!fp->aux->extable ||
WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
return -EINVAL;
/*
* Program is first written to image before copying to the
* final location (fimage). Accordingly, update in the image first.
* As all offsets used are relative, copying as is to the
* final location should be alright.
*/
pc = (unsigned long)&image[insn_idx];
ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
fixup = (void *)ex -
(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
(ctx->exentry_idx * BPF_FIXUP_LEN * 4);
fixup[0] = PPC_RAW_LI(dst_reg, 0);
if (IS_ENABLED(CONFIG_PPC32))
fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
fixup[BPF_FIXUP_LEN - 1] =
PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
ex_entry = &ex[ctx->exentry_idx];
offset = pc - (long)&ex_entry->insn;
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
return -ERANGE;
ex_entry->insn = offset;
offset = (long)fixup - (long)&ex_entry->fixup;
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
return -ERANGE;
ex_entry->fixup = offset;
ctx->exentry_idx++;
return 0;
}
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
int err;
if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
return ERR_PTR(-EINVAL);
mutex_lock(&text_mutex);
err = patch_instructions(dst, src, len, false);
mutex_unlock(&text_mutex);
return err ? ERR_PTR(err) : dst;
}
int bpf_arch_text_invalidate(void *dst, size_t len)
{
u32 insn = BREAKPOINT_INSTRUCTION;
int ret;
if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
return -EINVAL;
mutex_lock(&text_mutex);
ret = patch_instructions(dst, &insn, len, true);
mutex_unlock(&text_mutex);
return ret;
}
void bpf_jit_free(struct bpf_prog *fp)
{
if (fp->jited) {
struct powerpc_jit_data *jit_data = fp->aux->jit_data;
struct bpf_binary_header *hdr;
/*
* If we fail the final pass of JIT (from jit_subprogs),
* the program may not be finalized yet. Call finalize here
* before freeing it.
*/
if (jit_data) {
bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
kvfree(jit_data->addrs);
kfree(jit_data);
}
hdr = bpf_jit_binary_pack_hdr(fp);
bpf_jit_binary_pack_free(hdr, NULL);
WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
}
bpf_prog_unlock_free(fp);
}
bool bpf_jit_supports_kfunc_call(void)
{
return true;
}
bool bpf_jit_supports_far_kfunc_call(void)
{
return IS_ENABLED(CONFIG_PPC64);
}
Codebase Browser
linux