linux/kernel/bpf/arena.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/err.h>
#include <linux/btf_ids.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>

/*
 * bpf_arena is a sparsely populated shared memory region between bpf program and
 * user space process.
 *
 * For example on x86-64 the values could be:
 * user_vm_start 7f7d26200000     // picked by mmap()
 * kern_vm_start ffffc90001e69000 // picked by get_vm_area()
 * For user space all pointers within the arena are normal 8-byte addresses.
 * In this example 7f7d26200000 is the address of the first page (pgoff=0).
 * The bpf program will access it as: kern_vm_start + lower_32bit_of_user_ptr
 * (u32)7f7d26200000 -> 26200000
 * hence
 * ffffc90001e69000 + 26200000 == ffffc90028069000 is "pgoff=0" within 4Gb
 * kernel memory region.
 *
 * BPF JITs generate the following code to access arena:
 *   mov eax, eax  // eax has lower 32-bit of user pointer
 *   mov word ptr [rax + r12 + off], bx
 * where r12 == kern_vm_start and off is s16.
 * Hence allocate 4Gb + GUARD_SZ/2 on each side.
 *
 * Initially kernel vm_area and user vma are not populated.
 * User space can fault-in any address which will insert the page
 * into kernel and user vma.
 * bpf program can allocate a page via bpf_arena_alloc_pages() kfunc
 * which will insert it into kernel vm_area.
 * The later fault-in from user space will populate that page into user vma.
 */

/* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */
#define GUARD_SZ …
#define KERN_VM_SZ …

struct bpf_arena { … };

u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena)
{ … }

u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)
{ … }

static long arena_map_peek_elem(struct bpf_map *map, void *value)
{ … }

static long arena_map_push_elem(struct bpf_map *map, void *value, u64 flags)
{ … }

static long arena_map_pop_elem(struct bpf_map *map, void *value)
{ … }

static long arena_map_delete_elem(struct bpf_map *map, void *value)
{ … }

static int arena_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
{ … }

static long compute_pgoff(struct bpf_arena *arena, long uaddr)
{ … }

static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
{ … }

static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data)
{ … }

static void arena_map_free(struct bpf_map *map)
{ … }

static void *arena_map_lookup_elem(struct bpf_map *map, void *key)
{ … }

static long arena_map_update_elem(struct bpf_map *map, void *key,
				  void *value, u64 flags)
{ … }

static int arena_map_check_btf(const struct bpf_map *map, const struct btf *btf,
			       const struct btf_type *key_type, const struct btf_type *value_type)
{ … }

static u64 arena_map_mem_usage(const struct bpf_map *map)
{ … }

struct vma_list { … };

static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)
{ … }

static void arena_vm_open(struct vm_area_struct *vma)
{ … }

static void arena_vm_close(struct vm_area_struct *vma)
{ … }

#define MT_ENTRY …

static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
{ … }

static const struct vm_operations_struct arena_vm_ops = …;

static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long addr,
					     unsigned long len, unsigned long pgoff,
					     unsigned long flags)
{ … }

static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
{ … }

static int arena_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
{ … }

BTF_ID_LIST_SINGLE(bpf_arena_map_btf_ids, struct, bpf_arena)
const struct bpf_map_ops arena_map_ops = …;

static u64 clear_lo32(u64 val)
{ … }

/*
 * Allocate pages and vmap them into kernel vmalloc area.
 * Later the pages will be mmaped into user space vma.
 */
static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt, int node_id)
{ … }

/*
 * If page is present in vmalloc area, unmap it from vmalloc area,
 * unmap it from all user space vma-s,
 * and free it.
 */
static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
{ … }

static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
{ … }

__bpf_kfunc_start_defs();

__bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt,
					int node_id, u64 flags)
{ … }

__bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt)
{ … }
__bpf_kfunc_end_defs();

BTF_KFUNCS_START(arena_kfuncs)
BTF_ID_FLAGS(…)
BTF_ID_FLAGS(…)
BTF_KFUNCS_END(…)

static const struct btf_kfunc_id_set common_kfunc_set = …;

static int __init kfunc_init(void)
{ … }
late_initcall(kfunc_init);