linux/kernel/bpf/bpf_task_storage.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2020 Facebook
 * Copyright 2020 Google LLC.
 */

#include <linux/pid.h>
#include <linux/sched.h>
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/bpf_local_storage.h>
#include <linux/filter.h>
#include <uapi/linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/fdtable.h>
#include <linux/rcupdate_trace.h>

DEFINE_BPF_STORAGE_CACHE(…);

static DEFINE_PER_CPU(int, bpf_task_storage_busy);

static void bpf_task_storage_lock(void)
{ … }

static void bpf_task_storage_unlock(void)
{ … }

static bool bpf_task_storage_trylock(void)
{ … }

static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
{ … }

static struct bpf_local_storage_data *
task_storage_lookup(struct task_struct *task, struct bpf_map *map,
		    bool cacheit_lockit)
{ … }

void bpf_task_storage_free(struct task_struct *task)
{ … }

static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
{ … }

static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
					     void *value, u64 map_flags)
{ … }

static int task_storage_delete(struct task_struct *task, struct bpf_map *map,
			       bool nobusy)
{ … }

static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)
{ … }

/* Called by bpf_task_storage_get*() helpers */
static void *__bpf_task_storage_get(struct bpf_map *map,
				    struct task_struct *task, void *value,
				    u64 flags, gfp_t gfp_flags, bool nobusy)
{ … }

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *,
	   task, void *, value, u64, flags, gfp_t, gfp_flags)
{ … }

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
	   task, void *, value, u64, flags, gfp_t, gfp_flags)
{ … }

BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *,
	   task)
{ … }

BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *,
	   task)
{ … }

static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
{ … }

static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
{ … }

static void task_storage_map_free(struct bpf_map *map)
{ … }

BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map)
const struct bpf_map_ops task_storage_map_ops = …;

const struct bpf_func_proto bpf_task_storage_get_recur_proto = …;

const struct bpf_func_proto bpf_task_storage_get_proto = …;

const struct bpf_func_proto bpf_task_storage_delete_recur_proto = …;

const struct bpf_func_proto bpf_task_storage_delete_proto = …;