// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Facebook */
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/bpf_local_storage.h>
#include <net/bpf_sk_storage.h>
#include <net/sock.h>
#include <uapi/linux/sock_diag.h>
#include <uapi/linux/btf.h>
#include <linux/rcupdate_trace.h>
DEFINE_BPF_STORAGE_CACHE(sk_cache);
static struct bpf_local_storage_data *
bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
{
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_map *smap;
sk_storage =
rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held());
if (!sk_storage)
return NULL;
smap = (struct bpf_local_storage_map *)map;
return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
}
static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
{
struct bpf_local_storage_data *sdata;
sdata = bpf_sk_storage_lookup(sk, map, false);
if (!sdata)
return -ENOENT;
bpf_selem_unlink(SELEM(sdata), false);
return 0;
}
/* Called by __sk_destruct() & bpf_sk_storage_clone() */
void bpf_sk_storage_free(struct sock *sk)
{
struct bpf_local_storage *sk_storage;
rcu_read_lock();
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage) {
rcu_read_unlock();
return;
}
bpf_local_storage_destroy(sk_storage);
rcu_read_unlock();
}
static void bpf_sk_storage_map_free(struct bpf_map *map)
{
bpf_local_storage_map_free(map, &sk_cache, NULL);
}
static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
{
return bpf_local_storage_map_alloc(attr, &sk_cache, false);
}
static int notsupp_get_next_key(struct bpf_map *map, void *key,
void *next_key)
{
return -ENOTSUPP;
}
static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
{
struct bpf_local_storage_data *sdata;
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
sdata = bpf_sk_storage_lookup(sock->sk, map, true);
sockfd_put(sock);
return sdata ? sdata->data : NULL;
}
return ERR_PTR(err);
}
static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
void *value, u64 map_flags)
{
struct bpf_local_storage_data *sdata;
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
sdata = bpf_local_storage_update(
sock->sk, (struct bpf_local_storage_map *)map, value,
map_flags, GFP_ATOMIC);
sockfd_put(sock);
return PTR_ERR_OR_ZERO(sdata);
}
return err;
}
static long bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
{
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
err = bpf_sk_storage_del(sock->sk, map);
sockfd_put(sock);
return err;
}
return err;
}
static struct bpf_local_storage_elem *
bpf_sk_storage_clone_elem(struct sock *newsk,
struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem)
{
struct bpf_local_storage_elem *copy_selem;
copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);
if (!copy_selem)
return NULL;
if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
SDATA(selem)->data, true);
else
copy_map_value(&smap->map, SDATA(copy_selem)->data,
SDATA(selem)->data);
return copy_selem;
}
int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
{
struct bpf_local_storage *new_sk_storage = NULL;
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_elem *selem;
int ret = 0;
RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
rcu_read_lock();
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage || hlist_empty(&sk_storage->list))
goto out;
hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
struct bpf_local_storage_elem *copy_selem;
struct bpf_local_storage_map *smap;
struct bpf_map *map;
smap = rcu_dereference(SDATA(selem)->smap);
if (!(smap->map.map_flags & BPF_F_CLONE))
continue;
/* Note that for lockless listeners adding new element
* here can race with cleanup in bpf_local_storage_map_free.
* Try to grab map refcnt to make sure that it's still
* alive and prevent concurrent removal.
*/
map = bpf_map_inc_not_zero(&smap->map);
if (IS_ERR(map))
continue;
copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
if (!copy_selem) {
ret = -ENOMEM;
bpf_map_put(map);
goto out;
}
if (new_sk_storage) {
bpf_selem_link_map(smap, copy_selem);
bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
} else {
ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
if (ret) {
bpf_selem_free(copy_selem, smap, true);
atomic_sub(smap->elem_size,
&newsk->sk_omem_alloc);
bpf_map_put(map);
goto out;
}
new_sk_storage =
rcu_dereference(copy_selem->local_storage);
}
bpf_map_put(map);
}
out:
rcu_read_unlock();
/* In case of an error, don't free anything explicitly here, the
* caller is responsible to call bpf_sk_storage_free.
*/
return ret;
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags, gfp_t, gfp_flags)
{
struct bpf_local_storage_data *sdata;
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
return (unsigned long)NULL;
sdata = bpf_sk_storage_lookup(sk, map, true);
if (sdata)
return (unsigned long)sdata->data;
if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
/* Cannot add new elem to a going away sk.
* Otherwise, the new elem may become a leak
* (and also other memory issues during map
* destruction).
*/
refcount_inc_not_zero(&sk->sk_refcnt)) {
sdata = bpf_local_storage_update(
sk, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST, gfp_flags);
/* sk must be a fullsock (guaranteed by verifier),
* so sock_gen_put() is unnecessary.
*/
sock_put(sk);
return IS_ERR(sdata) ?
(unsigned long)NULL : (unsigned long)sdata->data;
}
return (unsigned long)NULL;
}
BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
{
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (!sk || !sk_fullsock(sk))
return -EINVAL;
if (refcount_inc_not_zero(&sk->sk_refcnt)) {
int err;
err = bpf_sk_storage_del(sk, map);
sock_put(sk);
return err;
}
return -ENOENT;
}
static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
struct sock *sk = (struct sock *)owner;
int optmem_max;
optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
/* same check as in sock_kmalloc() */
if (size <= optmem_max &&
atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
atomic_add(size, &sk->sk_omem_alloc);
return 0;
}
return -ENOMEM;
}
static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
struct sock *sk = owner;
atomic_sub(size, &sk->sk_omem_alloc);
}
static struct bpf_local_storage __rcu **
bpf_sk_storage_ptr(void *owner)
{
struct sock *sk = owner;
return &sk->sk_bpf_storage;
}
const struct bpf_map_ops sk_storage_map_ops = {
.map_meta_equal = bpf_map_meta_equal,
.map_alloc_check = bpf_local_storage_map_alloc_check,
.map_alloc = bpf_sk_storage_map_alloc,
.map_free = bpf_sk_storage_map_free,
.map_get_next_key = notsupp_get_next_key,
.map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
.map_update_elem = bpf_fd_sk_storage_update_elem,
.map_delete_elem = bpf_fd_sk_storage_delete_elem,
.map_check_btf = bpf_local_storage_map_check_btf,
.map_btf_id = &bpf_local_storage_map_btf_id[0],
.map_local_storage_charge = bpf_sk_storage_charge,
.map_local_storage_uncharge = bpf_sk_storage_uncharge,
.map_owner_storage_ptr = bpf_sk_storage_ptr,
.map_mem_usage = bpf_local_storage_map_mem_usage,
};
const struct bpf_func_proto bpf_sk_storage_get_proto = {
.func = bpf_sk_storage_get,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
};
const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
.func = bpf_sk_storage_get,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
};
const struct bpf_func_proto bpf_sk_storage_delete_proto = {
.func = bpf_sk_storage_delete,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
};
static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
{
const struct btf *btf_vmlinux;
const struct btf_type *t;
const char *tname;
u32 btf_id;
if (prog->aux->dst_prog)
return false;
/* Ensure the tracing program is not tracing
* any bpf_sk_storage*() function and also
* use the bpf_sk_storage_(get|delete) helper.
*/
switch (prog->expected_attach_type) {
case BPF_TRACE_ITER:
case BPF_TRACE_RAW_TP:
/* bpf_sk_storage has no trace point */
return true;
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
btf_vmlinux = bpf_get_btf_vmlinux();
if (IS_ERR_OR_NULL(btf_vmlinux))
return false;
btf_id = prog->aux->attach_btf_id;
t = btf_type_by_id(btf_vmlinux, btf_id);
tname = btf_name_by_offset(btf_vmlinux, t->name_off);
return !!strncmp(tname, "bpf_sk_storage",
strlen("bpf_sk_storage"));
default:
return false;
}
return false;
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags, gfp_t, gfp_flags)
{
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (in_hardirq() || in_nmi())
return (unsigned long)NULL;
return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
gfp_flags);
}
BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
struct sock *, sk)
{
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (in_hardirq() || in_nmi())
return -EPERM;
return ____bpf_sk_storage_delete(map, sk);
}
const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
.func = bpf_sk_storage_get_tracing,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
.arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
.allowed = bpf_sk_storage_tracing_allowed,
};
const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
.func = bpf_sk_storage_delete_tracing,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
.arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
.allowed = bpf_sk_storage_tracing_allowed,
};
struct bpf_sk_storage_diag {
u32 nr_maps;
struct bpf_map *maps[];
};
/* The reply will be like:
* INET_DIAG_BPF_SK_STORAGES (nla_nest)
* SK_DIAG_BPF_STORAGE (nla_nest)
* SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
* SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
* SK_DIAG_BPF_STORAGE (nla_nest)
* SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
* SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
* ....
*/
static int nla_value_size(u32 value_size)
{
/* SK_DIAG_BPF_STORAGE (nla_nest)
* SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
* SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
*/
return nla_total_size(0) + nla_total_size(sizeof(u32)) +
nla_total_size_64bit(value_size);
}
void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
{
u32 i;
if (!diag)
return;
for (i = 0; i < diag->nr_maps; i++)
bpf_map_put(diag->maps[i]);
kfree(diag);
}
EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
const struct bpf_map *map)
{
u32 i;
for (i = 0; i < diag->nr_maps; i++) {
if (diag->maps[i] == map)
return true;
}
return false;
}
struct bpf_sk_storage_diag *
bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
{
struct bpf_sk_storage_diag *diag;
struct nlattr *nla;
u32 nr_maps = 0;
int rem, err;
/* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
* the map_alloc_check() side also does.
*/
if (!bpf_capable())
return ERR_PTR(-EPERM);
nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD,
nla_stgs, rem) {
if (nla_len(nla) != sizeof(u32))
return ERR_PTR(-EINVAL);
nr_maps++;
}
diag = kzalloc(struct_size(diag, maps, nr_maps), GFP_KERNEL);
if (!diag)
return ERR_PTR(-ENOMEM);
nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD,
nla_stgs, rem) {
int map_fd = nla_get_u32(nla);
struct bpf_map *map = bpf_map_get(map_fd);
if (IS_ERR(map)) {
err = PTR_ERR(map);
goto err_free;
}
if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
bpf_map_put(map);
err = -EINVAL;
goto err_free;
}
if (diag_check_dup(diag, map)) {
bpf_map_put(map);
err = -EEXIST;
goto err_free;
}
diag->maps[diag->nr_maps++] = map;
}
return diag;
err_free:
bpf_sk_storage_diag_free(diag);
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
{
struct nlattr *nla_stg, *nla_value;
struct bpf_local_storage_map *smap;
/* It cannot exceed max nlattr's payload */
BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
if (!nla_stg)
return -EMSGSIZE;
smap = rcu_dereference(sdata->smap);
if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
goto errout;
nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
smap->map.value_size,
SK_DIAG_BPF_STORAGE_PAD);
if (!nla_value)
goto errout;
if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
copy_map_value_locked(&smap->map, nla_data(nla_value),
sdata->data, true);
else
copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
nla_nest_end(skb, nla_stg);
return 0;
errout:
nla_nest_cancel(skb, nla_stg);
return -EMSGSIZE;
}
static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
int stg_array_type,
unsigned int *res_diag_size)
{
/* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
unsigned int diag_size = nla_total_size(0);
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_elem *selem;
struct bpf_local_storage_map *smap;
struct nlattr *nla_stgs;
unsigned int saved_len;
int err = 0;
rcu_read_lock();
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage || hlist_empty(&sk_storage->list)) {
rcu_read_unlock();
return 0;
}
nla_stgs = nla_nest_start(skb, stg_array_type);
if (!nla_stgs)
/* Continue to learn diag_size */
err = -EMSGSIZE;
saved_len = skb->len;
hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
smap = rcu_dereference(SDATA(selem)->smap);
diag_size += nla_value_size(smap->map.value_size);
if (nla_stgs && diag_get(SDATA(selem), skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
rcu_read_unlock();
if (nla_stgs) {
if (saved_len == skb->len)
nla_nest_cancel(skb, nla_stgs);
else
nla_nest_end(skb, nla_stgs);
}
if (diag_size == nla_total_size(0)) {
*res_diag_size = 0;
return 0;
}
*res_diag_size = diag_size;
return err;
}
int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
struct sock *sk, struct sk_buff *skb,
int stg_array_type,
unsigned int *res_diag_size)
{
/* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
unsigned int diag_size = nla_total_size(0);
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_data *sdata;
struct nlattr *nla_stgs;
unsigned int saved_len;
int err = 0;
u32 i;
*res_diag_size = 0;
/* No map has been specified. Dump all. */
if (!diag->nr_maps)
return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
res_diag_size);
rcu_read_lock();
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage || hlist_empty(&sk_storage->list)) {
rcu_read_unlock();
return 0;
}
nla_stgs = nla_nest_start(skb, stg_array_type);
if (!nla_stgs)
/* Continue to learn diag_size */
err = -EMSGSIZE;
saved_len = skb->len;
for (i = 0; i < diag->nr_maps; i++) {
sdata = bpf_local_storage_lookup(sk_storage,
(struct bpf_local_storage_map *)diag->maps[i],
false);
if (!sdata)
continue;
diag_size += nla_value_size(diag->maps[i]->value_size);
if (nla_stgs && diag_get(sdata, skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
rcu_read_unlock();
if (nla_stgs) {
if (saved_len == skb->len)
nla_nest_cancel(skb, nla_stgs);
else
nla_nest_end(skb, nla_stgs);
}
if (diag_size == nla_total_size(0)) {
*res_diag_size = 0;
return 0;
}
*res_diag_size = diag_size;
return err;
}
EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
struct bpf_iter_seq_sk_storage_map_info {
struct bpf_map *map;
unsigned int bucket_id;
unsigned skip_elems;
};
static struct bpf_local_storage_elem *
bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
struct bpf_local_storage_elem *prev_selem)
__acquires(RCU) __releases(RCU)
{
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_elem *selem;
u32 skip_elems = info->skip_elems;
struct bpf_local_storage_map *smap;
u32 bucket_id = info->bucket_id;
u32 i, count, n_buckets;
struct bpf_local_storage_map_bucket *b;
smap = (struct bpf_local_storage_map *)info->map;
n_buckets = 1U << smap->bucket_log;
if (bucket_id >= n_buckets)
return NULL;
/* try to find next selem in the same bucket */
selem = prev_selem;
count = 0;
while (selem) {
selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
struct bpf_local_storage_elem, map_node);
if (!selem) {
/* not found, unlock and go to the next bucket */
b = &smap->buckets[bucket_id++];
rcu_read_unlock();
skip_elems = 0;
break;
}
sk_storage = rcu_dereference(selem->local_storage);
if (sk_storage) {
info->skip_elems = skip_elems + count;
return selem;
}
count++;
}
for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
b = &smap->buckets[i];
rcu_read_lock();
count = 0;
hlist_for_each_entry_rcu(selem, &b->list, map_node) {
sk_storage = rcu_dereference(selem->local_storage);
if (sk_storage && count >= skip_elems) {
info->bucket_id = i;
info->skip_elems = count;
return selem;
}
count++;
}
rcu_read_unlock();
skip_elems = 0;
}
info->bucket_id = i;
info->skip_elems = 0;
return NULL;
}
static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
{
struct bpf_local_storage_elem *selem;
selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
if (!selem)
return NULL;
if (*pos == 0)
++*pos;
return selem;
}
static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
loff_t *pos)
{
struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
++*pos;
++info->skip_elems;
return bpf_sk_storage_map_seq_find_next(seq->private, v);
}
struct bpf_iter__bpf_sk_storage_map {
__bpf_md_ptr(struct bpf_iter_meta *, meta);
__bpf_md_ptr(struct bpf_map *, map);
__bpf_md_ptr(struct sock *, sk);
__bpf_md_ptr(void *, value);
};
DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
struct bpf_map *map, struct sock *sk,
void *value)
static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
struct bpf_local_storage_elem *selem)
{
struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
struct bpf_iter__bpf_sk_storage_map ctx = {};
struct bpf_local_storage *sk_storage;
struct bpf_iter_meta meta;
struct bpf_prog *prog;
int ret = 0;
meta.seq = seq;
prog = bpf_iter_get_info(&meta, selem == NULL);
if (prog) {
ctx.meta = &meta;
ctx.map = info->map;
if (selem) {
sk_storage = rcu_dereference(selem->local_storage);
ctx.sk = sk_storage->owner;
ctx.value = SDATA(selem)->data;
}
ret = bpf_iter_run_prog(prog, &ctx);
}
return ret;
}
static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
{
return __bpf_sk_storage_map_seq_show(seq, v);
}
static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
__releases(RCU)
{
if (!v)
(void)__bpf_sk_storage_map_seq_show(seq, v);
else
rcu_read_unlock();
}
static int bpf_iter_init_sk_storage_map(void *priv_data,
struct bpf_iter_aux_info *aux)
{
struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
bpf_map_inc_with_uref(aux->map);
seq_info->map = aux->map;
return 0;
}
static void bpf_iter_fini_sk_storage_map(void *priv_data)
{
struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
bpf_map_put_with_uref(seq_info->map);
}
static int bpf_iter_attach_map(struct bpf_prog *prog,
union bpf_iter_link_info *linfo,
struct bpf_iter_aux_info *aux)
{
struct bpf_map *map;
int err = -EINVAL;
if (!linfo->map.map_fd)
return -EBADF;
map = bpf_map_get_with_uref(linfo->map.map_fd);
if (IS_ERR(map))
return PTR_ERR(map);
if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
goto put_map;
if (prog->aux->max_rdwr_access > map->value_size) {
err = -EACCES;
goto put_map;
}
aux->map = map;
return 0;
put_map:
bpf_map_put_with_uref(map);
return err;
}
static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
{
bpf_map_put_with_uref(aux->map);
}
static const struct seq_operations bpf_sk_storage_map_seq_ops = {
.start = bpf_sk_storage_map_seq_start,
.next = bpf_sk_storage_map_seq_next,
.stop = bpf_sk_storage_map_seq_stop,
.show = bpf_sk_storage_map_seq_show,
};
static const struct bpf_iter_seq_info iter_seq_info = {
.seq_ops = &bpf_sk_storage_map_seq_ops,
.init_seq_private = bpf_iter_init_sk_storage_map,
.fini_seq_private = bpf_iter_fini_sk_storage_map,
.seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info),
};
static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
.target = "bpf_sk_storage_map",
.attach_target = bpf_iter_attach_map,
.detach_target = bpf_iter_detach_map,
.show_fdinfo = bpf_iter_map_show_fdinfo,
.fill_link_info = bpf_iter_map_fill_link_info,
.ctx_arg_info_size = 2,
.ctx_arg_info = {
{ offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
PTR_TO_BTF_ID_OR_NULL },
{ offsetof(struct bpf_iter__bpf_sk_storage_map, value),
PTR_TO_BUF | PTR_MAYBE_NULL },
},
.seq_info = &iter_seq_info,
};
static int __init bpf_sk_storage_map_iter_init(void)
{
bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
btf_sock_ids[BTF_SOCK_TYPE_SOCK];
return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
}
late_initcall(bpf_sk_storage_map_iter_init);
Codebase Browser
linux