#ifndef _LINUX_ALLOC_TAG_H
#define _LINUX_ALLOC_TAG_H
#include <linux/bug.h>
#include <linux/codetag.h>
#include <linux/container_of.h>
#include <linux/preempt.h>
#include <asm/percpu.h>
#include <linux/cpumask.h>
#include <linux/smp.h>
#include <linux/static_key.h>
#include <linux/irqflags.h>
struct alloc_tag_counters { … };
struct alloc_tag { … } __aligned(…);
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
#define CODETAG_EMPTY …
static inline bool is_codetag_empty(union codetag_ref *ref)
{
return ref->ct == CODETAG_EMPTY;
}
static inline void set_codetag_empty(union codetag_ref *ref)
{
if (ref)
ref->ct = CODETAG_EMPTY;
}
#else
static inline bool is_codetag_empty(union codetag_ref *ref) { … }
static inline void set_codetag_empty(union codetag_ref *ref) { … }
#endif
#ifdef CONFIG_MEM_ALLOC_PROFILING
struct codetag_bytes {
struct codetag *ct;
s64 bytes;
};
size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);
static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
{
return container_of(ct, struct alloc_tag, ct);
}
#ifdef ARCH_NEEDS_WEAK_PER_CPU
DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
#define DEFINE_ALLOC_TAG …
#else
#define DEFINE_ALLOC_TAG …
#endif
DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
mem_alloc_profiling_key);
static inline bool mem_alloc_profiling_enabled(void)
{
return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
&mem_alloc_profiling_key);
}
static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
{
struct alloc_tag_counters v = { 0, 0 };
struct alloc_tag_counters *counter;
int cpu;
for_each_possible_cpu(cpu) {
counter = per_cpu_ptr(tag->counters, cpu);
v.bytes += counter->bytes;
v.calls += counter->calls;
}
return v;
}
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
{
WARN_ONCE(ref && ref->ct,
"alloc_tag was not cleared (got tag for %s:%u)\n",
ref->ct->filename, ref->ct->lineno);
WARN_ONCE(!tag, "current->alloc_tag not set\n");
}
static inline void alloc_tag_sub_check(union codetag_ref *ref)
{
WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
}
#else
static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
#endif
static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
{
alloc_tag_add_check(ref, tag);
if (!ref || !tag)
return;
ref->ct = &tag->ct;
}
static inline void alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
{
__alloc_tag_ref_set(ref, tag);
this_cpu_inc(tag->counters->calls);
}
static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
{
alloc_tag_ref_set(ref, tag);
this_cpu_add(tag->counters->bytes, bytes);
}
static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
{
struct alloc_tag *tag;
alloc_tag_sub_check(ref);
if (!ref || !ref->ct)
return;
if (is_codetag_empty(ref)) {
ref->ct = NULL;
return;
}
tag = ct_to_alloc_tag(ref->ct);
this_cpu_sub(tag->counters->bytes, bytes);
this_cpu_dec(tag->counters->calls);
ref->ct = NULL;
}
#define alloc_tag_record …
#else
#define DEFINE_ALLOC_TAG(_alloc_tag) …
static inline bool mem_alloc_profiling_enabled(void) { … }
static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
size_t bytes) { … }
static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) { … }
#define alloc_tag_record(p) …
#endif
#define alloc_hooks_tag(_tag, _do_alloc) …
#define alloc_hooks(_do_alloc) …
#endif