jump_label.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0-only
/*
 * jump label support
 *
 * Copyright (C) 2009 Jason Baron <[email protected]>
 * Copyright (C) 2011 Peter Zijlstra
 *
 */
#include <linux/memory.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
#include <linux/static_key.h>
#include <linux/jump_label_ratelimit.h>
#include <linux/bug.h>
#include <linux/cpu.h>
#include <asm/sections.h>

/* mutex to protect coming/going of the jump_label table */
static DEFINE_MUTEX(jump_label_mutex);

void jump_label_lock(void)
{ … }

void jump_label_unlock(void)
{ … }

static int jump_label_cmp(const void *a, const void *b)
{ … }

static void jump_label_swap(void *a, void *b, int size)
{ … }

static void
jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
{ … }

static void jump_label_update(struct static_key *key);

/*
 * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
 * The use of 'atomic_read()' requires atomic.h and its problematic for some
 * kernel headers such as kernel.h and others. Since static_key_count() is not
 * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
 * to have it be a function here. Similarly, for 'static_key_enable()' and
 * 'static_key_disable()', which require bug.h. This should allow jump_label.h
 * to be included from most/all places for CONFIG_JUMP_LABEL.
 */
int static_key_count(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

/*
 * static_key_fast_inc_not_disabled - adds a user for a static key
 * @key: static key that must be already enabled
 *
 * The caller must make sure that the static key can't get disabled while
 * in this function. It doesn't patch jump labels, only adds a user to
 * an already enabled static key.
 *
 * Returns true if the increment was done. Unlike refcount_t the ref counter
 * is not saturated, but will fail to increment on overflow.
 */
bool static_key_fast_inc_not_disabled(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

bool static_key_slow_inc_cpuslocked(struct static_key *key)
{ … }

bool static_key_slow_inc(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_enable_cpuslocked(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_enable(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_disable_cpuslocked(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_disable(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

static bool static_key_slow_try_dec(struct static_key *key)
{ … }

static void __static_key_slow_dec_cpuslocked(struct static_key *key)
{ … }

static void __static_key_slow_dec(struct static_key *key)
{ … }

void jump_label_update_timeout(struct work_struct *work)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_slow_dec(struct static_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void static_key_slow_dec_cpuslocked(struct static_key *key)
{ … }

void __static_key_slow_dec_deferred(struct static_key *key,
				    struct delayed_work *work,
				    unsigned long timeout)
{ … }
EXPORT_SYMBOL_GPL(…);

void __static_key_deferred_flush(void *key, struct delayed_work *work)
{ … }
EXPORT_SYMBOL_GPL(…);

void jump_label_rate_limit(struct static_key_deferred *key,
		unsigned long rl)
{ … }
EXPORT_SYMBOL_GPL(…);

static int addr_conflict(struct jump_entry *entry, void *start, void *end)
{ … }

static int __jump_label_text_reserved(struct jump_entry *iter_start,
		struct jump_entry *iter_stop, void *start, void *end, bool init)
{ … }

#ifndef arch_jump_label_transform_static
static void arch_jump_label_transform_static(struct jump_entry *entry,
					     enum jump_label_type type)
{ … }
#endif

static inline struct jump_entry *static_key_entries(struct static_key *key)
{ … }

static inline bool static_key_type(struct static_key *key)
{ … }

static inline bool static_key_linked(struct static_key *key)
{ … }

static inline void static_key_clear_linked(struct static_key *key)
{ … }

static inline void static_key_set_linked(struct static_key *key)
{ … }

/***
 * A 'struct static_key' uses a union such that it either points directly
 * to a table of 'struct jump_entry' or to a linked list of modules which in
 * turn point to 'struct jump_entry' tables.
 *
 * The two lower bits of the pointer are used to keep track of which pointer
 * type is in use and to store the initial branch direction, we use an access
 * function which preserves these bits.
 */
static void static_key_set_entries(struct static_key *key,
				   struct jump_entry *entries)
{ … }

static enum jump_label_type jump_label_type(struct jump_entry *entry)
{ … }

static bool jump_label_can_update(struct jump_entry *entry, bool init)
{ … }

#ifndef HAVE_JUMP_LABEL_BATCH
static void __jump_label_update(struct static_key *key,
				struct jump_entry *entry,
				struct jump_entry *stop,
				bool init)
{
	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
		if (jump_label_can_update(entry, init))
			arch_jump_label_transform(entry, jump_label_type(entry));
	}
}
#else
static void __jump_label_update(struct static_key *key,
				struct jump_entry *entry,
				struct jump_entry *stop,
				bool init)
{ … }
#endif

void __init jump_label_init(void)
{ … }

static inline bool static_key_sealed(struct static_key *key)
{ … }

static inline void static_key_seal(struct static_key *key)
{ … }

void jump_label_init_ro(void)
{ … }

#ifdef CONFIG_MODULES

enum jump_label_type jump_label_init_type(struct jump_entry *entry)
{ … }

struct static_key_mod { … };

static inline struct static_key_mod *static_key_mod(struct static_key *key)
{ … }

/***
 * key->type and key->next are the same via union.
 * This sets key->next and preserves the type bits.
 *
 * See additional comments above static_key_set_entries().
 */
static void static_key_set_mod(struct static_key *key,
			       struct static_key_mod *mod)
{ … }

static int __jump_label_mod_text_reserved(void *start, void *end)
{ … }

static void __jump_label_mod_update(struct static_key *key)
{ … }

static int jump_label_add_module(struct module *mod)
{ … }

static void jump_label_del_module(struct module *mod)
{ … }

static int
jump_label_module_notify(struct notifier_block *self, unsigned long val,
			 void *data)
{ … }

static struct notifier_block jump_label_module_nb = …;

static __init int jump_label_init_module(void)
{ … }
early_initcall(jump_label_init_module);

#endif /* CONFIG_MODULES */

/***
 * jump_label_text_reserved - check if addr range is reserved
 * @start: start text addr
 * @end: end text addr
 *
 * checks if the text addr located between @start and @end
 * overlaps with any of the jump label patch addresses. Code
 * that wants to modify kernel text should first verify that
 * it does not overlap with any of the jump label addresses.
 * Caller must hold jump_label_mutex.
 *
 * returns 1 if there is an overlap, 0 otherwise
 */
int jump_label_text_reserved(void *start, void *end)
{ … }

static void jump_label_update(struct static_key *key)
{ … }

#ifdef CONFIG_STATIC_KEYS_SELFTEST
static DEFINE_STATIC_KEY_TRUE(sk_true);
static DEFINE_STATIC_KEY_FALSE(sk_false);

static __init int jump_label_test(void)
{ … }
early_initcall(jump_label_test);
#endif /* STATIC_KEYS_SELFTEST */
linux/kernel/jump_label.c
