linux/kernel/tracepoint.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2008-2014 Mathieu Desnoyers
 */
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/jhash.h>
#include <linux/list.h>
#include <linux/rcupdate.h>
#include <linux/tracepoint.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/static_key.h>

enum tp_func_state { … };

extern tracepoint_ptr_t __start___tracepoints_ptrs[];
extern tracepoint_ptr_t __stop___tracepoints_ptrs[];

DEFINE_SRCU(…);
EXPORT_SYMBOL_GPL(…);

enum tp_transition_sync { … };

struct tp_transition_snapshot { … };

/* Protected by tracepoints_mutex */
static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];

static void tp_rcu_get_state(enum tp_transition_sync sync)
{ … }

static void tp_rcu_cond_sync(enum tp_transition_sync sync)
{ … }

/* Set to 1 to enable tracepoint debug output */
static const int tracepoint_debug;

#ifdef CONFIG_MODULES
/*
 * Tracepoint module list mutex protects the local module list.
 */
static DEFINE_MUTEX(tracepoint_module_list_mutex);

/* Local list of struct tp_module */
static LIST_HEAD(tracepoint_module_list);
#endif /* CONFIG_MODULES */

/*
 * tracepoints_mutex protects the builtin and module tracepoints.
 * tracepoints_mutex nests inside tracepoint_module_list_mutex.
 */
static DEFINE_MUTEX(tracepoints_mutex);

static struct rcu_head *early_probes;
static bool ok_to_free_tracepoints;

/*
 * Note about RCU :
 * It is used to delay the free of multiple probes array until a quiescent
 * state is reached.
 */
struct tp_probes { … };

/* Called in removal of a func but failed to allocate a new tp_funcs */
static void tp_stub_func(void)
{ … }

static inline void *allocate_probes(int count)
{ … }

static void srcu_free_old_probes(struct rcu_head *head)
{ … }

static void rcu_free_old_probes(struct rcu_head *head)
{ … }

static __init int release_early_probes(void)
{ … }

/* SRCU is initialized at core_initcall */
postcore_initcall(release_early_probes);

static inline void release_probes(struct tracepoint_func *old)
{ … }

static void debug_print_probes(struct tracepoint_func *funcs)
{ … }

static struct tracepoint_func *
func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
	 int prio)
{ … }

static void *func_remove(struct tracepoint_func **funcs,
		struct tracepoint_func *tp_func)
{ … }

/*
 * Count the number of functions (enum tp_func_state) in a tp_funcs array.
 */
static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
{ … }

static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
{ … }

/*
 * Add the probe function to a tracepoint.
 */
static int tracepoint_add_func(struct tracepoint *tp,
			       struct tracepoint_func *func, int prio,
			       bool warn)
{ … }

/*
 * Remove a probe function from a tracepoint.
 * Note: only waiting an RCU period after setting elem->call to the empty
 * function insures that the original callback is not used anymore. This insured
 * by preempt_disable around the call site.
 */
static int tracepoint_remove_func(struct tracepoint *tp,
		struct tracepoint_func *func)
{ … }

/**
 * tracepoint_probe_register_prio_may_exist -  Connect a probe to a tracepoint with priority
 * @tp: tracepoint
 * @probe: probe handler
 * @data: tracepoint data
 * @prio: priority of this function over other registered functions
 *
 * Same as tracepoint_probe_register_prio() except that it will not warn
 * if the tracepoint is already registered.
 */
int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe,
					     void *data, int prio)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * tracepoint_probe_register_prio -  Connect a probe to a tracepoint with priority
 * @tp: tracepoint
 * @probe: probe handler
 * @data: tracepoint data
 * @prio: priority of this function over other registered functions
 *
 * Returns 0 if ok, error value on error.
 * Note: if @tp is within a module, the caller is responsible for
 * unregistering the probe before the module is gone. This can be
 * performed either with a tracepoint module going notifier, or from
 * within module exit functions.
 */
int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe,
				   void *data, int prio)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * tracepoint_probe_register -  Connect a probe to a tracepoint
 * @tp: tracepoint
 * @probe: probe handler
 * @data: tracepoint data
 *
 * Returns 0 if ok, error value on error.
 * Note: if @tp is within a module, the caller is responsible for
 * unregistering the probe before the module is gone. This can be
 * performed either with a tracepoint module going notifier, or from
 * within module exit functions.
 */
int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 * @tp: tracepoint
 * @probe: probe function pointer
 * @data: tracepoint data
 *
 * Returns 0 if ok, error value on error.
 */
int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
{ … }
EXPORT_SYMBOL_GPL(…);

static void for_each_tracepoint_range(
		tracepoint_ptr_t *begin, tracepoint_ptr_t *end,
		void (*fct)(struct tracepoint *tp, void *priv),
		void *priv)
{ … }

#ifdef CONFIG_MODULES
bool trace_module_has_bad_taint(struct module *mod)
{ … }

static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);

/**
 * register_tracepoint_module_notifier - register tracepoint coming/going notifier
 * @nb: notifier block
 *
 * Notifiers registered with this function are called on module
 * coming/going with the tracepoint_module_list_mutex held.
 * The notifier block callback should expect a "struct tp_module" data
 * pointer.
 */
int register_tracepoint_module_notifier(struct notifier_block *nb)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * unregister_tracepoint_module_notifier - unregister tracepoint coming/going notifier
 * @nb: notifier block
 *
 * The notifier block callback should expect a "struct tp_module" data
 * pointer.
 */
int unregister_tracepoint_module_notifier(struct notifier_block *nb)
{ … }
EXPORT_SYMBOL_GPL(…);

/*
 * Ensure the tracer unregistered the module's probes before the module
 * teardown is performed. Prevents leaks of probe and data pointers.
 */
static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv)
{ … }

static int tracepoint_module_coming(struct module *mod)
{ … }

static void tracepoint_module_going(struct module *mod)
{ … }

static int tracepoint_module_notify(struct notifier_block *self,
		unsigned long val, void *data)
{ … }

static struct notifier_block tracepoint_module_nb = …;

static __init int init_tracepoints(void)
{ … }
__initcall(init_tracepoints);
#endif /* CONFIG_MODULES */

/**
 * for_each_kernel_tracepoint - iteration on all kernel tracepoints
 * @fct: callback
 * @priv: private data
 */
void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
		void *priv)
{ … }
EXPORT_SYMBOL_GPL(…);

#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS

/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
static int sys_tracepoint_refcount;

int syscall_regfunc(void)
{ … }

void syscall_unregfunc(void)
{ … }
#endif