linux/kernel/trace/rethook.c

// SPDX-License-Identifier: GPL-2.0

#define pr_fmt(fmt) …

#include <linux/bug.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/preempt.h>
#include <linux/rethook.h>
#include <linux/slab.h>

/* Return hook list (shadow stack by list) */

/*
 * This function is called from delayed_put_task_struct() when a task is
 * dead and cleaned up to recycle any kretprobe instances associated with
 * this task. These left over instances represent probed functions that
 * have been called but will never return.
 */
void rethook_flush_task(struct task_struct *tk)
{ … }

static void rethook_free_rcu(struct rcu_head *head)
{ … }

/**
 * rethook_stop() - Stop using a rethook.
 * @rh: the struct rethook to stop.
 *
 * Stop using a rethook to prepare for freeing it. If you want to wait for
 * all running rethook handler before calling rethook_free(), you need to
 * call this first and wait RCU, and call rethook_free().
 */
void rethook_stop(struct rethook *rh)
{ … }

/**
 * rethook_free() - Free struct rethook.
 * @rh: the struct rethook to be freed.
 *
 * Free the rethook. Before calling this function, user must ensure the
 * @rh::data is cleaned if needed (or, the handler can access it after
 * calling this function.) This function will set the @rh to be freed
 * after all rethook_node are freed (not soon). And the caller must
 * not touch @rh after calling this.
 */
void rethook_free(struct rethook *rh)
{ … }

static int rethook_init_node(void *nod, void *context)
{ … }

static int rethook_fini_pool(struct objpool_head *head, void *context)
{ … }

static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
{ … }

/**
 * rethook_alloc() - Allocate struct rethook.
 * @data: a data to pass the @handler when hooking the return.
 * @handler: the return hook callback function, must NOT be NULL
 * @size: node size: rethook node and additional data
 * @num: number of rethook nodes to be preallocated
 *
 * Allocate and initialize a new rethook with @data and @handler.
 * Return pointer of new rethook, or error codes for failures.
 *
 * Note that @handler == NULL means this rethook is going to be freed.
 */
struct rethook *rethook_alloc(void *data, rethook_handler_t handler,
			      int size, int num)
{ … }

static void free_rethook_node_rcu(struct rcu_head *head)
{ … }

/**
 * rethook_recycle() - return the node to rethook.
 * @node: The struct rethook_node to be returned.
 *
 * Return back the @node to @node::rethook. If the @node::rethook is already
 * marked as freed, this will free the @node.
 */
void rethook_recycle(struct rethook_node *node)
{ … }
NOKPROBE_SYMBOL(rethook_recycle);

/**
 * rethook_try_get() - get an unused rethook node.
 * @rh: The struct rethook which pools the nodes.
 *
 * Get an unused rethook node from @rh. If the node pool is empty, this
 * will return NULL. Caller must disable preemption.
 */
struct rethook_node *rethook_try_get(struct rethook *rh)
{ … }
NOKPROBE_SYMBOL(rethook_try_get);

/**
 * rethook_hook() - Hook the current function return.
 * @node: The struct rethook node to hook the function return.
 * @regs: The struct pt_regs for the function entry.
 * @mcount: True if this is called from mcount(ftrace) context.
 *
 * Hook the current running function return. This must be called when the
 * function entry (or at least @regs must be the registers of the function
 * entry.) @mcount is used for identifying the context. If this is called
 * from ftrace (mcount) callback, @mcount must be set true. If this is called
 * from the real function entry (e.g. kprobes) @mcount must be set false.
 * This is because the way to hook the function return depends on the context.
 */
void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
{ … }
NOKPROBE_SYMBOL(rethook_hook);

/* This assumes the 'tsk' is the current task or is not running. */
static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
					     struct llist_node **cur)
{ … }
NOKPROBE_SYMBOL(__rethook_find_ret_addr);

/**
 * rethook_find_ret_addr -- Find correct return address modified by rethook
 * @tsk: Target task
 * @frame: A frame pointer
 * @cur: a storage of the loop cursor llist_node pointer for next call
 *
 * Find the correct return address modified by a rethook on @tsk in unsigned
 * long type.
 * The @tsk must be 'current' or a task which is not running. @frame is a hint
 * to get the currect return address - which is compared with the
 * rethook::frame field. The @cur is a loop cursor for searching the
 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
 * first call, but '@cur' itself must NOT NULL.
 *
 * Returns found address value or zero if not found.
 */
unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
				    struct llist_node **cur)
{ … }
NOKPROBE_SYMBOL(rethook_find_ret_addr);

void __weak arch_rethook_fixup_return(struct pt_regs *regs,
				      unsigned long correct_ret_addr)
{ … }

/* This function will be called from each arch-defined trampoline. */
unsigned long rethook_trampoline_handler(struct pt_regs *regs,
					 unsigned long frame)
{ … }
NOKPROBE_SYMBOL(rethook_trampoline_handler);