// SPDX-License-Identifier: GPL-2.0 /* * Infrastructure to took into function calls and returns. * Copyright (c) 2008-2009 Frederic Weisbecker <[email protected]> * Mostly borrowed from function tracer which * is Copyright (c) Steven Rostedt <[email protected]> * * Highly modified by Steven Rostedt (VMware). */ #include <linux/bits.h> #include <linux/jump_label.h> #include <linux/suspend.h> #include <linux/ftrace.h> #include <linux/static_call.h> #include <linux/slab.h> #include <trace/events/sched.h> #include "ftrace_internal.h" #include "trace.h" /* * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame */ #define FGRAPH_FRAME_SIZE … #define FGRAPH_FRAME_OFFSET … /* * On entry to a function (via function_graph_enter()), a new fgraph frame * (ftrace_ret_stack) is pushed onto the stack as well as a word that * holds a bitmask and a type (called "bitmap"). The bitmap is defined as: * * bits: 0 - 9 offset in words from the previous ftrace_ret_stack * * bits: 10 - 11 Type of storage * 0 - reserved * 1 - bitmap of fgraph_array index * 2 - reserved data * * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP): * bits: 12 - 27 The bitmap of fgraph_ops fgraph_array index * That is, it's a bitmask of 0-15 (16 bits) * where if a corresponding ops in the fgraph_array[] * expects a callback from the return of the function * it's corresponding bit will be set. * * * The top of the ret_stack (when not empty) will always have a reference * word that points to the last fgraph frame that was saved. * * For reserved data: * bits: 12 - 17 The size in words that is stored * bits: 18 - 23 The index of fgraph_array, which shows who is stored * * That is, at the end of function_graph_enter, if the first and forth * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called * on the return of the function being traced, and the forth fgraph_ops * stored two words of data, this is what will be on the task's shadow * ret_stack: (the stack grows upward) * * ret_stack[SHADOW_STACK_OFFSET] * | SHADOW_STACK_TASK_VARS(ret_stack)[15] | * ... * | SHADOW_STACK_TASK_VARS(ret_stack)[0] | * ret_stack[SHADOW_STACK_MAX_OFFSET] * ... * | | <- task->curr_ret_stack * +--------------------------------------------+ * | (3 << 12) | (3 << 10) | FGRAPH_FRAME_OFFSET| * | *or put another way* | * | (3 << FGRAPH_DATA_INDEX_SHIFT)| \ | This is for fgraph_ops[3]. * | ((2 - 1) << FGRAPH_DATA_SHIFT)| \ | The data size is 2 words. * | (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT)| \ | * | (offset2:FGRAPH_FRAME_OFFSET+3) | <- the offset2 is from here * +--------------------------------------------+ ( It is 4 words from the ret_stack) * | STORED DATA WORD 2 | * | STORED DATA WORD 1 | * +--------------------------------------------+ * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET| * | *or put another way* | * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \ | * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ | * | (offset1:FGRAPH_FRAME_OFFSET) | <- the offset1 is from here * +--------------------------------------------+ * | struct ftrace_ret_stack | * | (stores the saved ret pointer) | <- the offset points here * +--------------------------------------------+ * | (X) | (N) | ( N words away from * | | previous ret_stack) * ... * ret_stack[0] * * If a backtrace is required, and the real return pointer needs to be * fetched, then it looks at the task's curr_ret_stack offset, if it * is greater than zero (reserved, or right before popped), it would mask * the value by FGRAPH_FRAME_OFFSET_MASK to get the offset of the * ftrace_ret_stack structure stored on the shadow stack. */ /* * The following is for the top word on the stack: * * FGRAPH_FRAME_OFFSET (0-9) holds the offset delta to the fgraph frame * FGRAPH_TYPE (10-11) holds the type of word this is. * (RESERVED or BITMAP) */ #define FGRAPH_FRAME_OFFSET_BITS … #define FGRAPH_FRAME_OFFSET_MASK … #define FGRAPH_TYPE_BITS … #define FGRAPH_TYPE_MASK … #define FGRAPH_TYPE_SHIFT … enum { … }; /* * For BITMAP type: * FGRAPH_INDEX (12-27) bits holding the gops index wanting return callback called */ #define FGRAPH_INDEX_BITS … #define FGRAPH_INDEX_MASK … #define FGRAPH_INDEX_SHIFT … /* * For DATA type: * FGRAPH_DATA (12-17) bits hold the size of data (in words) * FGRAPH_INDEX (18-23) bits hold the index for which gops->idx the data is for * * Note: * data_size == 0 means 1 word, and 31 (=2^5 - 1) means 32 words. */ #define FGRAPH_DATA_BITS … #define FGRAPH_DATA_MASK … #define FGRAPH_DATA_SHIFT … #define FGRAPH_MAX_DATA_SIZE … #define FGRAPH_DATA_INDEX_BITS … #define FGRAPH_DATA_INDEX_MASK … #define FGRAPH_DATA_INDEX_SHIFT … #define FGRAPH_MAX_INDEX … #define FGRAPH_ARRAY_SIZE … /* * SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack * SHADOW_STACK_OFFSET: The size in long words of the shadow stack * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added */ #define SHADOW_STACK_SIZE … #define SHADOW_STACK_OFFSET … /* Leave on a buffer at the end */ #define SHADOW_STACK_MAX_OFFSET … /* RET_STACK(): Return the frame from a given @offset from task @t */ #define RET_STACK(t, offset) … /* * Each fgraph_ops has a reservered unsigned long at the end (top) of the * ret_stack to store task specific state. */ #define SHADOW_STACK_TASK_VARS(ret_stack) … DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE]; static unsigned long fgraph_array_bitmask; /* LRU index table for fgraph_array */ static int fgraph_lru_table[FGRAPH_ARRAY_SIZE]; static int fgraph_lru_next; static int fgraph_lru_last; /* Initialize fgraph_lru_table with unused index */ static void fgraph_lru_init(void) { … } /* Release the used index to the LRU table */ static int fgraph_lru_release_index(int idx) { … } /* Allocate a new index from LRU table */ static int fgraph_lru_alloc_index(void) { … } /* Get the offset to the fgraph frame from a ret_stack value */ static inline int __get_offset(unsigned long val) { … } /* Get the type of word from a ret_stack value */ static inline int __get_type(unsigned long val) { … } /* Get the data_index for a DATA type ret_stack word */ static inline int __get_data_index(unsigned long val) { … } /* Get the data_size for a DATA type ret_stack word */ static inline int __get_data_size(unsigned long val) { … } /* Get the word from the ret_stack at @offset */ static inline unsigned long get_fgraph_entry(struct task_struct *t, int offset) { … } /* Get the FRAME_OFFSET from the word from the @offset on ret_stack */ static inline int get_frame_offset(struct task_struct *t, int offset) { … } /* For BITMAP type: get the bitmask from the @offset at ret_stack */ static inline unsigned long get_bitmap_bits(struct task_struct *t, int offset) { … } /* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */ static inline void set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) { … } /* For DATA type: get the data saved under the ret_stack word at @offset */ static inline void *get_data_type_data(struct task_struct *t, int offset) { … } /* Create the ret_stack word for a DATA type */ static inline unsigned long make_data_type_val(int idx, int size, int offset) { … } /* ftrace_graph_entry set to this to tell some archs to run function graph */ static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops) { … } /* ftrace_graph_return set to this to tell some archs to run function graph */ static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops) { … } static void ret_stack_set_task_var(struct task_struct *t, int idx, long val) { … } static unsigned long * ret_stack_get_task_var(struct task_struct *t, int idx) { … } static void ret_stack_init_task_vars(unsigned long *ret_stack) { … } /** * fgraph_reserve_data - Reserve storage on the task's ret_stack * @idx: The index of fgraph_array * @size_bytes: The size in bytes to reserve * * Reserves space of up to FGRAPH_MAX_DATA_SIZE bytes on the * task's ret_stack shadow stack, for a given fgraph_ops during * the entryfunc() call. If entryfunc() returns zero, the storage * is discarded. An entryfunc() can only call this once per iteration. * The fgraph_ops retfunc() can retrieve this stored data with * fgraph_retrieve_data(). * * Returns: On success, a pointer to the data on the stack. * Otherwise, NULL if there's not enough space left on the * ret_stack for the data, or if fgraph_reserve_data() was called * more than once for a single entryfunc() call. */ void *fgraph_reserve_data(int idx, int size_bytes) { … } /** * fgraph_retrieve_data - Retrieve stored data from fgraph_reserve_data() * @idx: the index of fgraph_array (fgraph_ops::idx) * @size_bytes: pointer to retrieved data size. * * This is to be called by a fgraph_ops retfunc(), to retrieve data that * was stored by the fgraph_ops entryfunc() on the function entry. * That is, this will retrieve the data that was reserved on the * entry of the function that corresponds to the exit of the function * that the fgraph_ops retfunc() is called on. * * Returns: The stored data from fgraph_reserve_data() called by the * matching entryfunc() for the retfunc() this is called from. * Or NULL if there was nothing stored. */ void *fgraph_retrieve_data(int idx, int *size_bytes) { … } /** * fgraph_get_task_var - retrieve a task specific state variable * @gops: The ftrace_ops that owns the task specific variable * * Every registered fgraph_ops has a task state variable * reserved on the task's ret_stack. This function returns the * address to that variable. * * Returns the address to the fgraph_ops @gops tasks specific * unsigned long variable. */ unsigned long *fgraph_get_task_var(struct fgraph_ops *gops) { … } /* * @offset: The offset into @t->ret_stack to find the ret_stack entry * @frame_offset: Where to place the offset into @t->ret_stack of that entry * * Returns a pointer to the previous ret_stack below @offset or NULL * when it reaches the bottom of the stack. * * Calling this with: * * offset = task->curr_ret_stack; * do { * ret_stack = get_ret_stack(task, offset, &offset); * } while (ret_stack); * * Will iterate through all the ret_stack entries from curr_ret_stack * down to the first one. */ static inline struct ftrace_ret_stack * get_ret_stack(struct task_struct *t, int offset, int *frame_offset) { … } /* Both enabled by default (can be cleared by function_graph tracer flags */ static bool fgraph_sleep_time = …; #ifdef CONFIG_DYNAMIC_FTRACE /* * archs can override this function if they must do something * to enable hook for graph tracer. */ int __weak ftrace_enable_ftrace_graph_caller(void) { … } /* * archs can override this function if they must do something * to disable hook for graph tracer. */ int __weak ftrace_disable_ftrace_graph_caller(void) { … } #endif int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { … } static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { … } static struct fgraph_ops fgraph_stub = …; static struct fgraph_ops *fgraph_direct_gops = …; DEFINE_STATIC_CALL(…); DEFINE_STATIC_CALL(…); static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); /** * ftrace_graph_stop - set to permanently disable function graph tracing * * In case of an error int function graph tracing, this is called * to try to keep function graph tracing from causing any more harm. * Usually this is pretty severe and this is called to try to at least * get a warning out to the user. */ void ftrace_graph_stop(void) { … } /* Add a function return address to the trace stack on thread info.*/ static int ftrace_push_return_trace(unsigned long ret, unsigned long func, unsigned long frame_pointer, unsigned long *retp, int fgraph_idx) { … } /* * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct * functions. But those archs currently don't support direct functions * anyway, and ftrace_find_rec_direct() is just a stub for them. * Define MCOUNT_INSN_SIZE to keep those archs compiling. */ #ifndef MCOUNT_INSN_SIZE /* Make sure this only works without direct calls */ # ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS # error MCOUNT_INSN_SIZE not defined with direct calls enabled # endif #define MCOUNT_INSN_SIZE … #endif /* If the caller does not use ftrace, call this function. */ int function_graph_enter(unsigned long ret, unsigned long func, unsigned long frame_pointer, unsigned long *retp) { … } /* Retrieve a function return address to the trace stack on thread info.*/ static struct ftrace_ret_stack * ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, unsigned long frame_pointer, int *offset) { … } /* * Hibernation protection. * The state of the current task is too much unstable during * suspend/restore to disk. We want to protect against that. */ static int ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, void *unused) { … } static struct notifier_block ftrace_suspend_notifier = …; /* fgraph_ret_regs is not defined without CONFIG_FUNCTION_GRAPH_RETVAL */ struct fgraph_ret_regs; /* * Send the trace to the ring-buffer. * @return the original return address. */ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs, unsigned long frame_pointer) { … } /* * After all architecures have selected HAVE_FUNCTION_GRAPH_RETVAL, we can * leave only ftrace_return_to_handler(ret_regs). */ #ifdef CONFIG_HAVE_FUNCTION_GRAPH_RETVAL unsigned long ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs) { … } #else unsigned long ftrace_return_to_handler(unsigned long frame_pointer) { return __ftrace_return_to_handler(NULL, frame_pointer); } #endif /** * ftrace_graph_get_ret_stack - return the entry of the shadow stack * @task: The task to read the shadow stack from. * @idx: Index down the shadow stack * * Return the ret_struct on the shadow stack of the @task at the * call graph at @idx starting with zero. If @idx is zero, it * will return the last saved ret_stack entry. If it is greater than * zero, it will return the corresponding ret_stack for the depth * of saved return addresses. */ struct ftrace_ret_stack * ftrace_graph_get_ret_stack(struct task_struct *task, int idx) { … } /** * ftrace_graph_ret_addr - return the original value of the return address * @task: The task the unwinder is being executed on * @idx: An initialized pointer to the next stack index to use * @ret: The current return address (likely pointing to return_handler) * @retp: The address on the stack of the current return location * * This function can be called by stack unwinding code to convert a found stack * return address (@ret) to its original value, in case the function graph * tracer has modified it to be 'return_to_handler'. If the address hasn't * been modified, the unchanged value of @ret is returned. * * @idx holds the last index used to know where to start from. It should be * initialized to zero for the first iteration as that will mean to start * at the top of the shadow stack. If the location is found, this pointer * will be assigned that location so that if called again, it will continue * where it left off. * * @retp is a pointer to the return address on the stack. */ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { … } static struct ftrace_ops graph_ops = …; void fgraph_init_ops(struct ftrace_ops *dst_ops, struct ftrace_ops *src_ops) { … } void ftrace_graph_sleep_time_control(bool enable) { … } /* * Simply points to ftrace_stub, but with the proper protocol. * Defined by the linker script in linux/vmlinux.lds.h */ void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops); /* The callbacks that hook a function */ trace_func_graph_ret_t ftrace_graph_return = …; trace_func_graph_ent_t ftrace_graph_entry = …; /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ static int alloc_retstack_tasklist(unsigned long **ret_stack_list) { … } static void ftrace_graph_probe_sched_switch(void *ignore, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state) { … } static DEFINE_PER_CPU(unsigned long *, idle_ret_stack); static void graph_init_task(struct task_struct *t, unsigned long *ret_stack) { … } /* * Allocate a return stack for the idle task. May be the first * time through, or it may be done by CPU hotplug online. */ void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { … } /* Allocate a return stack for newly created task */ void ftrace_graph_init_task(struct task_struct *t) { … } void ftrace_graph_exit_task(struct task_struct *t) { … } #ifdef CONFIG_DYNAMIC_FTRACE static int fgraph_pid_func(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { … } void fgraph_update_pid_func(void) { … } #endif /* Allocate a return stack for each task */ static int start_graph_tracing(void) { … } static void init_task_vars(int idx) { … } static void ftrace_graph_enable_direct(bool enable_branch) { … } static void ftrace_graph_disable_direct(bool disable_branch) { … } int register_ftrace_graph(struct fgraph_ops *gops) { … } void unregister_ftrace_graph(struct fgraph_ops *gops) { … }
Codebase Browser
linux