// SPDX-License-Identifier: GPL-2.0 /* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <[email protected]> * Copyright (C) 2004-2008 Ingo Molnar <[email protected]> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <[email protected]> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h> #include <linux/clocksource.h> #include <linux/sched/task.h> #include <linux/kallsyms.h> #include <linux/security.h> #include <linux/seq_file.h> #include <linux/tracefs.h> #include <linux/hardirq.h> #include <linux/kthread.h> #include <linux/uaccess.h> #include <linux/bsearch.h> #include <linux/module.h> #include <linux/ftrace.h> #include <linux/sysctl.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/sort.h> #include <linux/list.h> #include <linux/hash.h> #include <linux/rcupdate.h> #include <linux/kprobes.h> #include <trace/events/sched.h> #include <asm/sections.h> #include <asm/setup.h> #include "ftrace_internal.h" #include "trace_output.h" #include "trace_stat.h" /* Flags that do not get reset */ #define FTRACE_NOCLEAR_FLAGS … #define FTRACE_INVALID_FUNCTION … #define FTRACE_WARN_ON(cond) … #define FTRACE_WARN_ON_ONCE(cond) … /* hash bits for specific function selection */ #define FTRACE_HASH_DEFAULT_BITS … #define FTRACE_HASH_MAX_BITS … #ifdef CONFIG_DYNAMIC_FTRACE #define INIT_OPS_HASH(opsname) … #else #define INIT_OPS_HASH … #endif enum { … }; struct ftrace_ops ftrace_list_end __read_mostly = …; /* ftrace_enabled is a method to turn ftrace on or off */ int ftrace_enabled __read_mostly; static int __maybe_unused last_ftrace_enabled; /* Current function tracing op */ struct ftrace_ops *function_trace_op __read_mostly = …; /* What to set function_trace_op to */ static struct ftrace_ops *set_function_trace_op; bool ftrace_pids_enabled(struct ftrace_ops *ops) { … } static void ftrace_update_trampoline(struct ftrace_ops *ops); /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */ static int ftrace_disabled __read_mostly; DEFINE_MUTEX(…) …; struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = …; ftrace_func_t ftrace_trace_function __read_mostly = …; struct ftrace_ops global_ops; /* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */ void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS /* * Stub used to invoke the list ops without requiring a separate trampoline. */ const struct ftrace_ops ftrace_list_ops = { .func = ftrace_ops_list_func, .flags = FTRACE_OPS_FL_STUB, }; static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { /* do nothing */ } /* * Stub used when a call site is disabled. May be called transiently by threads * which have made it into ftrace_caller but haven't yet recovered the ops at * the point the call site is disabled. */ const struct ftrace_ops ftrace_nop_ops = { .func = ftrace_ops_nop_func, .flags = FTRACE_OPS_FL_STUB, }; #endif static inline void ftrace_ops_init(struct ftrace_ops *ops) { … } /* Call this function for when a callback filters on set_ftrace_pid */ static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { … } static void ftrace_sync_ipi(void *data) { … } static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) { … } static void update_ftrace_function(void) { … } static void add_ftrace_ops(struct ftrace_ops __rcu **list, struct ftrace_ops *ops) { … } static int remove_ftrace_ops(struct ftrace_ops __rcu **list, struct ftrace_ops *ops) { … } static void ftrace_update_trampoline(struct ftrace_ops *ops); int __register_ftrace_function(struct ftrace_ops *ops) { … } int __unregister_ftrace_function(struct ftrace_ops *ops) { … } static void ftrace_update_pid_func(void) { … } #ifdef CONFIG_FUNCTION_PROFILER struct ftrace_profile { … }; struct ftrace_profile_page { … }; struct ftrace_profile_stat { … }; #define PROFILE_RECORDS_SIZE … #define PROFILES_PER_PAGE … static int ftrace_profile_enabled __read_mostly; /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ static DEFINE_MUTEX(ftrace_profile_lock); static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); #define FTRACE_PROFILE_HASH_BITS … #define FTRACE_PROFILE_HASH_SIZE … static void * function_stat_next(void *v, int idx) { … } static void *function_stat_start(struct tracer_stat *trace) { … } #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* function graph compares on total time */ static int function_stat_cmp(const void *p1, const void *p2) { … } #else /* not function graph compares against hits */ static int function_stat_cmp(const void *p1, const void *p2) { const struct ftrace_profile *a = p1; const struct ftrace_profile *b = p2; if (a->counter < b->counter) return -1; if (a->counter > b->counter) return 1; else return 0; } #endif static int function_stat_headers(struct seq_file *m) { … } static int function_stat_show(struct seq_file *m, void *v) { … } static void ftrace_profile_reset(struct ftrace_profile_stat *stat) { … } static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) { … } static int ftrace_profile_init_cpu(int cpu) { … } static int ftrace_profile_init(void) { … } /* interrupts must be disabled */ static struct ftrace_profile * ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) { … } static void ftrace_add_profile(struct ftrace_profile_stat *stat, struct ftrace_profile *rec) { … } /* * The memory is already allocated, this simply finds a new record to use. */ static struct ftrace_profile * ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) { … } static void function_profile_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { … } #ifdef CONFIG_FUNCTION_GRAPH_TRACER static bool fgraph_graph_time = …; void ftrace_graph_graph_time_control(bool enable) { … } static int profile_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { … } static void profile_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { … } static struct fgraph_ops fprofiler_ops = …; static int register_ftrace_profiler(void) { … } static void unregister_ftrace_profiler(void) { … } #else static struct ftrace_ops ftrace_profile_ops __read_mostly = { .func = function_profile_call, .flags = FTRACE_OPS_FL_INITIALIZED, INIT_OPS_HASH(ftrace_profile_ops) }; static int register_ftrace_profiler(void) { return register_ftrace_function(&ftrace_profile_ops); } static void unregister_ftrace_profiler(void) { unregister_ftrace_function(&ftrace_profile_ops); } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ static ssize_t ftrace_profile_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } static ssize_t ftrace_profile_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { … } static const struct file_operations ftrace_profile_fops = …; /* used to initialize the real stat files */ static struct tracer_stat function_stats __initdata = …; static __init void ftrace_profile_tracefs(struct dentry *d_tracer) { … } #else /* CONFIG_FUNCTION_PROFILER */ static __init void ftrace_profile_tracefs(struct dentry *d_tracer) { } #endif /* CONFIG_FUNCTION_PROFILER */ #ifdef CONFIG_DYNAMIC_FTRACE static struct ftrace_ops *removed_ops; /* * Set when doing a global update, like enabling all recs or disabling them. * It is not set when just updating a single ftrace_ops. */ static bool update_all_ops; #ifndef CONFIG_FTRACE_MCOUNT_RECORD # error Dynamic ftrace depends on MCOUNT_RECORD #endif struct ftrace_func_probe { … }; /* * We make these constant because no one should touch them, * but they are used as the default "empty hash", to avoid allocating * it all the time. These are in a read only section such that if * anyone does try to modify it, it will cause an exception. */ static const struct hlist_head empty_buckets[1]; static const struct ftrace_hash empty_hash = …; #define EMPTY_HASH … struct ftrace_ops global_ops = …; /* * Used by the stack unwinder to know about dynamic ftrace trampolines. */ struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr) { … } /* * This is used by __kernel_text_address() to return true if the * address is on a dynamically allocated trampoline that would * not return true for either core_kernel_text() or * is_module_text_address(). */ bool is_ftrace_trampoline(unsigned long addr) { … } struct ftrace_page { … }; #define ENTRY_SIZE … #define ENTRIES_PER_PAGE … static struct ftrace_page *ftrace_pages_start; static struct ftrace_page *ftrace_pages; static __always_inline unsigned long ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip) { … } /* Only use this function if ftrace_hash_empty() has already been tested */ static __always_inline struct ftrace_func_entry * __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) { … } /** * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash * @hash: The hash to look at * @ip: The instruction pointer to test * * Search a given @hash to see if a given instruction pointer (@ip) * exists in it. * * Returns: the entry that holds the @ip if found. NULL otherwise. */ struct ftrace_func_entry * ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) { … } static void __add_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { … } static struct ftrace_func_entry * add_hash_entry(struct ftrace_hash *hash, unsigned long ip) { … } static void free_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { … } static void remove_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { … } static void ftrace_hash_clear(struct ftrace_hash *hash) { … } static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod) { … } static void clear_ftrace_mod_list(struct list_head *head) { … } static void free_ftrace_hash(struct ftrace_hash *hash) { … } static void __free_ftrace_hash_rcu(struct rcu_head *rcu) { … } static void free_ftrace_hash_rcu(struct ftrace_hash *hash) { … } /** * ftrace_free_filter - remove all filters for an ftrace_ops * @ops: the ops to remove the filters from */ void ftrace_free_filter(struct ftrace_ops *ops) { … } EXPORT_SYMBOL_GPL(…); static struct ftrace_hash *alloc_ftrace_hash(int size_bits) { … } /* Used to save filters on functions for modules not loaded yet */ static int ftrace_add_mod(struct trace_array *tr, const char *func, const char *module, int enable) { … } static struct ftrace_hash * alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) { … } static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops); static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops); static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, struct ftrace_hash *new_hash); /* * Allocate a new hash and remove entries from @src and move them to the new hash. * On success, the @src hash will be empty and should be freed. */ static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) { … } /* Move the @src entries to a newly allocated hash */ static struct ftrace_hash * __ftrace_hash_move(struct ftrace_hash *src) { … } /** * ftrace_hash_move - move a new hash to a filter and do updates * @ops: The ops with the hash that @dst points to * @enable: True if for the filter hash, false for the notrace hash * @dst: Points to the @ops hash that should be updated * @src: The hash to update @dst with * * This is called when an ftrace_ops hash is being updated and the * the kernel needs to reflect this. Note, this only updates the kernel * function callbacks if the @ops is enabled (not to be confused with * @enable above). If the @ops is enabled, its hash determines what * callbacks get called. This function gets called when the @ops hash * is updated and it requires new callbacks. * * On success the elements of @src is moved to @dst, and @dst is updated * properly, as well as the functions determined by the @ops hashes * are now calling the @ops callback function. * * Regardless of return type, @src should be freed with free_ftrace_hash(). */ static int ftrace_hash_move(struct ftrace_ops *ops, int enable, struct ftrace_hash **dst, struct ftrace_hash *src) { … } static bool hash_contains_ip(unsigned long ip, struct ftrace_ops_hash *hash) { … } /* * Test the hashes for this ops to see if we want to call * the ops->func or not. * * It's a match if the ip is in the ops->filter_hash or * the filter_hash does not exist or is empty, * AND * the ip is not in the ops->notrace_hash. * * This needs to be called with preemption disabled as * the hashes are freed with call_rcu(). */ int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) { … } /* * This is a double for. Do not use 'break' to break out of the loop, * you must use a goto. */ #define do_for_each_ftrace_rec(pg, rec) … #define while_for_each_ftrace_rec() … static int ftrace_cmp_recs(const void *a, const void *b) { … } static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end) { … } /** * ftrace_location_range - return the first address of a traced location * if it touches the given ip range * @start: start of range to search. * @end: end of range to search (inclusive). @end points to the last byte * to check. * * Returns: rec->ip if the related ftrace location is a least partly within * the given address range. That is, the first address of the instruction * that is either a NOP or call to the function tracer. It checks the ftrace * internal tables to determine if the address belongs or not. */ unsigned long ftrace_location_range(unsigned long start, unsigned long end) { … } /** * ftrace_location - return the ftrace location * @ip: the instruction pointer to check * * Returns: * * If @ip matches the ftrace location, return @ip. * * If @ip matches sym+0, return sym's ftrace location. * * Otherwise, return 0. */ unsigned long ftrace_location(unsigned long ip) { … } /** * ftrace_text_reserved - return true if range contains an ftrace location * @start: start of range to search * @end: end of range to search (inclusive). @end points to the last byte to check. * * Returns: 1 if @start and @end contains a ftrace location. * That is, the instruction that is either a NOP or call to * the function tracer. It checks the ftrace internal tables to * determine if the address belongs or not. */ int ftrace_text_reserved(const void *start, const void *end) { … } /* Test if ops registered to this rec needs regs */ static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) { … } static struct ftrace_ops * ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); static struct ftrace_ops * ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude); static struct ftrace_ops * ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); static bool skip_record(struct dyn_ftrace *rec) { … } /* * This is the main engine to the ftrace updates to the dyn_ftrace records. * * It will iterate through all the available ftrace functions * (the ones that ftrace can have callbacks to) and set the flags * in the associated dyn_ftrace records. * * @inc: If true, the functions associated to @ops are added to * the dyn_ftrace records, otherwise they are removed. */ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, bool inc) { … } /* * This is called when an ops is removed from tracing. It will decrement * the counters of the dyn_ftrace records for all the functions that * the @ops attached to. */ static bool ftrace_hash_rec_disable(struct ftrace_ops *ops) { … } /* * This is called when an ops is added to tracing. It will increment * the counters of the dyn_ftrace records for all the functions that * the @ops attached to. */ static bool ftrace_hash_rec_enable(struct ftrace_ops *ops) { … } /* * This function will update what functions @ops traces when its filter * changes. * * The @inc states if the @ops callbacks are going to be added or removed. * When one of the @ops hashes are updated to a "new_hash" the dyn_ftrace * records are update via: * * ftrace_hash_rec_disable_modify(ops); * ops->hash = new_hash * ftrace_hash_rec_enable_modify(ops); * * Where the @ops is removed from all the records it is tracing using * its old hash. The @ops hash is updated to the new hash, and then * the @ops is added back to the records so that it is tracing all * the new functions. */ static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, bool inc) { … } static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops) { … } static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops) { … } /* * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK * or no-needed to update, -EBUSY if it detects a conflict of the flag * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. * Note that old_hash and new_hash has below meanings * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) * - If the hash is EMPTY_HASH, it hits nothing * - Anything else hits the recs which match the hash entries. * * DIRECT ops does not have IPMODIFY flag, but we still need to check it * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate * the return value to the caller and eventually to the owner of the DIRECT * ops. */ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, struct ftrace_hash *old_hash, struct ftrace_hash *new_hash) { … } static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) { … } /* Disabling always succeeds */ static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) { … } static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, struct ftrace_hash *new_hash) { … } static void print_ip_ins(const char *fmt, const unsigned char *p) { … } enum ftrace_bug_type ftrace_bug_type; const void *ftrace_expected; static void print_bug_type(void) { … } /** * ftrace_bug - report and shutdown function tracer * @failed: The failed type (EFAULT, EINVAL, EPERM) * @rec: The record that failed * * The arch code that enables or disables the function tracing * can call ftrace_bug() when it has detected a problem in * modifying the code. @failed should be one of either: * EFAULT - if the problem happens on reading the @ip address * EINVAL - if what is read at @ip is not what was expected * EPERM - if the problem happens on writing to the @ip address */ void ftrace_bug(int failed, struct dyn_ftrace *rec) { … } static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) { … } /** * ftrace_update_record - set a record that now is tracing or not * @rec: the record to update * @enable: set to true if the record is tracing, false to force disable * * The records that represent all functions that can be traced need * to be updated when tracing has been enabled. */ int ftrace_update_record(struct dyn_ftrace *rec, bool enable) { … } /** * ftrace_test_record - check if the record has been enabled or not * @rec: the record to test * @enable: set to true to check if enabled, false if it is disabled * * The arch code may need to test if a record is already set to * tracing to determine how to modify the function code that it * represents. */ int ftrace_test_record(struct dyn_ftrace *rec, bool enable) { … } static struct ftrace_ops * ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) { … } static struct ftrace_ops * ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude) { … } static struct ftrace_ops * ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *op) { … } static struct ftrace_ops * ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) { … } static struct ftrace_ops * ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) { … } struct ftrace_ops * ftrace_find_unique_ops(struct dyn_ftrace *rec) { … } #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* Protected by rcu_tasks for reading, and direct_mutex for writing */ static struct ftrace_hash __rcu *direct_functions = …; static DEFINE_MUTEX(direct_mutex); /* * Search the direct_functions hash to see if the given instruction pointer * has a direct caller attached to it. */ unsigned long ftrace_find_rec_direct(unsigned long ip) { … } static void call_direct_funcs(unsigned long ip, unsigned long pip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { … } #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ /** * ftrace_get_addr_new - Get the call address to set to * @rec: The ftrace record descriptor * * If the record has the FTRACE_FL_REGS set, that means that it * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS * is not set, then it wants to convert to the normal callback. * * Returns: the address of the trampoline to set to */ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) { … } /** * ftrace_get_addr_curr - Get the call address that is already there * @rec: The ftrace record descriptor * * The FTRACE_FL_REGS_EN is set when the record already points to * a function that saves all the regs. Basically the '_EN' version * represents the current state of the function. * * Returns: the address of the trampoline that is currently being called */ unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) { … } static int __ftrace_replace_code(struct dyn_ftrace *rec, bool enable) { … } void __weak ftrace_replace_code(int mod_flags) { … } struct ftrace_rec_iter { … }; /** * ftrace_rec_iter_start - start up iterating over traced functions * * Returns: an iterator handle that is used to iterate over all * the records that represent address locations where functions * are traced. * * May return NULL if no records are available. */ struct ftrace_rec_iter *ftrace_rec_iter_start(void) { … } /** * ftrace_rec_iter_next - get the next record to process. * @iter: The handle to the iterator. * * Returns: the next iterator after the given iterator @iter. */ struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) { … } /** * ftrace_rec_iter_record - get the record at the iterator location * @iter: The current iterator location * * Returns: the record that the current @iter is at. */ struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) { … } static int ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec) { … } /* * archs can override this function if they must do something * before the modifying code is performed. */ void __weak ftrace_arch_code_modify_prepare(void) { … } /* * archs can override this function if they must do something * after the modifying code is performed. */ void __weak ftrace_arch_code_modify_post_process(void) { … } static int update_ftrace_func(ftrace_func_t func) { … } void ftrace_modify_all_code(int command) { … } static int __ftrace_modify_code(void *data) { … } /** * ftrace_run_stop_machine - go back to the stop machine method * @command: The command to tell ftrace what to do * * If an arch needs to fall back to the stop machine method, the * it can call this function. */ void ftrace_run_stop_machine(int command) { … } /** * arch_ftrace_update_code - modify the code to trace or not trace * @command: The command that needs to be done * * Archs can override this function if it does not need to * run stop_machine() to modify code. */ void __weak arch_ftrace_update_code(int command) { … } static void ftrace_run_update_code(int command) { … } static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, struct ftrace_ops_hash *old_hash) { … } static ftrace_func_t saved_ftrace_func; static int ftrace_start_up; void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) { … } /* List of trace_ops that have allocated trampolines */ static LIST_HEAD(ftrace_ops_trampoline_list); static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops) { … } static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops) { … } /* * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is * not a module. */ #define FTRACE_TRAMPOLINE_MOD … #define FTRACE_TRAMPOLINE_SYM … static void ftrace_trampoline_free(struct ftrace_ops *ops) { … } static void ftrace_startup_enable(int command) { … } static void ftrace_startup_all(int command) { … } int ftrace_startup(struct ftrace_ops *ops, int command) { … } int ftrace_shutdown(struct ftrace_ops *ops, int command) { … } /* Simply make a copy of @src and return it */ static struct ftrace_hash *copy_hash(struct ftrace_hash *src) { … } /* * Append @new_hash entries to @hash: * * If @hash is the EMPTY_HASH then it traces all functions and nothing * needs to be done. * * If @new_hash is the EMPTY_HASH, then make *hash the EMPTY_HASH so * that it traces everything. * * Otherwise, go through all of @new_hash and add anything that @hash * doesn't already have, to @hash. * * The filter_hash updates uses just the append_hash() function * and the notrace_hash does not. */ static int append_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash) { … } /* * Add to @hash only those that are in both @new_hash1 and @new_hash2 * * The notrace_hash updates uses just the intersect_hash() function * and the filter_hash does not. */ static int intersect_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash1, struct ftrace_hash *new_hash2) { … } /* Return a new hash that has a union of all @ops->filter_hash entries */ static struct ftrace_hash *append_hashes(struct ftrace_ops *ops) { … } /* Make @ops trace evenything except what all its subops do not trace */ static struct ftrace_hash *intersect_hashes(struct ftrace_ops *ops) { … } static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B) { … } static void ftrace_ops_update_code(struct ftrace_ops *ops, struct ftrace_ops_hash *old_hash); static int __ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, struct ftrace_hash **orig_hash, struct ftrace_hash *hash, int enable) { … } static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_hash, struct ftrace_hash *notrace_hash) { … } /** * ftrace_startup_subops - enable tracing for subops of an ops * @ops: Manager ops (used to pick all the functions of its subops) * @subops: A new ops to add to @ops * @command: Extra commands to use to enable tracing * * The @ops is a manager @ops that has the filter that includes all the functions * that its list of subops are tracing. Adding a new @subops will add the * functions of @subops to @ops. */ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) { … } /** * ftrace_shutdown_subops - Remove a subops from a manager ops * @ops: A manager ops to remove @subops from * @subops: The subops to remove from @ops * @command: Any extra command flags to add to modifying the text * * Removes the functions being traced by the @subops from @ops. Note, it * will not affect functions that are being traced by other subops that * still exist in @ops. * * If the last subops is removed from @ops, then @ops is shutdown normally. */ int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) { … } static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops, struct ftrace_hash **orig_subhash, struct ftrace_hash *hash, int enable) { … } static u64 ftrace_update_time; unsigned long ftrace_update_tot_cnt; unsigned long ftrace_number_of_pages; unsigned long ftrace_number_of_groups; static inline int ops_traces_mod(struct ftrace_ops *ops) { … } static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) { … } static int ftrace_allocate_records(struct ftrace_page *pg, int count) { … } static void ftrace_free_pages(struct ftrace_page *pages) { … } static struct ftrace_page * ftrace_allocate_pages(unsigned long num_to_init) { … } #define FTRACE_BUFF_MAX … struct ftrace_iterator { … }; static void * t_probe_next(struct seq_file *m, loff_t *pos) { … } static void *t_probe_start(struct seq_file *m, loff_t *pos) { … } static int t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) { … } static void * t_mod_next(struct seq_file *m, loff_t *pos) { … } static void *t_mod_start(struct seq_file *m, loff_t *pos) { … } static int t_mod_show(struct seq_file *m, struct ftrace_iterator *iter) { … } static void * t_func_next(struct seq_file *m, loff_t *pos) { … } static void * t_next(struct seq_file *m, void *v, loff_t *pos) { … } static void reset_iter_read(struct ftrace_iterator *iter) { … } static void *t_start(struct seq_file *m, loff_t *pos) { … } static void t_stop(struct seq_file *m, void *p) { … } void * __weak arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) { … } static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, struct dyn_ftrace *rec) { … } #ifdef FTRACE_MCOUNT_MAX_OFFSET /* * Weak functions can still have an mcount/fentry that is saved in * the __mcount_loc section. These can be detected by having a * symbol offset of greater than FTRACE_MCOUNT_MAX_OFFSET, as the * symbol found by kallsyms is not the function that the mcount/fentry * is part of. The offset is much greater in these cases. * * Test the record to make sure that the ip points to a valid kallsyms * and if not, mark it disabled. */ static int test_for_valid_rec(struct dyn_ftrace *rec) { … } static struct workqueue_struct *ftrace_check_wq __initdata; static struct work_struct ftrace_check_work __initdata; /* * Scan all the mcount/fentry entries to make sure they are valid. */ static __init void ftrace_check_work_func(struct work_struct *work) { … } static int __init ftrace_check_for_weak_functions(void) { … } static int __init ftrace_check_sync(void) { … } late_initcall_sync(ftrace_check_sync); subsys_initcall(ftrace_check_for_weak_functions); static int print_rec(struct seq_file *m, unsigned long ip) { … } #else static inline int test_for_valid_rec(struct dyn_ftrace *rec) { return 1; } static inline int print_rec(struct seq_file *m, unsigned long ip) { seq_printf(m, "%ps", (void *)ip); return 0; } #endif static int t_show(struct seq_file *m, void *v) { … } static const struct seq_operations show_ftrace_seq_ops = …; static int ftrace_avail_open(struct inode *inode, struct file *file) { … } static int ftrace_enabled_open(struct inode *inode, struct file *file) { … } static int ftrace_touched_open(struct inode *inode, struct file *file) { … } static int ftrace_avail_addrs_open(struct inode *inode, struct file *file) { … } /** * ftrace_regex_open - initialize function tracer filter files * @ops: The ftrace_ops that hold the hash filters * @flag: The type of filter to process * @inode: The inode, usually passed in to your open routine * @file: The file, usually passed in to your open routine * * ftrace_regex_open() initializes the filter files for the * @ops. Depending on @flag it may process the filter hash or * the notrace hash of @ops. With this called from the open * routine, you can use ftrace_filter_write() for the write * routine if @flag has FTRACE_ITER_FILTER set, or * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. * tracing_lseek() should be used as the lseek routine, and * release must call ftrace_regex_release(). * * Returns: 0 on success or a negative errno value on failure */ int ftrace_regex_open(struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file) { … } static int ftrace_filter_open(struct inode *inode, struct file *file) { … } static int ftrace_notrace_open(struct inode *inode, struct file *file) { … } /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ struct ftrace_glob { … }; /* * If symbols in an architecture don't correspond exactly to the user-visible * name of what they represent, it is possible to define this function to * perform the necessary adjustments. */ char * __weak arch_ftrace_match_adjust(char *str, const char *search) { … } static int ftrace_match(char *str, struct ftrace_glob *g) { … } static int enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) { … } static int add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g, int clear_filter) { … } #ifdef FTRACE_MCOUNT_MAX_OFFSET static int lookup_ip(unsigned long ip, char **modname, char *str) { … } #else static int lookup_ip(unsigned long ip, char **modname, char *str) { kallsyms_lookup(ip, NULL, NULL, modname, str); return 0; } #endif static int ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, struct ftrace_glob *mod_g, int exclude_mod) { … } static int match_records(struct ftrace_hash *hash, char *func, int len, char *mod) { … } static int ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) { … } static void ftrace_ops_update_code(struct ftrace_ops *ops, struct ftrace_ops_hash *old_hash) { … } static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, struct ftrace_hash **orig_hash, struct ftrace_hash *hash, int enable) { … } static bool module_exists(const char *module) { … } static int cache_mod(struct trace_array *tr, const char *func, char *module, int enable) { … } static int ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, int reset, int enable); #ifdef CONFIG_MODULES static void process_mod_list(struct list_head *head, struct ftrace_ops *ops, char *mod, bool enable) { … } static void process_cached_mods(const char *mod_name) { … } #endif /* * We register the module command as a template to show others how * to register the a command as well. */ static int ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash, char *func_orig, char *cmd, char *module, int enable) { … } static struct ftrace_func_command ftrace_mod_cmd = …; static int __init ftrace_mod_cmd_init(void) { … } core_initcall(ftrace_mod_cmd_init); static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { … } struct ftrace_func_map { … }; struct ftrace_func_mapper { … }; /** * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper * * Returns: a ftrace_func_mapper descriptor that can be used to map ips to data. */ struct ftrace_func_mapper *allocate_ftrace_func_mapper(void) { … } /** * ftrace_func_mapper_find_ip - Find some data mapped to an ip * @mapper: The mapper that has the ip maps * @ip: the instruction pointer to find the data for * * Returns: the data mapped to @ip if found otherwise NULL. The return * is actually the address of the mapper data pointer. The address is * returned for use cases where the data is no bigger than a long, and * the user can use the data pointer as its data instead of having to * allocate more memory for the reference. */ void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper, unsigned long ip) { … } /** * ftrace_func_mapper_add_ip - Map some data to an ip * @mapper: The mapper that has the ip maps * @ip: The instruction pointer address to map @data to * @data: The data to map to @ip * * Returns: 0 on success otherwise an error. */ int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, unsigned long ip, void *data) { … } /** * ftrace_func_mapper_remove_ip - Remove an ip from the mapping * @mapper: The mapper that has the ip maps * @ip: The instruction pointer address to remove the data from * * Returns: the data if it is found, otherwise NULL. * Note, if the data pointer is used as the data itself, (see * ftrace_func_mapper_find_ip(), then the return value may be meaningless, * if the data pointer was set to zero. */ void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper, unsigned long ip) { … } /** * free_ftrace_func_mapper - free a mapping of ips and data * @mapper: The mapper that has the ip maps * @free_func: A function to be called on each data item. * * This is used to free the function mapper. The @free_func is optional * and can be used if the data needs to be freed as well. */ void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, ftrace_mapper_func free_func) { … } static void release_probe(struct ftrace_func_probe *probe) { … } static void acquire_probe_locked(struct ftrace_func_probe *probe) { … } int register_ftrace_function_probe(char *glob, struct trace_array *tr, struct ftrace_probe_ops *probe_ops, void *data) { … } int unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, struct ftrace_probe_ops *probe_ops) { … } void clear_ftrace_function_probes(struct trace_array *tr) { … } static LIST_HEAD(ftrace_commands); static DEFINE_MUTEX(ftrace_cmd_mutex); /* * Currently we only register ftrace commands from __init, so mark this * __init too. */ __init int register_ftrace_command(struct ftrace_func_command *cmd) { … } /* * Currently we only unregister ftrace commands from __init, so mark * this __init too. */ __init int unregister_ftrace_command(struct ftrace_func_command *cmd) { … } static int ftrace_process_regex(struct ftrace_iterator *iter, char *buff, int len, int enable) { … } static ssize_t ftrace_regex_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos, int enable) { … } ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } static int __ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) { … } static int ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips, unsigned int cnt, int remove) { … } static int ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, unsigned long *ips, unsigned int cnt, int remove, int reset, int enable) { … } static int ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt, int remove, int reset, int enable) { … } #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int register_ftrace_function_nolock(struct ftrace_ops *ops); /* * If there are multiple ftrace_ops, use SAVE_REGS by default, so that direct * call will be jumped from ftrace_regs_caller. Only if the architecture does * not support ftrace_regs_caller but direct_call, use SAVE_ARGS so that it * jumps from ftrace_caller for multiple ftrace_ops. */ #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS #define MULTI_FLAGS … #else #define MULTI_FLAGS … #endif static int check_direct_multi(struct ftrace_ops *ops) { … } static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr) { … } static void register_ftrace_direct_cb(struct rcu_head *rhp) { … } /** * register_ftrace_direct - Call a custom trampoline directly * for multiple functions registered in @ops * @ops: The address of the struct ftrace_ops object * @addr: The address of the trampoline to call at @ops functions * * This is used to connect a direct calls to @addr from the nop locations * of the functions registered in @ops (with by ftrace_set_filter_ip * function). * * The location that it calls (@addr) must be able to handle a direct call, * and save the parameters of the function being traced, and restore them * (or inject new ones if needed), before returning. * * Returns: * 0 on success * -EINVAL - The @ops object was already registered with this call or * when there are no functions in @ops object. * -EBUSY - Another direct function is already attached (there can be only one) * -ENODEV - @ip does not point to a ftrace nop location (or not supported) * -ENOMEM - There was an allocation failure. */ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) { … } EXPORT_SYMBOL_GPL(…); /** * unregister_ftrace_direct - Remove calls to custom trampoline * previously registered by register_ftrace_direct for @ops object. * @ops: The address of the struct ftrace_ops object * @addr: The address of the direct function that is called by the @ops functions * @free_filters: Set to true to remove all filters for the ftrace_ops, false otherwise * * This is used to remove a direct calls to @addr from the nop locations * of the functions registered in @ops (with by ftrace_set_filter_ip * function). * * Returns: * 0 on success * -EINVAL - The @ops object was not properly registered. */ int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, bool free_filters) { … } EXPORT_SYMBOL_GPL(…); static int __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) { … } /** * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call * to call something else * @ops: The address of the struct ftrace_ops object * @addr: The address of the new trampoline to call at @ops functions * * This is used to unregister currently registered direct caller and * register new one @addr on functions registered in @ops object. * * Note there's window between ftrace_shutdown and ftrace_startup calls * where there will be no callbacks called. * * Caller should already have direct_mutex locked, so we don't lock * direct_mutex here. * * Returns: zero on success. Non zero on error, which includes: * -EINVAL - The @ops object was not properly registered. */ int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr) { … } EXPORT_SYMBOL_GPL(…); /** * modify_ftrace_direct - Modify an existing direct 'multi' call * to call something else * @ops: The address of the struct ftrace_ops object * @addr: The address of the new trampoline to call at @ops functions * * This is used to unregister currently registered direct caller and * register new one @addr on functions registered in @ops object. * * Note there's window between ftrace_shutdown and ftrace_startup calls * where there will be no callbacks called. * * Returns: zero on success. Non zero on error, which includes: * -EINVAL - The @ops object was not properly registered. */ int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) { … } EXPORT_SYMBOL_GPL(…); #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ /** * ftrace_set_filter_ip - set a function to filter on in ftrace by address * @ops: the ops to set the filter with * @ip: the address to add to or remove from the filter. * @remove: non zero to remove the ip from the filter * @reset: non zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled * If @ip is NULL, it fails to update filter. * * This can allocate memory which must be freed before @ops can be freed, * either by removing each filtered addr or by using * ftrace_free_filter(@ops). */ int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, int remove, int reset) { … } EXPORT_SYMBOL_GPL(…); /** * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses * @ops: the ops to set the filter with * @ips: the array of addresses to add to or remove from the filter. * @cnt: the number of addresses in @ips * @remove: non zero to remove ips from the filter * @reset: non zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled * If @ips array or any ip specified within is NULL , it fails to update filter. * * This can allocate memory which must be freed before @ops can be freed, * either by removing each filtered addr or by using * ftrace_free_filter(@ops). */ int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt, int remove, int reset) { … } EXPORT_SYMBOL_GPL(…); /** * ftrace_ops_set_global_filter - setup ops to use global filters * @ops: the ops which will use the global filters * * ftrace users who need global function trace filtering should call this. * It can set the global filter only if ops were not initialized before. */ void ftrace_ops_set_global_filter(struct ftrace_ops *ops) { … } EXPORT_SYMBOL_GPL(…); static int ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, int reset, int enable) { … } /** * ftrace_set_filter - set a function to filter on in ftrace * @ops: the ops to set the filter with * @buf: the string that holds the function filter text. * @len: the length of the string. * @reset: non-zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled. * If @buf is NULL and reset is set, all functions will be enabled for tracing. * * This can allocate memory which must be freed before @ops can be freed, * either by removing each filtered addr or by using * ftrace_free_filter(@ops). */ int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, int len, int reset) { … } EXPORT_SYMBOL_GPL(…); /** * ftrace_set_notrace - set a function to not trace in ftrace * @ops: the ops to set the notrace filter with * @buf: the string that holds the function notrace text. * @len: the length of the string. * @reset: non-zero to reset all filters before applying this filter. * * Notrace Filters denote which functions should not be enabled when tracing * is enabled. If @buf is NULL and reset is set, all functions will be enabled * for tracing. * * This can allocate memory which must be freed before @ops can be freed, * either by removing each filtered addr or by using * ftrace_free_filter(@ops). */ int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, int len, int reset) { … } EXPORT_SYMBOL_GPL(…); /** * ftrace_set_global_filter - set a function to filter on with global tracers * @buf: the string that holds the function filter text. * @len: the length of the string. * @reset: non-zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled. * If @buf is NULL and reset is set, all functions will be enabled for tracing. */ void ftrace_set_global_filter(unsigned char *buf, int len, int reset) { … } EXPORT_SYMBOL_GPL(…); /** * ftrace_set_global_notrace - set a function to not trace with global tracers * @buf: the string that holds the function notrace text. * @len: the length of the string. * @reset: non-zero to reset all filters before applying this filter. * * Notrace Filters denote which functions should not be enabled when tracing * is enabled. If @buf is NULL and reset is set, all functions will be enabled * for tracing. */ void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) { … } EXPORT_SYMBOL_GPL(…); /* * command line interface to allow users to set filters on boot up. */ #define FTRACE_FILTER_SIZE … static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; /* Used by function selftest to not test if filter is set */ bool ftrace_filter_param __initdata; static int __init set_ftrace_notrace(char *str) { … } __setup(…); static int __init set_ftrace_filter(char *str) { … } __setup(…); #ifdef CONFIG_FUNCTION_GRAPH_TRACER static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer); static int __init set_graph_function(char *str) { … } __setup(…); static int __init set_graph_notrace_function(char *str) { … } __setup(…); static int __init set_graph_max_depth_function(char *str) { … } __setup(…); static void __init set_ftrace_early_graph(char *buf, int enable) { … } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ void __init ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) { … } static void __init set_ftrace_early_filters(void) { … } int ftrace_regex_release(struct inode *inode, struct file *file) { … } static const struct file_operations ftrace_avail_fops = …; static const struct file_operations ftrace_enabled_fops = …; static const struct file_operations ftrace_touched_fops = …; static const struct file_operations ftrace_avail_addrs_fops = …; static const struct file_operations ftrace_filter_fops = …; static const struct file_operations ftrace_notrace_fops = …; #ifdef CONFIG_FUNCTION_GRAPH_TRACER static DEFINE_MUTEX(graph_lock); struct ftrace_hash __rcu *ftrace_graph_hash = …; struct ftrace_hash __rcu *ftrace_graph_notrace_hash = …; enum graph_filter_type { … }; #define FTRACE_GRAPH_EMPTY … struct ftrace_graph_data { … }; static void * __g_next(struct seq_file *m, loff_t *pos) { … } static void * g_next(struct seq_file *m, void *v, loff_t *pos) { … } static void *g_start(struct seq_file *m, loff_t *pos) { … } static void g_stop(struct seq_file *m, void *p) { … } static int g_show(struct seq_file *m, void *v) { … } static const struct seq_operations ftrace_graph_seq_ops = …; static int __ftrace_graph_open(struct inode *inode, struct file *file, struct ftrace_graph_data *fgd) { … } static int ftrace_graph_open(struct inode *inode, struct file *file) { … } static int ftrace_graph_notrace_open(struct inode *inode, struct file *file) { … } static int ftrace_graph_release(struct inode *inode, struct file *file) { … } static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer) { … } static ssize_t ftrace_graph_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } static const struct file_operations ftrace_graph_fops = …; static const struct file_operations ftrace_graph_notrace_fops = …; #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ void ftrace_create_filter_files(struct ftrace_ops *ops, struct dentry *parent) { … } /* * The name "destroy_filter_files" is really a misnomer. Although * in the future, it may actually delete the files, but this is * really intended to make sure the ops passed in are disabled * and that when this function returns, the caller is free to * free the ops. * * The "destroy" name is only to match the "create" name that this * should be paired with. */ void ftrace_destroy_filter_files(struct ftrace_ops *ops) { … } static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { … } static int ftrace_cmp_ips(const void *a, const void *b) { … } #ifdef CONFIG_FTRACE_SORT_STARTUP_TEST static void test_is_sorted(unsigned long *start, unsigned long count) { … } #else static void test_is_sorted(unsigned long *start, unsigned long count) { } #endif static int ftrace_process_locs(struct module *mod, unsigned long *start, unsigned long *end) { … } struct ftrace_mod_func { … }; struct ftrace_mod_map { … }; static int ftrace_get_trampoline_kallsym(unsigned int symnum, unsigned long *value, char *type, char *name, char *module_name, int *exported) { … } #if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES) /* * Check if the current ops references the given ip. * * If the ops traces all functions, then it was already accounted for. * If the ops does not trace the current record function, skip it. * If the ops ignores the function via notrace filter, skip it. */ static bool ops_references_ip(struct ftrace_ops *ops, unsigned long ip) { … } #endif #ifdef CONFIG_MODULES #define next_to_ftrace_page(p) … static LIST_HEAD(ftrace_mod_maps); static int referenced_filters(struct dyn_ftrace *rec) { … } static void clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash) { … } /* Clear any records from hashes */ static void clear_mod_from_hashes(struct ftrace_page *pg) { … } static void ftrace_free_mod_map(struct rcu_head *rcu) { … } void ftrace_release_mod(struct module *mod) { … } void ftrace_module_enable(struct module *mod) { … } void ftrace_module_init(struct module *mod) { … } static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, struct dyn_ftrace *rec) { … } static struct ftrace_mod_map * allocate_ftrace_mod_map(struct module *mod, unsigned long start, unsigned long end) { … } static int ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, unsigned long addr, unsigned long *size, unsigned long *off, char *sym) { … } int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, unsigned long *off, char **modname, char *sym) { … } int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *name, char *module_name, int *exported) { … } #else static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, struct dyn_ftrace *rec) { } static inline struct ftrace_mod_map * allocate_ftrace_mod_map(struct module *mod, unsigned long start, unsigned long end) { return NULL; } int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *name, char *module_name, int *exported) { int ret; preempt_disable(); ret = ftrace_get_trampoline_kallsym(symnum, value, type, name, module_name, exported); preempt_enable(); return ret; } #endif /* CONFIG_MODULES */ struct ftrace_init_func { … }; /* Clear any init ips from hashes */ static void clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash) { … } static void clear_func_from_hashes(struct ftrace_init_func *func) { … } static void add_to_clear_hash_list(struct list_head *clear_list, struct dyn_ftrace *rec) { … } void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) { … } void __init ftrace_free_init_mem(void) { … } int __init __weak ftrace_dyn_arch_init(void) { … } void __init ftrace_init(void) { … } /* Do nothing if arch does not support this */ void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) { … } static void ftrace_update_trampoline(struct ftrace_ops *ops) { … } void ftrace_init_trace_array(struct trace_array *tr) { … } #else struct ftrace_ops global_ops = { .func = ftrace_stub, .flags = FTRACE_OPS_FL_INITIALIZED | FTRACE_OPS_FL_PID, }; static int __init ftrace_nodyn_init(void) { ftrace_enabled = 1; return 0; } core_initcall(ftrace_nodyn_init); static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } static inline void ftrace_startup_all(int command) { } static void ftrace_update_trampoline(struct ftrace_ops *ops) { } #endif /* CONFIG_DYNAMIC_FTRACE */ __init void ftrace_init_global_array_ops(struct trace_array *tr) { … } void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) { … } void ftrace_reset_array_ops(struct trace_array *tr) { … } static nokprobe_inline void __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ignored, struct ftrace_regs *fregs) { … } /* * Some archs only support passing ip and parent_ip. Even though * the list function ignores the op parameter, we do not want any * C side effects, where a function is called without the caller * sending a third parameter. * Archs are to support both the regs and ftrace_ops at the same time. * If they support ftrace_ops, it is assumed they support regs. * If call backs want to use regs, they must either check for regs * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. * An architecture can pass partial regs with ftrace_ops and still * set the ARCH_SUPPORTS_FTRACE_OPS. * * In vmlinux.lds.h, ftrace_ops_list_func() is defined to be * arch_ftrace_ops_list_func. */ #if ARCH_SUPPORTS_FTRACE_OPS void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { … } #else void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip) { __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); } #endif NOKPROBE_SYMBOL(arch_ftrace_ops_list_func); /* * If there's only one function registered but it does not support * recursion, needs RCU protection, then this function will be called * by the mcount trampoline. */ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { … } NOKPROBE_SYMBOL(ftrace_ops_assist_func); /** * ftrace_ops_get_func - get the function a trampoline should call * @ops: the ops to get the function for * * Normally the mcount trampoline will call the ops->func, but there * are times that it should not. For example, if the ops does not * have its own recursion protection, then it should call the * ftrace_ops_assist_func() instead. * * Returns: the function that the trampoline should call for @ops. */ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) { … } static void ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state) { … } static void ftrace_pid_follow_sched_process_fork(void *data, struct task_struct *self, struct task_struct *task) { … } static void ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task) { … } void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { … } static void clear_ftrace_pids(struct trace_array *tr, int type) { … } void ftrace_clear_pids(struct trace_array *tr) { … } static void ftrace_pid_reset(struct trace_array *tr, int type) { … } /* Greater than any max PID */ #define FTRACE_NO_PIDS … static void *fpid_start(struct seq_file *m, loff_t *pos) __acquires(RCU) { … } static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) { … } static void fpid_stop(struct seq_file *m, void *p) __releases(RCU) { … } static int fpid_show(struct seq_file *m, void *v) { … } static const struct seq_operations ftrace_pid_sops = …; static void *fnpid_start(struct seq_file *m, loff_t *pos) __acquires(RCU) { … } static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos) { … } static const struct seq_operations ftrace_no_pid_sops = …; static int pid_open(struct inode *inode, struct file *file, int type) { … } static int ftrace_pid_open(struct inode *inode, struct file *file) { … } static int ftrace_no_pid_open(struct inode *inode, struct file *file) { … } static void ignore_task_cpu(void *data) { … } static ssize_t pid_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, int type) { … } static ssize_t ftrace_pid_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } static ssize_t ftrace_no_pid_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { … } static int ftrace_pid_release(struct inode *inode, struct file *file) { … } static const struct file_operations ftrace_pid_fops = …; static const struct file_operations ftrace_no_pid_fops = …; void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) { … } void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d_tracer) { … } /** * ftrace_kill - kill ftrace * * This function should be used by panic code. It stops ftrace * but in a not so nice way. If you need to simply kill ftrace * from a non-atomic section, use ftrace_kill. */ void ftrace_kill(void) { … } /** * ftrace_is_dead - Test if ftrace is dead or not. * * Returns: 1 if ftrace is "dead", zero otherwise. */ int ftrace_is_dead(void) { … } #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* * When registering ftrace_ops with IPMODIFY, it is necessary to make sure * it doesn't conflict with any direct ftrace_ops. If there is existing * direct ftrace_ops on a kernel function being patched, call * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing. * * @ops: ftrace_ops being registered. * * Returns: * 0 on success; * Negative on failure. */ static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) { … } /* * Similar to prepare_direct_functions_for_ipmodify, clean up after ops * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT * ops. */ static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) { … } #define lock_direct_mutex() … #define unlock_direct_mutex() … #else /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) { return 0; } static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) { } #define lock_direct_mutex … #define unlock_direct_mutex … #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ /* * Similar to register_ftrace_function, except we don't lock direct_mutex. */ static int register_ftrace_function_nolock(struct ftrace_ops *ops) { … } /** * register_ftrace_function - register a function for profiling * @ops: ops structure that holds the function for profiling. * * Register a function to be called by all functions in the * kernel. * * Note: @ops->func and all the functions it calls must be labeled * with "notrace", otherwise it will go into a * recursive loop. */ int register_ftrace_function(struct ftrace_ops *ops) { … } EXPORT_SYMBOL_GPL(…); /** * unregister_ftrace_function - unregister a function for profiling. * @ops: ops structure that holds the function to unregister * * Unregister a function that was added to be called by ftrace profiling. */ int unregister_ftrace_function(struct ftrace_ops *ops) { … } EXPORT_SYMBOL_GPL(…); static int symbols_cmp(const void *a, const void *b) { … } struct kallsyms_data { … }; /* This function gets called for all kernel and module symbols * and returns 1 in case we resolved all the requested symbols, * 0 otherwise. */ static int kallsyms_callback(void *data, const char *name, unsigned long addr) { … } /** * ftrace_lookup_symbols - Lookup addresses for array of symbols * * @sorted_syms: array of symbols pointers symbols to resolve, * must be alphabetically sorted * @cnt: number of symbols/addresses in @syms/@addrs arrays * @addrs: array for storing resulting addresses * * This function looks up addresses for array of symbols provided in * @syms array (must be alphabetically sorted) and stores them in * @addrs array, which needs to be big enough to store at least @cnt * addresses. * * Returns: 0 if all provided symbols are found, -ESRCH otherwise. */ int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs) { … } #ifdef CONFIG_SYSCTL #ifdef CONFIG_DYNAMIC_FTRACE static void ftrace_startup_sysctl(void) { … } static void ftrace_shutdown_sysctl(void) { … } #else #define ftrace_startup_sysctl … #define ftrace_shutdown_sysctl … #endif /* CONFIG_DYNAMIC_FTRACE */ static bool is_permanent_ops_registered(void) { … } static int ftrace_enable_sysctl(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { … } static struct ctl_table ftrace_sysctls[] = …; static int __init ftrace_sysctl_init(void) { … } late_initcall(ftrace_sysctl_init); #endif
Codebase Browser
linux