// SPDX-License-Identifier: GPL-2.0 /* * Kprobes-based tracing events * * Created by Masami Hiramatsu <[email protected]> * */ #define pr_fmt(fmt) … #include <linux/bpf-cgroup.h> #include <linux/security.h> #include <linux/module.h> #include <linux/uaccess.h> #include <linux/rculist.h> #include <linux/error-injection.h> #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ #include "trace_dynevent.h" #include "trace_kprobe_selftest.h" #include "trace_probe.h" #include "trace_probe_tmpl.h" #include "trace_probe_kernel.h" #define KPROBE_EVENT_SYSTEM … #define KRETPROBE_MAXACTIVE_MAX … /* Kprobe early definition from command line */ static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; static int __init set_kprobe_boot_events(char *str) { … } __setup(…); static int trace_kprobe_create(const char *raw_command); static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); static int trace_kprobe_release(struct dyn_event *ev); static bool trace_kprobe_is_busy(struct dyn_event *ev); static bool trace_kprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev); static struct dyn_event_operations trace_kprobe_ops = …; /* * Kprobe event core functions */ struct trace_kprobe { … }; static bool is_trace_kprobe(struct dyn_event *ev) { … } static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) { … } /** * for_each_trace_kprobe - iterate over the trace_kprobe list * @pos: the struct trace_kprobe * for each entry * @dpos: the struct dyn_event * to use as a loop cursor */ #define for_each_trace_kprobe(pos, dpos) … static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) { … } static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) { … } static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) { … } static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) { … } static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, struct module *mod) { … } #ifdef CONFIG_MODULES static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) { … } #else static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) { return false; } #endif static bool trace_kprobe_is_busy(struct dyn_event *ev) { … } static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, int argc, const char **argv) { … } static bool trace_kprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev) { … } static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) { … } static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) { … } /* Return 0 if it fails to find the symbol address */ static nokprobe_inline unsigned long trace_kprobe_address(struct trace_kprobe *tk) { … } static nokprobe_inline struct trace_kprobe * trace_kprobe_primary_from_call(struct trace_event_call *call) { … } bool trace_kprobe_on_func_entry(struct trace_event_call *call) { … } bool trace_kprobe_error_injectable(struct trace_event_call *call) { … } static int register_kprobe_event(struct trace_kprobe *tk); static int unregister_kprobe_event(struct trace_kprobe *tk); static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); static int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs); static void free_trace_kprobe(struct trace_kprobe *tk) { … } /* * Allocate new trace_probe and initialize it (including kprobes). */ static struct trace_kprobe *alloc_trace_kprobe(const char *group, const char *event, void *addr, const char *symbol, unsigned long offs, int maxactive, int nargs, bool is_return) { … } static struct trace_kprobe *find_trace_kprobe(const char *event, const char *group) { … } static inline int __enable_trace_kprobe(struct trace_kprobe *tk) { … } static void __disable_trace_kprobe(struct trace_probe *tp) { … } /* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */ static int enable_trace_kprobe(struct trace_event_call *call, struct trace_event_file *file) { … } /* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */ static int disable_trace_kprobe(struct trace_event_call *call, struct trace_event_file *file) { … } #if defined(CONFIG_DYNAMIC_FTRACE) && \ !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) static bool __within_notrace_func(unsigned long addr) { unsigned long offset, size; if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) return false; /* Get the entry address of the target function */ addr -= offset; /* * Since ftrace_location_range() does inclusive range check, we need * to subtract 1 byte from the end address. */ return !ftrace_location_range(addr, addr + size - 1); } static bool within_notrace_func(struct trace_kprobe *tk) { unsigned long addr = trace_kprobe_address(tk); char symname[KSYM_NAME_LEN], *p; if (!__within_notrace_func(addr)) return false; /* Check if the address is on a suffixed-symbol */ if (!lookup_symbol_name(addr, symname)) { p = strchr(symname, '.'); if (!p) return true; *p = '\0'; addr = (unsigned long)kprobe_lookup_name(symname, 0); if (addr) return __within_notrace_func(addr); } return true; } #else #define within_notrace_func(tk) … #endif /* Internal register function - just handle k*probes and flags */ static int __register_trace_kprobe(struct trace_kprobe *tk) { … } /* Internal unregister function - just handle k*probes and flags */ static void __unregister_trace_kprobe(struct trace_kprobe *tk) { … } /* Unregister a trace_probe and probe_event */ static int unregister_trace_kprobe(struct trace_kprobe *tk) { … } static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, struct trace_kprobe *comp) { … } static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) { … } /* Register a trace_probe and probe_event */ static int register_trace_kprobe(struct trace_kprobe *tk) { … } #ifdef CONFIG_MODULES static int validate_module_probe_symbol(const char *modname, const char *symbol); static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) { … } /* Module notifier call back, checking event on the module */ static int trace_kprobe_module_callback(struct notifier_block *nb, unsigned long val, void *data) { … } static struct notifier_block trace_kprobe_module_nb = …; static int trace_kprobe_register_module_notifier(void) { … } #else static int trace_kprobe_register_module_notifier(void) { return 0; } #endif /* CONFIG_MODULES */ static int count_symbols(void *data, unsigned long unused) { … } struct sym_count_ctx { … }; static int count_mod_symbols(void *data, const char *name, unsigned long unused) { … } static unsigned int number_of_same_symbols(const char *mod, const char *func_name) { … } static int validate_module_probe_symbol(const char *modname, const char *symbol) { … } #ifdef CONFIG_MODULES /* Return NULL if the module is not loaded or under unloading. */ static struct module *try_module_get_by_name(const char *name) { … } #else #define try_module_get_by_name … #endif static int validate_probe_symbol(char *symbol) { … } static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs); static int __trace_kprobe_create(int argc, const char *argv[]) { … } static int trace_kprobe_create(const char *raw_command) { … } static int create_or_delete_trace_kprobe(const char *raw_command) { … } static int trace_kprobe_run_command(struct dynevent_cmd *cmd) { … } /** * kprobe_event_cmd_init - Initialize a kprobe event command object * @cmd: A pointer to the dynevent_cmd struct representing the new event * @buf: A pointer to the buffer used to build the command * @maxlen: The length of the buffer passed in @buf * * Initialize a synthetic event command object. Use this before * calling any of the other kprobe_event functions. */ void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) { … } EXPORT_SYMBOL_GPL(…); /** * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list * @cmd: A pointer to the dynevent_cmd struct representing the new event * @kretprobe: Is this a return probe? * @name: The name of the kprobe event * @loc: The location of the kprobe event * @...: Variable number of arg (pairs), one pair for each field * * NOTE: Users normally won't want to call this function directly, but * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically * adds a NULL to the end of the arg list. If this function is used * directly, make sure the last arg in the variable arg list is NULL. * * Generate a kprobe event command to be executed by * kprobe_event_gen_cmd_end(). This function can be used to generate the * complete command or only the first part of it; in the latter case, * kprobe_event_add_fields() can be used to add more fields following this. * * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This * returns -EINVAL if @loc == NULL. * * Return: 0 if successful, error otherwise. */ int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, const char *name, const char *loc, ...) { … } EXPORT_SYMBOL_GPL(…); /** * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list * @cmd: A pointer to the dynevent_cmd struct representing the new event * @...: Variable number of arg (pairs), one pair for each field * * NOTE: Users normally won't want to call this function directly, but * rather use the kprobe_event_add_fields() wrapper, which * automatically adds a NULL to the end of the arg list. If this * function is used directly, make sure the last arg in the variable * arg list is NULL. * * Add probe fields to an existing kprobe command using a variable * list of args. Fields are added in the same order they're listed. * * Return: 0 if successful, error otherwise. */ int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) { … } EXPORT_SYMBOL_GPL(…); /** * kprobe_event_delete - Delete a kprobe event * @name: The name of the kprobe event to delete * * Delete a kprobe event with the give @name from kernel code rather * than directly from the command line. * * Return: 0 if successful, error otherwise. */ int kprobe_event_delete(const char *name) { … } EXPORT_SYMBOL_GPL(…); static int trace_kprobe_release(struct dyn_event *ev) { … } static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) { … } static int probes_seq_show(struct seq_file *m, void *v) { … } static const struct seq_operations probes_seq_op = …; static int probes_open(struct inode *inode, struct file *file) { … } static ssize_t probes_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { … } static const struct file_operations kprobe_events_ops = …; static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) { … } /* Probes profiling interfaces */ static int probes_profile_seq_show(struct seq_file *m, void *v) { … } static const struct seq_operations profile_seq_op = …; static int profile_open(struct inode *inode, struct file *file) { … } static const struct file_operations kprobe_profile_ops = …; /* Note that we don't verify it, since the code does not come from user space */ static int process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, void *dest, void *base) { … } NOKPROBE_SYMBOL(…) /* Kprobe handler */ static nokprobe_inline void __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, struct trace_event_file *trace_file) { … } static void kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kprobe_trace_func); /* Kretprobe handler */ static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) { … } static nokprobe_inline void __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, struct pt_regs *regs, struct trace_event_file *trace_file) { … } static void kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kretprobe_trace_func); /* Event entry printers */ static enum print_line_t print_kprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event) { … } static enum print_line_t print_kretprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event) { … } static int kprobe_event_define_fields(struct trace_event_call *event_call) { … } static int kretprobe_event_define_fields(struct trace_event_call *event_call) { … } #ifdef CONFIG_PERF_EVENTS /* Kprobe profile handler */ static int kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kprobe_perf_func); /* Kretprobe profile handler */ static void kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kretprobe_perf_func); int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, const char **symbol, u64 *probe_offset, u64 *probe_addr, unsigned long *missed, bool perf_type_tracepoint) { … } #endif /* CONFIG_PERF_EVENTS */ /* * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. * * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe * lockless, but we can't race with this __init function. */ static int kprobe_register(struct trace_event_call *event, enum trace_reg type, void *data) { … } static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kprobe_dispatcher); static int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(kretprobe_dispatcher); static struct trace_event_functions kretprobe_funcs = …; static struct trace_event_functions kprobe_funcs = …; static struct trace_event_fields kretprobe_fields_array[] = …; static struct trace_event_fields kprobe_fields_array[] = …; static inline void init_trace_event_call(struct trace_kprobe *tk) { … } static int register_kprobe_event(struct trace_kprobe *tk) { … } static int unregister_kprobe_event(struct trace_kprobe *tk) { … } #ifdef CONFIG_PERF_EVENTS /* create a trace_kprobe, but don't add it to global lists */ struct trace_event_call * create_local_trace_kprobe(char *func, void *addr, unsigned long offs, bool is_return) { … } void destroy_local_trace_kprobe(struct trace_event_call *event_call) { … } #endif /* CONFIG_PERF_EVENTS */ static __init void enable_boot_kprobe_events(void) { … } static __init void setup_boot_kprobe_events(void) { … } /* * Register dynevent at core_initcall. This allows kernel to setup kprobe * events in postcore_initcall without tracefs. */ static __init int init_kprobe_trace_early(void) { … } core_initcall(init_kprobe_trace_early); /* Make a tracefs interface for controlling probe points */ static __init int init_kprobe_trace(void) { … } fs_initcall(init_kprobe_trace); #ifdef CONFIG_FTRACE_STARTUP_TEST static __init struct trace_event_file * find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) { … } /* * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this * stage, we can do this lockless. */ static __init int kprobe_trace_self_tests_init(void) { … } late_initcall(kprobe_trace_self_tests_init); #endif
Codebase Browser
linux