// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2021, Microsoft Corporation. * * Authors: * Beau Belgrave <[email protected]> */ #include <linux/bitmap.h> #include <linux/cdev.h> #include <linux/hashtable.h> #include <linux/list.h> #include <linux/io.h> #include <linux/uio.h> #include <linux/ioctl.h> #include <linux/jhash.h> #include <linux/refcount.h> #include <linux/trace_events.h> #include <linux/tracefs.h> #include <linux/types.h> #include <linux/uaccess.h> #include <linux/highmem.h> #include <linux/init.h> #include <linux/user_events.h> #include "trace_dynevent.h" #include "trace_output.h" #include "trace.h" #define USER_EVENTS_PREFIX_LEN … #define FIELD_DEPTH_TYPE … #define FIELD_DEPTH_NAME … #define FIELD_DEPTH_SIZE … /* Limit how long of an event name plus args within the subsystem. */ #define MAX_EVENT_DESC … #define EVENT_NAME(user_event) … #define EVENT_TP_NAME(user_event) … #define MAX_FIELD_ARRAY_SIZE … /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These * bits are compared against an internal byte on the event to determine which * probes to print out to the user. * * These do not reflect the mapped bytes between the user and kernel space. */ #define EVENT_STATUS_FTRACE … #define EVENT_STATUS_PERF … #define EVENT_STATUS_OTHER … /* * Stores the system name, tables, and locks for a group of events. This * allows isolation for events by various means. */ struct user_event_group { … }; /* Group for init_user_ns mapping, top-most group */ static struct user_event_group *init_group; /* Max allowed events for the whole system */ static unsigned int max_user_events = …; /* Current number of events on the whole system */ static unsigned int current_user_events; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the * refcnt reaches one. */ struct user_event { … }; /* * Stores per-mm/event properties that enable an address to be * updated properly for each task. As tasks are forked, we use * these to track enablement sites that are tied to an event. */ struct user_event_enabler { … }; /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */ #define ENABLE_VAL_BIT_MASK … /* Bit 6 is for faulting status of enablement */ #define ENABLE_VAL_FAULTING_BIT … /* Bit 7 is for freeing status of enablement */ #define ENABLE_VAL_FREEING_BIT … /* Bit 8 is for marking 32-bit on 64-bit */ #define ENABLE_VAL_32_ON_64_BIT … #define ENABLE_VAL_COMPAT_MASK … /* Only duplicate the bit and compat values */ #define ENABLE_VAL_DUP_MASK … #define ENABLE_BITOPS(e) … #define ENABLE_BIT(e) … #define EVENT_MULTI_FORMAT(f) … /* Used for asynchronous faulting in of pages */ struct user_event_enabler_fault { … }; static struct kmem_cache *fault_cache; /* Global list of memory descriptors using user_events */ static LIST_HEAD(user_event_mms); static DEFINE_SPINLOCK(user_event_mms_lock); /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. * The lifetime of this struct is tied to the lifetime of the file. * These are not shared and only accessible by the file that created it. */ struct user_event_refs { … }; struct user_event_file_info { … }; #define VALIDATOR_ENSURE_NULL … #define VALIDATOR_REL … struct user_event_validator { … }; static inline void align_addr_bit(unsigned long *addr, int *bit, unsigned long *flags) { … } user_event_func_t; static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser, int reg_flags); static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm); static struct user_event_mm *user_event_mm_get_all(struct user_event *user); static void user_event_mm_put(struct user_event_mm *mm); static int destroy_user_event(struct user_event *user); static bool user_fields_match(struct user_event *user, int argc, const char **argv); static u32 user_event_key(char *name) { … } static bool user_event_capable(u16 reg_flags) { … } static struct user_event *user_event_get(struct user_event *user) { … } static void delayed_destroy_user_event(struct work_struct *work) { … } static void user_event_put(struct user_event *user, bool locked) { … } static void user_event_group_destroy(struct user_event_group *group) { … } static char *user_event_group_system_name(void) { … } static char *user_event_group_system_multi_name(void) { … } static struct user_event_group *current_user_event_group(void) { … } static struct user_event_group *user_event_group_create(void) { struct user_event_group *group; group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return NULL; group->system_name = user_event_group_system_name(); if (!group->system_name) goto error; group->system_multi_name = user_event_group_system_multi_name(); if (!group->system_multi_name) goto error; mutex_init(&group->reg_mutex); hash_init(group->register_table); return group; error: if (group) user_event_group_destroy(group); return NULL; }; static void user_event_enabler_destroy(struct user_event_enabler *enabler, bool locked) { … } static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr, int attempt) { … } static int user_event_enabler_write(struct user_event_mm *mm, struct user_event_enabler *enabler, bool fixup_fault, int *attempt); static void user_event_enabler_fault_fixup(struct work_struct *work) { … } static bool user_event_enabler_queue_fault(struct user_event_mm *mm, struct user_event_enabler *enabler, int attempt) { … } static int user_event_enabler_write(struct user_event_mm *mm, struct user_event_enabler *enabler, bool fixup_fault, int *attempt) { … } static bool user_event_enabler_exists(struct user_event_mm *mm, unsigned long uaddr, unsigned char bit) { … } static void user_event_enabler_update(struct user_event *user) { … } static bool user_event_enabler_dup(struct user_event_enabler *orig, struct user_event_mm *mm) { … } static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm) { … } static struct user_event_mm *user_event_mm_get_all(struct user_event *user) { … } static struct user_event_mm *user_event_mm_alloc(struct task_struct *t) { … } static void user_event_mm_attach(struct user_event_mm *user_mm, struct task_struct *t) { … } static struct user_event_mm *current_user_event_mm(void) { … } static void user_event_mm_destroy(struct user_event_mm *mm) { … } static void user_event_mm_put(struct user_event_mm *mm) { … } static void delayed_user_event_mm_put(struct work_struct *work) { … } void user_event_mm_remove(struct task_struct *t) { … } void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm) { … } static bool current_user_event_enabler_exists(unsigned long uaddr, unsigned char bit) { … } static struct user_event_enabler *user_event_enabler_create(struct user_reg *reg, struct user_event *user, int *write_result) { … } static __always_inline __must_check bool user_event_last_ref(struct user_event *user) { … } static __always_inline __must_check size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i) { … } static struct list_head *user_event_get_fields(struct trace_event_call *call) { … } /* * Parses a register command for user_events * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]] * * Example event named 'test' with a 20 char 'msg' field with an unsigned int * 'id' field after: * test char[20] msg;unsigned int id * * NOTE: Offsets are from the user data perspective, they are not from the * trace_entry/buffer perspective. We automatically add the common properties * sizes to the offset for the user. * * Upon success user_event has its ref count increased by 1. */ static int user_event_parse_cmd(struct user_event_group *group, char *raw_command, struct user_event **newuser, int reg_flags) { … } static int user_field_array_size(const char *type) { … } static int user_field_size(const char *type) { … } static void user_event_destroy_validators(struct user_event *user) { … } static void user_event_destroy_fields(struct user_event *user) { … } static int user_event_add_field(struct user_event *user, const char *type, const char *name, int offset, int size, int is_signed, int filter_type) { … } /* * Parses the values of a field within the description * Format: type name [size] */ static int user_event_parse_field(char *field, struct user_event *user, u32 *offset) { … } static int user_event_parse_fields(struct user_event *user, char *args) { … } static struct trace_event_fields user_event_fields_array[1]; static const char *user_field_format(const char *type) { … } static bool user_field_is_dyn_string(const char *type, const char **str_func) { … } #define LEN_OR_ZERO … static int user_dyn_field_set_string(int argc, const char **argv, int *iout, char *buf, int len, bool *colon) { … } static int user_field_set_string(struct ftrace_event_field *field, char *buf, int len, bool colon) { … } static int user_event_set_print_fmt(struct user_event *user, char *buf, int len) { … } #undef LEN_OR_ZERO static int user_event_create_print_fmt(struct user_event *user) { … } static enum print_line_t user_event_print_trace(struct trace_iterator *iter, int flags, struct trace_event *event) { … } static struct trace_event_functions user_event_funcs = …; static int user_event_set_call_visible(struct user_event *user, bool visible) { … } static int destroy_user_event(struct user_event *user) { … } static struct user_event *find_user_event(struct user_event_group *group, char *name, int argc, const char **argv, u32 flags, u32 *outkey) { … } static int user_event_validate(struct user_event *user, void *data, int len) { … } /* * Writes the user supplied payload out to a trace file. */ static void user_event_ftrace(struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted) { … } #ifdef CONFIG_PERF_EVENTS /* * Writes the user supplied payload out to perf ring buffer. */ static void user_event_perf(struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted) { … } #endif /* * Update the enabled bit among all user processes. */ static void update_enable_bit_for(struct user_event *user) { … } /* * Register callback for our events from tracing sub-systems. */ static int user_event_reg(struct trace_event_call *call, enum trace_reg type, void *data) { … } static int user_event_create(const char *raw_command) { … } static int user_event_show(struct seq_file *m, struct dyn_event *ev) { … } static bool user_event_is_busy(struct dyn_event *ev) { … } static int user_event_free(struct dyn_event *ev) { … } static bool user_field_match(struct ftrace_event_field *field, int argc, const char **argv, int *iout) { … } static bool user_fields_match(struct user_event *user, int argc, const char **argv) { … } static bool user_event_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev) { … } static struct dyn_event_operations user_event_dops = …; static int user_event_trace_register(struct user_event *user) { … } static int user_event_set_tp_name(struct user_event *user) { … } /* * Counts how many ';' without a trailing space are in the args. */ static int count_semis_no_space(char *args) { … } /* * Copies the arguments while ensuring all ';' have a trailing space. */ static char *insert_space_after_semis(char *args, int count) { … } static char **user_event_argv_split(char *args, int *argc) { … } /* * Parses the event name, arguments and flags then registers if successful. * The name buffer lifetime is owned by this method for success cases only. * Upon success the returned user_event has its ref count increased by 1. */ static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser, int reg_flags) { … } /* * Deletes previously created events if they are no longer being used. */ static int delete_user_event(struct user_event_group *group, char *name) { … } /* * Validates the user payload and writes via iterator. */ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) { … } static int user_events_open(struct inode *node, struct file *file) { … } static ssize_t user_events_write(struct file *file, const char __user *ubuf, size_t count, loff_t *ppos) { … } static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i) { … } static int user_events_ref_add(struct user_event_file_info *info, struct user_event *user) { … } static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg) { … } /* * Registers a user_event on behalf of a user process. */ static long user_events_ioctl_reg(struct user_event_file_info *info, unsigned long uarg) { … } /* * Deletes a user_event on behalf of a user process. */ static long user_events_ioctl_del(struct user_event_file_info *info, unsigned long uarg) { … } static long user_unreg_get(struct user_unreg __user *ureg, struct user_unreg *kreg) { … } static int user_event_mm_clear_bit(struct user_event_mm *user_mm, unsigned long uaddr, unsigned char bit, unsigned long flags) { … } /* * Unregisters an enablement address/bit within a task/user mm. */ static long user_events_ioctl_unreg(unsigned long uarg) { … } /* * Handles the ioctl from user mode to register or alter operations. */ static long user_events_ioctl(struct file *file, unsigned int cmd, unsigned long uarg) { … } /* * Handles the final close of the file from user mode. */ static int user_events_release(struct inode *node, struct file *file) { … } static const struct file_operations user_data_fops = …; static void *user_seq_start(struct seq_file *m, loff_t *pos) { … } static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos) { … } static void user_seq_stop(struct seq_file *m, void *p) { … } static int user_seq_show(struct seq_file *m, void *p) { … } static const struct seq_operations user_seq_ops = …; static int user_status_open(struct inode *node, struct file *file) { … } static const struct file_operations user_status_fops = …; /* * Creates a set of tracefs files to allow user mode interactions. */ static int create_user_tracefs(void) { … } static int set_max_user_events_sysctl(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { … } static struct ctl_table user_event_sysctls[] = …; static int __init trace_events_user_init(void) { … } fs_initcall(trace_events_user_init);
Codebase Browser
linux