// SPDX-License-Identifier: GPL-2.0 #define pr_fmt(fmt) … #define DISABLE_BRANCH_PROFILING #include <linux/atomic.h> #include <linux/compiler.h> #include <linux/errno.h> #include <linux/export.h> #include <linux/types.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/hashtable.h> #include <linux/init.h> #include <linux/kmsan-checks.h> #include <linux/mm.h> #include <linux/preempt.h> #include <linux/printk.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> #include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/kcov.h> #include <linux/refcount.h> #include <linux/log2.h> #include <asm/setup.h> #define kcov_debug(fmt, ...) … /* Number of 64-bit words written per one comparison: */ #define KCOV_WORDS_PER_CMP … /* * kcov descriptor (one per opened debugfs file). * State transitions of the descriptor: * - initial state after open() * - then there must be a single ioctl(KCOV_INIT_TRACE) call * - then, mmap() call (several calls are allowed but not useful) * - then, ioctl(KCOV_ENABLE, arg), where arg is * KCOV_TRACE_PC - to trace only the PCs * or * KCOV_TRACE_CMP - to trace only the comparison operands * - then, ioctl(KCOV_DISABLE) to disable the task. * Enabling/disabling ioctls can be repeated (only one task a time allowed). */ struct kcov { … }; struct kcov_remote_area { … }; struct kcov_remote { … }; static DEFINE_SPINLOCK(kcov_remote_lock); static DEFINE_HASHTABLE(kcov_remote_map, 4); static struct list_head kcov_remote_areas = …; struct kcov_percpu_data { … }; static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = …; /* Must be called with kcov_remote_lock locked. */ static struct kcov_remote *kcov_remote_find(u64 handle) { … } /* Must be called with kcov_remote_lock locked. */ static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) { … } /* Must be called with kcov_remote_lock locked. */ static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) { … } /* Must be called with kcov_remote_lock locked. */ static void kcov_remote_area_put(struct kcov_remote_area *area, unsigned int size) { … } /* * Unlike in_serving_softirq(), this function returns false when called during * a hardirq or an NMI that happened in the softirq context. */ static inline bool in_softirq_really(void) { … } static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) { … } static notrace unsigned long canonicalize_ip(unsigned long ip) { … } /* * Entry point from instrumented code. * This is called once per basic-block/edge. */ void notrace __sanitizer_cov_trace_pc(void) { … } EXPORT_SYMBOL(…); #ifdef CONFIG_KCOV_ENABLE_COMPARISONS static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip) { … } void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_cmp8(kcov_u64 arg1, kcov_u64 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_const_cmp8(kcov_u64 arg1, kcov_u64 arg2) { … } EXPORT_SYMBOL(…); void notrace __sanitizer_cov_trace_switch(kcov_u64 val, void *arg) { … } EXPORT_SYMBOL(…); #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ static void kcov_start(struct task_struct *t, struct kcov *kcov, unsigned int size, void *area, enum kcov_mode mode, int sequence) { … } static void kcov_stop(struct task_struct *t) { … } static void kcov_task_reset(struct task_struct *t) { … } void kcov_task_init(struct task_struct *t) { … } static void kcov_reset(struct kcov *kcov) { … } static void kcov_remote_reset(struct kcov *kcov) { … } static void kcov_disable(struct task_struct *t, struct kcov *kcov) { … } static void kcov_get(struct kcov *kcov) { … } static void kcov_put(struct kcov *kcov) { … } void kcov_task_exit(struct task_struct *t) { … } static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) { … } static int kcov_open(struct inode *inode, struct file *filep) { … } static int kcov_close(struct inode *inode, struct file *filep) { … } static int kcov_get_mode(unsigned long arg) { … } /* * Fault in a lazily-faulted vmalloc area before it can be used by * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the * vmalloc fault handling path is instrumented. */ static void kcov_fault_in_area(struct kcov *kcov) { … } static inline bool kcov_check_handle(u64 handle, bool common_valid, bool uncommon_valid, bool zero_valid) { … } static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, unsigned long arg) { … } static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { … } static const struct file_operations kcov_fops = …; /* * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section * of code in a kernel background thread or in a softirq to allow kcov to be * used to collect coverage from that part of code. * * The handle argument of kcov_remote_start() identifies a code section that is * used for coverage collection. A userspace process passes this handle to * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting * coverage for the code section identified by this handle. * * The usage of these annotations in the kernel code is different depending on * the type of the kernel thread whose code is being annotated. * * For global kernel threads that are spawned in a limited number of instances * (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for * softirqs, each instance must be assigned a unique 4-byte instance id. The * instance id is then combined with a 1-byte subsystem id to get a handle via * kcov_remote_handle(subsystem_id, instance_id). * * For local kernel threads that are spawned from system calls handler when a * user interacts with some kernel interface (e.g. vhost workers), a handle is * passed from a userspace process as the common_handle field of the * kcov_remote_arg struct (note, that the user must generate a handle by using * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an * arbitrary 4-byte non-zero number as the instance id). This common handle * then gets saved into the task_struct of the process that issued the * KCOV_REMOTE_ENABLE ioctl. When this process issues system calls that spawn * kernel threads, the common handle must be retrieved via kcov_common_handle() * and passed to the spawned threads via custom annotations. Those kernel * threads must in turn be annotated with kcov_remote_start(common_handle) and * kcov_remote_stop(). All of the threads that are spawned by the same process * obtain the same handle, hence the name "common". * * See Documentation/dev-tools/kcov.rst for more details. * * Internally, kcov_remote_start() looks up the kcov device associated with the * provided handle, allocates an area for coverage collection, and saves the * pointers to kcov and area into the current task_struct to allow coverage to * be collected via __sanitizer_cov_trace_pc(). * In turns kcov_remote_stop() clears those pointers from task_struct to stop * collecting coverage and copies all collected coverage into the kcov area. */ static inline bool kcov_mode_enabled(unsigned int mode) { … } static void kcov_remote_softirq_start(struct task_struct *t) { … } static void kcov_remote_softirq_stop(struct task_struct *t) { … } void kcov_remote_start(u64 handle) { … } EXPORT_SYMBOL(…); static void kcov_move_area(enum kcov_mode mode, void *dst_area, unsigned int dst_area_size, void *src_area) { … } /* See the comment before kcov_remote_start() for usage details. */ void kcov_remote_stop(void) { … } EXPORT_SYMBOL(…); /* See the comment before kcov_remote_start() for usage details. */ u64 kcov_common_handle(void) { … } EXPORT_SYMBOL(…); static int __init kcov_init(void) { … } device_initcall(kcov_init);
Codebase Browser
linux