// SPDX-License-Identifier: GPL-2.0 /* * KFENCE guarded object allocator and fault handling. * * Copyright (C) 2020, Google LLC. */ #define pr_fmt(fmt) … #include <linux/atomic.h> #include <linux/bug.h> #include <linux/debugfs.h> #include <linux/hash.h> #include <linux/irq_work.h> #include <linux/jhash.h> #include <linux/kcsan-checks.h> #include <linux/kfence.h> #include <linux/kmemleak.h> #include <linux/list.h> #include <linux/lockdep.h> #include <linux/log2.h> #include <linux/memblock.h> #include <linux/moduleparam.h> #include <linux/notifier.h> #include <linux/panic_notifier.h> #include <linux/random.h> #include <linux/rcupdate.h> #include <linux/sched/clock.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/string.h> #include <asm/kfence.h> #include "kfence.h" /* Disables KFENCE on the first warning assuming an irrecoverable error. */ #define KFENCE_WARN_ON(cond) … /* === Data ================================================================= */ static bool kfence_enabled __read_mostly; static bool disabled_by_warn __read_mostly; unsigned long kfence_sample_interval __read_mostly = …; EXPORT_SYMBOL_GPL(…); /* Export for test modules. */ #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif #define MODULE_PARAM_PREFIX … static int kfence_enable_late(void); static int param_set_sample_interval(const char *val, const struct kernel_param *kp) { … } static int param_get_sample_interval(char *buffer, const struct kernel_param *kp) { … } static const struct kernel_param_ops sample_interval_param_ops = …; module_param_cb(…); /* Pool usage% threshold when currently covered allocations are skipped. */ static unsigned long kfence_skip_covered_thresh __read_mostly = …; module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong, 0644); /* Allocation burst count: number of excess KFENCE allocations per sample. */ static unsigned int kfence_burst __read_mostly; module_param_named(burst, kfence_burst, uint, 0644); /* If true, use a deferrable timer. */ static bool kfence_deferrable __read_mostly = … IS_ENABLED(…); module_param_named(deferrable, kfence_deferrable, bool, 0444); /* If true, check all canary bytes on panic. */ static bool kfence_check_on_panic __read_mostly; module_param_named(check_on_panic, kfence_check_on_panic, bool, 0444); /* The pool of pages used for guard pages and objects. */ char *__kfence_pool __read_mostly; EXPORT_SYMBOL(…); /* Export for test modules. */ /* * Per-object metadata, with one-to-one mapping of object metadata to * backing pages (in __kfence_pool). */ static_assert(…); struct kfence_metadata *kfence_metadata __read_mostly; /* * If kfence_metadata is not NULL, it may be accessed by kfence_shutdown_cache(). * So introduce kfence_metadata_init to initialize metadata, and then make * kfence_metadata visible after initialization is successful. This prevents * potential UAF or access to uninitialized metadata. */ static struct kfence_metadata *kfence_metadata_init __read_mostly; /* Freelist with available objects. */ static struct list_head kfence_freelist = …; static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */ /* * The static key to set up a KFENCE allocation; or if static keys are not used * to gate allocations, to avoid a load and compare if KFENCE is disabled. */ DEFINE_STATIC_KEY_FALSE(kfence_allocation_key); /* Gates the allocation, ensuring only one succeeds in a given period. */ atomic_t kfence_allocation_gate = …; /* * A Counting Bloom filter of allocation coverage: limits currently covered * allocations of the same source filling up the pool. * * Assuming a range of 15%-85% unique allocations in the pool at any point in * time, the below parameters provide a probablity of 0.02-0.33 for false * positive hits respectively: * * P(alloc_traces) = (1 - e^(-HNUM * (alloc_traces / SIZE)) ^ HNUM */ #define ALLOC_COVERED_HNUM … #define ALLOC_COVERED_ORDER … #define ALLOC_COVERED_SIZE … #define ALLOC_COVERED_HNEXT(h) … #define ALLOC_COVERED_MASK … static atomic_t alloc_covered[ALLOC_COVERED_SIZE]; /* Stack depth used to determine uniqueness of an allocation. */ #define UNIQUE_ALLOC_STACK_DEPTH … /* * Randomness for stack hashes, making the same collisions across reboots and * different machines less likely. */ static u32 stack_hash_seed __ro_after_init; /* Statistics counters for debugfs. */ enum kfence_counter_id { … }; static atomic_long_t counters[KFENCE_COUNTER_COUNT]; static const char *const counter_names[] = …; static_assert(…); /* === Internals ============================================================ */ static inline bool should_skip_covered(void) { … } static u32 get_alloc_stack_hash(unsigned long *stack_entries, size_t num_entries) { … } /* * Adds (or subtracts) count @val for allocation stack trace hash * @alloc_stack_hash from Counting Bloom filter. */ static void alloc_covered_add(u32 alloc_stack_hash, int val) { … } /* * Returns true if the allocation stack trace hash @alloc_stack_hash is * currently contained (non-zero count) in Counting Bloom filter. */ static bool alloc_covered_contains(u32 alloc_stack_hash) { … } static bool kfence_protect(unsigned long addr) { … } static bool kfence_unprotect(unsigned long addr) { … } static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta) { … } static inline bool kfence_obj_allocated(const struct kfence_metadata *meta) { … } /* * Update the object's metadata state, including updating the alloc/free stacks * depending on the state transition. */ static noinline void metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next, unsigned long *stack_entries, size_t num_stack_entries) { … } #ifdef CONFIG_KMSAN #define check_canary_attributes … #else #define check_canary_attributes … #endif /* Check canary byte at @addr. */ static check_canary_attributes bool check_canary_byte(u8 *addr) { … } static inline void set_canary(const struct kfence_metadata *meta) { … } static check_canary_attributes void check_canary(const struct kfence_metadata *meta) { … } static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t gfp, unsigned long *stack_entries, size_t num_stack_entries, u32 alloc_stack_hash) { … } static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie) { … } static void rcu_guarded_free(struct rcu_head *h) { … } /* * Initialization of the KFENCE pool after its allocation. * Returns 0 on success; otherwise returns the address up to * which partial initialization succeeded. */ static unsigned long kfence_init_pool(void) { … } static bool __init kfence_init_pool_early(void) { … } /* === DebugFS Interface ==================================================== */ static int stats_show(struct seq_file *seq, void *v) { … } DEFINE_SHOW_ATTRIBUTE(…); /* * debugfs seq_file operations for /sys/kernel/debug/kfence/objects. * start_object() and next_object() return the object index + 1, because NULL is used * to stop iteration. */ static void *start_object(struct seq_file *seq, loff_t *pos) { … } static void stop_object(struct seq_file *seq, void *v) { … } static void *next_object(struct seq_file *seq, void *v, loff_t *pos) { … } static int show_object(struct seq_file *seq, void *v) { … } static const struct seq_operations objects_sops = …; DEFINE_SEQ_ATTRIBUTE(…); static int kfence_debugfs_init(void) { … } late_initcall(kfence_debugfs_init); /* === Panic Notifier ====================================================== */ static void kfence_check_all_canary(void) { … } static int kfence_check_canary_callback(struct notifier_block *nb, unsigned long reason, void *arg) { … } static struct notifier_block kfence_check_canary_notifier = …; /* === Allocation Gate Timer ================================================ */ static struct delayed_work kfence_timer; #ifdef CONFIG_KFENCE_STATIC_KEYS /* Wait queue to wake up allocation-gate timer task. */ static DECLARE_WAIT_QUEUE_HEAD(allocation_wait); static void wake_up_kfence_timer(struct irq_work *work) { … } static DEFINE_IRQ_WORK(wake_up_kfence_timer_work, wake_up_kfence_timer); #endif /* * Set up delayed work, which will enable and disable the static key. We need to * use a work queue (rather than a simple timer), since enabling and disabling a * static key cannot be done from an interrupt. * * Note: Toggling a static branch currently causes IPIs, and here we'll end up * with a total of 2 IPIs to all CPUs. If this ends up a problem in future (with * more aggressive sampling intervals), we could get away with a variant that * avoids IPIs, at the cost of not immediately capturing allocations if the * instructions remain cached. */ static void toggle_allocation_gate(struct work_struct *work) { … } /* === Public interface ===================================================== */ void __init kfence_alloc_pool_and_metadata(void) { … } static void kfence_init_enable(void) { … } void __init kfence_init(void) { … } static int kfence_init_late(void) { … } static int kfence_enable_late(void) { … } void kfence_shutdown_cache(struct kmem_cache *s) { … } void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) { … } size_t kfence_ksize(const void *addr) { … } void *kfence_object_start(const void *addr) { … } void __kfence_free(void *addr) { … } bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) { … }
Codebase Browser
linux