// SPDX-License-Identifier: GPL-2.0 /* Support for MMIO probes. * Benefit many code from kprobes * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>. * 2007 Alexander Eichner * 2008 Pekka Paalanen <pq@iki.fi> */ #define pr_fmt(fmt) … #include <linux/list.h> #include <linux/rculist.h> #include <linux/spinlock.h> #include <linux/hash.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/uaccess.h> #include <linux/ptrace.h> #include <linux/preempt.h> #include <linux/percpu.h> #include <linux/kdebug.h> #include <linux/mutex.h> #include <linux/io.h> #include <linux/slab.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <linux/errno.h> #include <asm/debugreg.h> #include <linux/mmiotrace.h> #define KMMIO_PAGE_HASH_BITS … #define KMMIO_PAGE_TABLE_SIZE … struct kmmio_fault_page { … }; struct kmmio_delayed_release { … }; struct kmmio_context { … }; /* * The kmmio_lock is taken in int3 context, which is treated as NMI context. * This causes lockdep to complain about it bein in both NMI and normal * context. Hide it from lockdep, as it should not have any other locks * taken under it, and this is only enabled for debugging mmio anyway. */ static arch_spinlock_t kmmio_lock = …; /* Protected by kmmio_lock */ unsigned int kmmio_count; /* Read-protected by RCU, write-protected by kmmio_lock. */ static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE]; static LIST_HEAD(kmmio_probes); static struct list_head *kmmio_page_list(unsigned long addr) { … } /* Accessed per-cpu */ static DEFINE_PER_CPU(struct kmmio_context, kmmio_ctx); /* * this is basically a dynamic stabbing problem: * Could use the existing prio tree code or * Possible better implementations: * The Interval Skip List: A Data Structure for Finding All Intervals That * Overlap a Point (might be simple) * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup */ /* Get the kmmio at this addr (if any). You must be holding RCU read lock. */ static struct kmmio_probe *get_kmmio_probe(unsigned long addr) { … } /* You must be holding RCU read lock. */ static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long addr) { … } static void clear_pmd_presence(pmd_t *pmd, bool clear, pmdval_t *old) { … } static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old) { … } static int clear_page_presence(struct kmmio_fault_page *f, bool clear) { … } /* * Mark the given page as not present. Access to it will trigger a fault. * * Struct kmmio_fault_page is protected by RCU and kmmio_lock, but the * protection is ignored here. RCU read lock is assumed held, so the struct * will not disappear unexpectedly. Furthermore, the caller must guarantee, * that double arming the same virtual address (page) cannot occur. * * Double disarming on the other hand is allowed, and may occur when a fault * and mmiotrace shutdown happen simultaneously. */ static int arm_kmmio_fault_page(struct kmmio_fault_page *f) { … } /** Restore the given page to saved presence state. */ static void disarm_kmmio_fault_page(struct kmmio_fault_page *f) { … } /* * This is being called from do_page_fault(). * * We may be in an interrupt or a critical section. Also prefecthing may * trigger a page fault. We may be in the middle of process switch. * We cannot take any locks, because we could be executing especially * within a kmmio critical section. * * Local interrupts are disabled, so preemption cannot happen. * Do not enable interrupts, do not sleep, and watch out for other CPUs. */ /* * Interrupts are disabled on entry as trap3 is an interrupt gate * and they remain disabled throughout this function. */ int kmmio_handler(struct pt_regs *regs, unsigned long addr) { … } /* * Interrupts are disabled on entry as trap1 is an interrupt gate * and they remain disabled throughout this function. * This must always get called as the pair to kmmio_handler(). */ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs) { … } /* You must be holding kmmio_lock. */ static int add_kmmio_fault_page(unsigned long addr) { … } /* You must be holding kmmio_lock. */ static void release_kmmio_fault_page(unsigned long addr, struct kmmio_fault_page **release_list) { … } /* * With page-unaligned ioremaps, one or two armed pages may contain * addresses from outside the intended mapping. Events for these addresses * are currently silently dropped. The events may result only from programming * mistakes by accessing addresses before the beginning or past the end of a * mapping. */ int register_kmmio_probe(struct kmmio_probe *p) { … } EXPORT_SYMBOL(…); static void rcu_free_kmmio_fault_pages(struct rcu_head *head) { … } static void remove_kmmio_fault_pages(struct rcu_head *head) { … } /* * Remove a kmmio probe. You have to synchronize_rcu() before you can be * sure that the callbacks will not be called anymore. Only after that * you may actually release your struct kmmio_probe. * * Unregistering a kmmio fault page has three steps: * 1. release_kmmio_fault_page() * Disarm the page, wait a grace period to let all faults finish. * 2. remove_kmmio_fault_pages() * Remove the pages from kmmio_page_table. * 3. rcu_free_kmmio_fault_pages() * Actually free the kmmio_fault_page structs as with RCU. */ void unregister_kmmio_probe(struct kmmio_probe *p) { … } EXPORT_SYMBOL(…); static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val, void *args) { … } static struct notifier_block nb_die = …; int kmmio_init(void) { … } void kmmio_cleanup(void) { … }