// SPDX-License-Identifier: GPL-2.0-only /* * kernel/lockdep.c * * Runtime locking correctness validator * * Started by Ingo Molnar: * * Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <[email protected]> * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra * * this code maps all the lock dependencies as they occur in a live kernel * and will warn about the following classes of locking bugs: * * - lock inversion scenarios * - circular lock dependencies * - hardirq/softirq safe/unsafe locking bugs * * Bugs are reported even if the current locking scenario does not cause * any deadlock at this point. * * I.e. if anytime in the past two locks were taken in a different order, * even if it happened for another task, even if those were different * locks (but of the same class as this lock), this code will detect it. * * Thanks to Arjan van de Ven for coming up with the initial idea of * mapping lock dependencies runtime. */ #define DISABLE_BRANCH_PROFILING #include <linux/mutex.h> #include <linux/sched.h> #include <linux/sched/clock.h> #include <linux/sched/task.h> #include <linux/sched/mm.h> #include <linux/delay.h> #include <linux/module.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/spinlock.h> #include <linux/kallsyms.h> #include <linux/interrupt.h> #include <linux/stacktrace.h> #include <linux/debug_locks.h> #include <linux/irqflags.h> #include <linux/utsname.h> #include <linux/hash.h> #include <linux/ftrace.h> #include <linux/stringify.h> #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/gfp.h> #include <linux/random.h> #include <linux/jhash.h> #include <linux/nmi.h> #include <linux/rcupdate.h> #include <linux/kprobes.h> #include <linux/lockdep.h> #include <linux/context_tracking.h> #include <asm/sections.h> #include "lockdep_internals.h" #include <trace/events/lock.h> #ifdef CONFIG_PROVE_LOCKING static int prove_locking = …; module_param(prove_locking, int, 0644); #else #define prove_locking … #endif #ifdef CONFIG_LOCK_STAT static int lock_stat = …; module_param(lock_stat, int, 0644); #else #define lock_stat … #endif #ifdef CONFIG_SYSCTL static struct ctl_table kern_lockdep_table[] = …; static __init int kernel_lockdep_sysctls_init(void) { … } late_initcall(kernel_lockdep_sysctls_init); #endif /* CONFIG_SYSCTL */ DEFINE_PER_CPU(unsigned int, lockdep_recursion); EXPORT_PER_CPU_SYMBOL_GPL(…); static __always_inline bool lockdep_enabled(void) { … } /* * lockdep_lock: protects the lockdep graph, the hashes and the * class/list/hash allocators. * * This is one of the rare exceptions where it's justified * to use a raw spinlock - we really dont want the spinlock * code to recurse back into the lockdep code... */ static arch_spinlock_t __lock = …; static struct task_struct *__owner; static inline void lockdep_lock(void) { … } static inline void lockdep_unlock(void) { … } static inline bool lockdep_assert_locked(void) { … } static struct task_struct *lockdep_selftest_task_struct; static int graph_lock(void) { … } static inline void graph_unlock(void) { … } /* * Turn lock debugging off and return with 0 if it was off already, * and also release the graph lock: */ static inline int debug_locks_off_graph_unlock(void) { … } unsigned long nr_list_entries; static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; static DECLARE_BITMAP(list_entries_in_use, MAX_LOCKDEP_ENTRIES); /* * All data structures here are protected by the global debug_lock. * * nr_lock_classes is the number of elements of lock_classes[] that is * in use. */ #define KEYHASH_BITS … #define KEYHASH_SIZE … static struct hlist_head lock_keys_hash[KEYHASH_SIZE]; unsigned long nr_lock_classes; unsigned long nr_zapped_classes; unsigned long max_lock_class_idx; struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS); static inline struct lock_class *hlock_class(struct held_lock *hlock) { … } #ifdef CONFIG_LOCK_STAT static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats); static inline u64 lockstat_clock(void) { … } static int lock_point(unsigned long points[], unsigned long ip) { … } static void lock_time_inc(struct lock_time *lt, u64 time) { … } static inline void lock_time_add(struct lock_time *src, struct lock_time *dst) { … } struct lock_class_stats lock_stats(struct lock_class *class) { … } void clear_lock_stats(struct lock_class *class) { … } static struct lock_class_stats *get_lock_stats(struct lock_class *class) { … } static void lock_release_holdtime(struct held_lock *hlock) { … } #else static inline void lock_release_holdtime(struct held_lock *hlock) { } #endif /* * We keep a global list of all lock classes. The list is only accessed with * the lockdep spinlock lock held. free_lock_classes is a list with free * elements. These elements are linked together by the lock_entry member in * struct lock_class. */ static LIST_HEAD(all_lock_classes); static LIST_HEAD(free_lock_classes); /** * struct pending_free - information about data structures about to be freed * @zapped: Head of a list with struct lock_class elements. * @lock_chains_being_freed: Bitmap that indicates which lock_chains[] elements * are about to be freed. */ struct pending_free { … }; /** * struct delayed_free - data structures used for delayed freeing * * A data structure for delayed freeing of data structures that may be * accessed by RCU readers at the time these were freed. * * @rcu_head: Used to schedule an RCU callback for freeing data structures. * @index: Index of @pf to which freed data structures are added. * @scheduled: Whether or not an RCU callback has been scheduled. * @pf: Array with information about data structures about to be freed. */ static struct delayed_free { … } delayed_free; /* * The lockdep classes are in a hash-table as well, for fast lookup: */ #define CLASSHASH_BITS … #define CLASSHASH_SIZE … #define __classhashfn(key) … #define classhashentry(key) … static struct hlist_head classhash_table[CLASSHASH_SIZE]; /* * We put the lock dependency chains into a hash-table as well, to cache * their existence: */ #define CHAINHASH_BITS … #define CHAINHASH_SIZE … #define __chainhashfn(chain) … #define chainhashentry(chain) … static struct hlist_head chainhash_table[CHAINHASH_SIZE]; /* * the id of held_lock */ static inline u16 hlock_id(struct held_lock *hlock) { … } static inline unsigned int chain_hlock_class_idx(u16 hlock_id) { … } /* * The hash key of the lock dependency chains is a hash itself too: * it's a hash of all locks taken up to that lock, including that lock. * It's a 64-bit hash, because it's important for the keys to be * unique. */ static inline u64 iterate_chain_key(u64 key, u32 idx) { … } void lockdep_init_task(struct task_struct *task) { … } static __always_inline void lockdep_recursion_inc(void) { … } static __always_inline void lockdep_recursion_finish(void) { … } void lockdep_set_selftest_task(struct task_struct *task) { … } /* * Debugging switches: */ #define VERBOSE … #define VERY_VERBOSE … #if VERBOSE #define HARDIRQ_VERBOSE … #define SOFTIRQ_VERBOSE … #else #define HARDIRQ_VERBOSE … #define SOFTIRQ_VERBOSE … #endif #if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE /* * Quick filtering for interesting events: */ static int class_filter(struct lock_class *class) { #if 0 /* Example */ if (class->name_version == 1 && !strcmp(class->name, "lockname")) return 1; if (class->name_version == 1 && !strcmp(class->name, "&struct->lockfield")) return 1; #endif /* Filter everything else. 1 would be to allow everything else */ return 0; } #endif static int verbose(struct lock_class *class) { … } static void print_lockdep_off(const char *bug_msg) { … } unsigned long nr_stack_trace_entries; #ifdef CONFIG_PROVE_LOCKING /** * struct lock_trace - single stack backtrace * @hash_entry: Entry in a stack_trace_hash[] list. * @hash: jhash() of @entries. * @nr_entries: Number of entries in @entries. * @entries: Actual stack backtrace. */ struct lock_trace { … }; #define LOCK_TRACE_SIZE_IN_LONGS … /* * Stack-trace: sequence of lock_trace structures. Protected by the graph_lock. */ static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES]; static struct hlist_head stack_trace_hash[STACK_TRACE_HASH_SIZE]; static bool traces_identical(struct lock_trace *t1, struct lock_trace *t2) { … } static struct lock_trace *save_trace(void) { … } /* Return the number of stack traces in the stack_trace[] array. */ u64 lockdep_stack_trace_count(void) { … } /* Return the number of stack hash chains that have at least one stack trace. */ u64 lockdep_stack_hash_count(void) { … } #endif unsigned int nr_hardirq_chains; unsigned int nr_softirq_chains; unsigned int nr_process_chains; unsigned int max_lockdep_depth; #ifdef CONFIG_DEBUG_LOCKDEP /* * Various lockdep statistics: */ DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats); #endif #ifdef CONFIG_PROVE_LOCKING /* * Locking printouts: */ #define __USAGE(__STATE) … static const char *usage_str[] = …; #endif const char *__get_key_name(const struct lockdep_subclass_key *key, char *str) { … } static inline unsigned long lock_flag(enum lock_usage_bit bit) { … } static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit) { … } void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS]) { … } static void __print_lock_name(struct held_lock *hlock, struct lock_class *class) { … } static void print_lock_name(struct held_lock *hlock, struct lock_class *class) { … } static void print_lockdep_cache(struct lockdep_map *lock) { … } static void print_lock(struct held_lock *hlock) { … } static void lockdep_print_held_locks(struct task_struct *p) { … } static void print_kernel_ident(void) { … } static int very_verbose(struct lock_class *class) { … } /* * Is this the address of a static object: */ #ifdef __KERNEL__ static int static_obj(const void *obj) { … } #endif /* * To make lock name printouts unique, we calculate a unique * class->name_version generation counter. The caller must hold the graph * lock. */ static int count_matching_names(struct lock_class *new_class) { … } /* used from NMI context -- must be lockless */ static noinstr struct lock_class * look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) { … } /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the * canonical address of the lock (per cpu offset removed) is used. */ static bool assign_lock_key(struct lockdep_map *lock) { … } #ifdef CONFIG_DEBUG_LOCKDEP /* Check whether element @e occurs in list @h */ static bool in_list(struct list_head *e, struct list_head *h) { … } /* * Check whether entry @e occurs in any of the locks_after or locks_before * lists. */ static bool in_any_class_list(struct list_head *e) { … } static bool class_lock_list_valid(struct lock_class *c, struct list_head *h) { … } #ifdef CONFIG_PROVE_LOCKING static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS]; #endif static bool check_lock_chain_key(struct lock_chain *chain) { … } static bool in_any_zapped_class_list(struct lock_class *class) { … } static bool __check_data_structures(void) { … } int check_consistency = …; module_param(check_consistency, int, 0644); static void check_data_structures(void) { … } #else /* CONFIG_DEBUG_LOCKDEP */ static inline void check_data_structures(void) { } #endif /* CONFIG_DEBUG_LOCKDEP */ static void init_chain_block_buckets(void); /* * Initialize the lock_classes[] array elements, the free_lock_classes list * and also the delayed_free structure. */ static void init_data_structures_once(void) { … } static inline struct hlist_head *keyhashentry(const struct lock_class_key *key) { … } /* Register a dynamically allocated key. */ void lockdep_register_key(struct lock_class_key *key) { … } EXPORT_SYMBOL_GPL(…); /* Check whether a key has been registered as a dynamic key. */ static bool is_dynamic_key(const struct lock_class_key *key) { … } /* * Register a lock's class in the hash-table, if the class is not present * yet. Otherwise we look it up. We cache the result in the lock object * itself, so actual lookup of the hash should be once per lock object. */ static struct lock_class * register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) { … } #ifdef CONFIG_PROVE_LOCKING /* * Allocate a lockdep entry. (assumes the graph_lock held, returns * with NULL on failure) */ static struct lock_list *alloc_list_entry(void) { … } /* * Add a new dependency to the head of the list: */ static int add_lock_to_list(struct lock_class *this, struct lock_class *links_to, struct list_head *head, u16 distance, u8 dep, const struct lock_trace *trace) { … } /* * For good efficiency of modular, we use power of 2 */ #define MAX_CIRCULAR_QUEUE_SIZE … #define CQ_MASK … /* * The circular_queue and helpers are used to implement graph * breadth-first search (BFS) algorithm, by which we can determine * whether there is a path from a lock to another. In deadlock checks, * a path from the next lock to be acquired to a previous held lock * indicates that adding the <prev> -> <next> lock dependency will * produce a circle in the graph. Breadth-first search instead of * depth-first search is used in order to find the shortest (circular) * path. */ struct circular_queue { … }; static struct circular_queue lock_cq; unsigned int max_bfs_queue_depth; static unsigned int lockdep_dependency_gen_id; static inline void __cq_init(struct circular_queue *cq) { … } static inline int __cq_empty(struct circular_queue *cq) { … } static inline int __cq_full(struct circular_queue *cq) { … } static inline int __cq_enqueue(struct circular_queue *cq, struct lock_list *elem) { … } /* * Dequeue an element from the circular_queue, return a lock_list if * the queue is not empty, or NULL if otherwise. */ static inline struct lock_list * __cq_dequeue(struct circular_queue *cq) { … } static inline unsigned int __cq_get_elem_count(struct circular_queue *cq) { … } static inline void mark_lock_accessed(struct lock_list *lock) { … } static inline void visit_lock_entry(struct lock_list *lock, struct lock_list *parent) { … } static inline unsigned long lock_accessed(struct lock_list *lock) { … } static inline struct lock_list *get_lock_parent(struct lock_list *child) { … } static inline int get_lock_depth(struct lock_list *child) { … } /* * Return the forward or backward dependency list. * * @lock: the lock_list to get its class's dependency list * @offset: the offset to struct lock_class to determine whether it is * locks_after or locks_before */ static inline struct list_head *get_dep_list(struct lock_list *lock, int offset) { … } /* * Return values of a bfs search: * * BFS_E* indicates an error * BFS_R* indicates a result (match or not) * * BFS_EINVALIDNODE: Find a invalid node in the graph. * * BFS_EQUEUEFULL: The queue is full while doing the bfs. * * BFS_RMATCH: Find the matched node in the graph, and put that node into * *@target_entry. * * BFS_RNOMATCH: Haven't found the matched node and keep *@target_entry * _unchanged_. */ enum bfs_result { … }; /* * bfs_result < 0 means error */ static inline bool bfs_error(enum bfs_result res) { … } /* * DEP_*_BIT in lock_list::dep * * For dependency @prev -> @next: * * SR: @prev is shared reader (->read != 0) and @next is recursive reader * (->read == 2) * ER: @prev is exclusive locker (->read == 0) and @next is recursive reader * SN: @prev is shared reader and @next is non-recursive locker (->read != 2) * EN: @prev is exclusive locker and @next is non-recursive locker * * Note that we define the value of DEP_*_BITs so that: * bit0 is prev->read == 0 * bit1 is next->read != 2 */ #define DEP_SR_BIT … #define DEP_ER_BIT … #define DEP_SN_BIT … #define DEP_EN_BIT … #define DEP_SR_MASK … #define DEP_ER_MASK … #define DEP_SN_MASK … #define DEP_EN_MASK … static inline unsigned int __calc_dep_bit(struct held_lock *prev, struct held_lock *next) { … } static inline u8 calc_dep(struct held_lock *prev, struct held_lock *next) { … } /* * calculate the dep_bit for backwards edges. We care about whether @prev is * shared and whether @next is recursive. */ static inline unsigned int __calc_dep_bitb(struct held_lock *prev, struct held_lock *next) { … } static inline u8 calc_depb(struct held_lock *prev, struct held_lock *next) { … } /* * Initialize a lock_list entry @lock belonging to @class as the root for a BFS * search. */ static inline void __bfs_init_root(struct lock_list *lock, struct lock_class *class) { … } /* * Initialize a lock_list entry @lock based on a lock acquisition @hlock as the * root for a BFS search. * * ->only_xr of the initial lock node is set to @hlock->read == 2, to make sure * that <prev> -> @hlock and @hlock -> <whatever __bfs() found> is not -(*R)-> * and -(S*)->. */ static inline void bfs_init_root(struct lock_list *lock, struct held_lock *hlock) { … } /* * Similar to bfs_init_root() but initialize the root for backwards BFS. * * ->only_xr of the initial lock node is set to @hlock->read != 0, to make sure * that <next> -> @hlock and @hlock -> <whatever backwards BFS found> is not * -(*S)-> and -(R*)-> (reverse order of -(*R)-> and -(S*)->). */ static inline void bfs_init_rootb(struct lock_list *lock, struct held_lock *hlock) { … } static inline struct lock_list *__bfs_next(struct lock_list *lock, int offset) { … } /* * Breadth-First Search to find a strong path in the dependency graph. * * @source_entry: the source of the path we are searching for. * @data: data used for the second parameter of @match function * @match: match function for the search * @target_entry: pointer to the target of a matched path * @offset: the offset to struct lock_class to determine whether it is * locks_after or locks_before * * We may have multiple edges (considering different kinds of dependencies, * e.g. ER and SN) between two nodes in the dependency graph. But * only the strong dependency path in the graph is relevant to deadlocks. A * strong dependency path is a dependency path that doesn't have two adjacent * dependencies as -(*R)-> -(S*)->, please see: * * Documentation/locking/lockdep-design.rst * * for more explanation of the definition of strong dependency paths * * In __bfs(), we only traverse in the strong dependency path: * * In lock_list::only_xr, we record whether the previous dependency only * has -(*R)-> in the search, and if it does (prev only has -(*R)->), we * filter out any -(S*)-> in the current dependency and after that, the * ->only_xr is set according to whether we only have -(*R)-> left. */ static enum bfs_result __bfs(struct lock_list *source_entry, void *data, bool (*match)(struct lock_list *entry, void *data), bool (*skip)(struct lock_list *entry, void *data), struct lock_list **target_entry, int offset) { … } static inline enum bfs_result __bfs_forwards(struct lock_list *src_entry, void *data, bool (*match)(struct lock_list *entry, void *data), bool (*skip)(struct lock_list *entry, void *data), struct lock_list **target_entry) { … } static inline enum bfs_result __bfs_backwards(struct lock_list *src_entry, void *data, bool (*match)(struct lock_list *entry, void *data), bool (*skip)(struct lock_list *entry, void *data), struct lock_list **target_entry) { … } static void print_lock_trace(const struct lock_trace *trace, unsigned int spaces) { … } /* * Print a dependency chain entry (this is only done when a deadlock * has been detected): */ static noinline void print_circular_bug_entry(struct lock_list *target, int depth) { … } static void print_circular_lock_scenario(struct held_lock *src, struct held_lock *tgt, struct lock_list *prt) { … } /* * When a circular dependency is detected, print the * header first: */ static noinline void print_circular_bug_header(struct lock_list *entry, unsigned int depth, struct held_lock *check_src, struct held_lock *check_tgt) { … } /* * We are about to add A -> B into the dependency graph, and in __bfs() a * strong dependency path A -> .. -> B is found: hlock_class equals * entry->class. * * If A -> .. -> B can replace A -> B in any __bfs() search (means the former * is _stronger_ than or equal to the latter), we consider A -> B as redundant. * For example if A -> .. -> B is -(EN)-> (i.e. A -(E*)-> .. -(*N)-> B), and A * -> B is -(ER)-> or -(EN)->, then we don't need to add A -> B into the * dependency graph, as any strong path ..-> A -> B ->.. we can get with * having dependency A -> B, we could already get a equivalent path ..-> A -> * .. -> B -> .. with A -> .. -> B. Therefore A -> B is redundant. * * We need to make sure both the start and the end of A -> .. -> B is not * weaker than A -> B. For the start part, please see the comment in * check_redundant(). For the end part, we need: * * Either * * a) A -> B is -(*R)-> (everything is not weaker than that) * * or * * b) A -> .. -> B is -(*N)-> (nothing is stronger than this) * */ static inline bool hlock_equal(struct lock_list *entry, void *data) { … } /* * We are about to add B -> A into the dependency graph, and in __bfs() a * strong dependency path A -> .. -> B is found: hlock_class equals * entry->class. * * We will have a deadlock case (conflict) if A -> .. -> B -> A is a strong * dependency cycle, that means: * * Either * * a) B -> A is -(E*)-> * * or * * b) A -> .. -> B is -(*N)-> (i.e. A -> .. -(*N)-> B) * * as then we don't have -(*R)-> -(S*)-> in the cycle. */ static inline bool hlock_conflict(struct lock_list *entry, void *data) { … } static noinline void print_circular_bug(struct lock_list *this, struct lock_list *target, struct held_lock *check_src, struct held_lock *check_tgt) { … } static noinline void print_bfs_bug(int ret) { … } static bool noop_count(struct lock_list *entry, void *data) { … } static unsigned long __lockdep_count_forward_deps(struct lock_list *this) { … } unsigned long lockdep_count_forward_deps(struct lock_class *class) { … } static unsigned long __lockdep_count_backward_deps(struct lock_list *this) { … } unsigned long lockdep_count_backward_deps(struct lock_class *class) { … } /* * Check that the dependency graph starting at <src> can lead to * <target> or not. */ static noinline enum bfs_result check_path(struct held_lock *target, struct lock_list *src_entry, bool (*match)(struct lock_list *entry, void *data), bool (*skip)(struct lock_list *entry, void *data), struct lock_list **target_entry) { … } static void print_deadlock_bug(struct task_struct *, struct held_lock *, struct held_lock *); /* * Prove that the dependency graph starting at <src> can not * lead to <target>. If it can, there is a circle when adding * <target> -> <src> dependency. * * Print an error and return BFS_RMATCH if it does. */ static noinline enum bfs_result check_noncircular(struct held_lock *src, struct held_lock *target, struct lock_trace **const trace) { … } #ifdef CONFIG_TRACE_IRQFLAGS /* * Forwards and backwards subgraph searching, for the purposes of * proving that two subgraphs can be connected by a new dependency * without creating any illegal irq-safe -> irq-unsafe lock dependency. * * A irq safe->unsafe deadlock happens with the following conditions: * * 1) We have a strong dependency path A -> ... -> B * * 2) and we have ENABLED_IRQ usage of B and USED_IN_IRQ usage of A, therefore * irq can create a new dependency B -> A (consider the case that a holder * of B gets interrupted by an irq whose handler will try to acquire A). * * 3) the dependency circle A -> ... -> B -> A we get from 1) and 2) is a * strong circle: * * For the usage bits of B: * a) if A -> B is -(*N)->, then B -> A could be any type, so any * ENABLED_IRQ usage suffices. * b) if A -> B is -(*R)->, then B -> A must be -(E*)->, so only * ENABLED_IRQ_*_READ usage suffices. * * For the usage bits of A: * c) if A -> B is -(E*)->, then B -> A could be any type, so any * USED_IN_IRQ usage suffices. * d) if A -> B is -(S*)->, then B -> A must be -(*N)->, so only * USED_IN_IRQ_*_READ usage suffices. */ /* * There is a strong dependency path in the dependency graph: A -> B, and now * we need to decide which usage bit of A should be accumulated to detect * safe->unsafe bugs. * * Note that usage_accumulate() is used in backwards search, so ->only_xr * stands for whether A -> B only has -(S*)-> (in this case ->only_xr is true). * * As above, if only_xr is false, which means A -> B has -(E*)-> dependency * path, any usage of A should be considered. Otherwise, we should only * consider _READ usage. */ static inline bool usage_accumulate(struct lock_list *entry, void *mask) { … } /* * There is a strong dependency path in the dependency graph: A -> B, and now * we need to decide which usage bit of B conflicts with the usage bits of A, * i.e. which usage bit of B may introduce safe->unsafe deadlocks. * * As above, if only_xr is false, which means A -> B has -(*N)-> dependency * path, any usage of B should be considered. Otherwise, we should only * consider _READ usage. */ static inline bool usage_match(struct lock_list *entry, void *mask) { … } static inline bool usage_skip(struct lock_list *entry, void *mask) { … } /* * Find a node in the forwards-direction dependency sub-graph starting * at @root->class that matches @bit. * * Return BFS_MATCH if such a node exists in the subgraph, and put that node * into *@target_entry. */ static enum bfs_result find_usage_forwards(struct lock_list *root, unsigned long usage_mask, struct lock_list **target_entry) { … } /* * Find a node in the backwards-direction dependency sub-graph starting * at @root->class that matches @bit. */ static enum bfs_result find_usage_backwards(struct lock_list *root, unsigned long usage_mask, struct lock_list **target_entry) { … } static void print_lock_class_header(struct lock_class *class, int depth) { … } /* * Dependency path printing: * * After BFS we get a lock dependency path (linked via ->parent of lock_list), * printing out each lock in the dependency path will help on understanding how * the deadlock could happen. Here are some details about dependency path * printing: * * 1) A lock_list can be either forwards or backwards for a lock dependency, * for a lock dependency A -> B, there are two lock_lists: * * a) lock_list in the ->locks_after list of A, whose ->class is B and * ->links_to is A. In this case, we can say the lock_list is * "A -> B" (forwards case). * * b) lock_list in the ->locks_before list of B, whose ->class is A * and ->links_to is B. In this case, we can say the lock_list is * "B <- A" (bacwards case). * * The ->trace of both a) and b) point to the call trace where B was * acquired with A held. * * 2) A "helper" lock_list is introduced during BFS, this lock_list doesn't * represent a certain lock dependency, it only provides an initial entry * for BFS. For example, BFS may introduce a "helper" lock_list whose * ->class is A, as a result BFS will search all dependencies starting with * A, e.g. A -> B or A -> C. * * The notation of a forwards helper lock_list is like "-> A", which means * we should search the forwards dependencies starting with "A", e.g A -> B * or A -> C. * * The notation of a bacwards helper lock_list is like "<- B", which means * we should search the backwards dependencies ending with "B", e.g. * B <- A or B <- C. */ /* * printk the shortest lock dependencies from @root to @leaf in reverse order. * * We have a lock dependency path as follow: * * @root @leaf * | | * V V * ->parent ->parent * | lock_list | <--------- | lock_list | ... | lock_list | <--------- | lock_list | * | -> L1 | | L1 -> L2 | ... |Ln-2 -> Ln-1| | Ln-1 -> Ln| * * , so it's natural that we start from @leaf and print every ->class and * ->trace until we reach the @root. */ static void __used print_shortest_lock_dependencies(struct lock_list *leaf, struct lock_list *root) { … } /* * printk the shortest lock dependencies from @leaf to @root. * * We have a lock dependency path (from a backwards search) as follow: * * @leaf @root * | | * V V * ->parent ->parent * | lock_list | ---------> | lock_list | ... | lock_list | ---------> | lock_list | * | L2 <- L1 | | L3 <- L2 | ... | Ln <- Ln-1 | | <- Ln | * * , so when we iterate from @leaf to @root, we actually print the lock * dependency path L1 -> L2 -> .. -> Ln in the non-reverse order. * * Another thing to notice here is that ->class of L2 <- L1 is L1, while the * ->trace of L2 <- L1 is the call trace of L2, in fact we don't have the call * trace of L1 in the dependency path, which is alright, because most of the * time we can figure out where L1 is held from the call trace of L2. */ static void __used print_shortest_lock_dependencies_backwards(struct lock_list *leaf, struct lock_list *root) { … } static void print_irq_lock_scenario(struct lock_list *safe_entry, struct lock_list *unsafe_entry, struct lock_class *prev_class, struct lock_class *next_class) { … } static void print_bad_irq_dependency(struct task_struct *curr, struct lock_list *prev_root, struct lock_list *next_root, struct lock_list *backwards_entry, struct lock_list *forwards_entry, struct held_lock *prev, struct held_lock *next, enum lock_usage_bit bit1, enum lock_usage_bit bit2, const char *irqclass) { … } static const char *state_names[] = …; static const char *state_rnames[] = …; static inline const char *state_name(enum lock_usage_bit bit) { … } /* * The bit number is encoded like: * * bit0: 0 exclusive, 1 read lock * bit1: 0 used in irq, 1 irq enabled * bit2-n: state */ static int exclusive_bit(int new_bit) { … } /* * Observe that when given a bitmask where each bitnr is encoded as above, a * right shift of the mask transforms the individual bitnrs as -1 and * conversely, a left shift transforms into +1 for the individual bitnrs. * * So for all bits whose number have LOCK_ENABLED_* set (bitnr1 == 1), we can * create the mask with those bit numbers using LOCK_USED_IN_* (bitnr1 == 0) * instead by subtracting the bit number by 2, or shifting the mask right by 2. * * Similarly, bitnr1 == 0 becomes bitnr1 == 1 by adding 2, or shifting left 2. * * So split the mask (note that LOCKF_ENABLED_IRQ_ALL|LOCKF_USED_IN_IRQ_ALL is * all bits set) and recompose with bitnr1 flipped. */ static unsigned long invert_dir_mask(unsigned long mask) { … } /* * Note that a LOCK_ENABLED_IRQ_*_READ usage and a LOCK_USED_IN_IRQ_*_READ * usage may cause deadlock too, for example: * * P1 P2 * <irq disabled> * write_lock(l1); <irq enabled> * read_lock(l2); * write_lock(l2); * <in irq> * read_lock(l1); * * , in above case, l1 will be marked as LOCK_USED_IN_IRQ_HARDIRQ_READ and l2 * will marked as LOCK_ENABLE_IRQ_HARDIRQ_READ, and this is a possible * deadlock. * * In fact, all of the following cases may cause deadlocks: * * LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_* * LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_* * LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_*_READ * LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_*_READ * * As a result, to calculate the "exclusive mask", first we invert the * direction (USED_IN/ENABLED) of the original mask, and 1) for all bits with * bitnr0 set (LOCK_*_READ), add those with bitnr0 cleared (LOCK_*). 2) for all * bits with bitnr0 cleared (LOCK_*_READ), add those with bitnr0 set (LOCK_*). */ static unsigned long exclusive_mask(unsigned long mask) { … } /* * Retrieve the _possible_ original mask to which @mask is * exclusive. Ie: this is the opposite of exclusive_mask(). * Note that 2 possible original bits can match an exclusive * bit: one has LOCK_USAGE_READ_MASK set, the other has it * cleared. So both are returned for each exclusive bit. */ static unsigned long original_mask(unsigned long mask) { … } /* * Find the first pair of bit match between an original * usage mask and an exclusive usage mask. */ static int find_exclusive_match(unsigned long mask, unsigned long excl_mask, enum lock_usage_bit *bitp, enum lock_usage_bit *excl_bitp) { … } /* * Prove that the new dependency does not connect a hardirq-safe(-read) * lock with a hardirq-unsafe lock - to achieve this we search * the backwards-subgraph starting at <prev>, and the * forwards-subgraph starting at <next>: */ static int check_irq_usage(struct task_struct *curr, struct held_lock *prev, struct held_lock *next) { … } #else static inline int check_irq_usage(struct task_struct *curr, struct held_lock *prev, struct held_lock *next) { return 1; } static inline bool usage_skip(struct lock_list *entry, void *mask) { return false; } #endif /* CONFIG_TRACE_IRQFLAGS */ #ifdef CONFIG_LOCKDEP_SMALL /* * Check that the dependency graph starting at <src> can lead to * <target> or not. If it can, <src> -> <target> dependency is already * in the graph. * * Return BFS_RMATCH if it does, or BFS_RNOMATCH if it does not, return BFS_E* if * any error appears in the bfs search. */ static noinline enum bfs_result check_redundant(struct held_lock *src, struct held_lock *target) { enum bfs_result ret; struct lock_list *target_entry; struct lock_list src_entry; bfs_init_root(&src_entry, src); /* * Special setup for check_redundant(). * * To report redundant, we need to find a strong dependency path that * is equal to or stronger than <src> -> <target>. So if <src> is E, * we need to let __bfs() only search for a path starting at a -(E*)->, * we achieve this by setting the initial node's ->only_xr to true in * that case. And if <prev> is S, we set initial ->only_xr to false * because both -(S*)-> (equal) and -(E*)-> (stronger) are redundant. */ src_entry.only_xr = src->read == 0; debug_atomic_inc(nr_redundant_checks); /* * Note: we skip local_lock() for redundant check, because as the * comment in usage_skip(), A -> local_lock() -> B and A -> B are not * the same. */ ret = check_path(target, &src_entry, hlock_equal, usage_skip, &target_entry); if (ret == BFS_RMATCH) debug_atomic_inc(nr_redundant); return ret; } #else static inline enum bfs_result check_redundant(struct held_lock *src, struct held_lock *target) { … } #endif static void inc_chains(int irq_context) { … } static void dec_chains(int irq_context) { … } static void print_deadlock_scenario(struct held_lock *nxt, struct held_lock *prv) { … } static void print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, struct held_lock *next) { … } /* * Check whether we are holding such a class already. * * (Note that this has to be done separately, because the graph cannot * detect such classes of deadlocks.) * * Returns: 0 on deadlock detected, 1 on OK, 2 if another lock with the same * lock class is held but nest_lock is also held, i.e. we rely on the * nest_lock to avoid the deadlock. */ static int check_deadlock(struct task_struct *curr, struct held_lock *next) { … } /* * There was a chain-cache miss, and we are about to add a new dependency * to a previous lock. We validate the following rules: * * - would the adding of the <prev> -> <next> dependency create a * circular dependency in the graph? [== circular deadlock] * * - does the new prev->next dependency connect any hardirq-safe lock * (in the full backwards-subgraph starting at <prev>) with any * hardirq-unsafe lock (in the full forwards-subgraph starting at * <next>)? [== illegal lock inversion with hardirq contexts] * * - does the new prev->next dependency connect any softirq-safe lock * (in the full backwards-subgraph starting at <prev>) with any * softirq-unsafe lock (in the full forwards-subgraph starting at * <next>)? [== illegal lock inversion with softirq contexts] * * any of these scenarios could lead to a deadlock. * * Then if all the validations pass, we add the forwards and backwards * dependency. */ static int check_prev_add(struct task_struct *curr, struct held_lock *prev, struct held_lock *next, u16 distance, struct lock_trace **const trace) { … } /* * Add the dependency to all directly-previous locks that are 'relevant'. * The ones that are relevant are (in increasing distance from curr): * all consecutive trylock entries and the final non-trylock entry - or * the end of this context's lock-chain - whichever comes first. */ static int check_prevs_add(struct task_struct *curr, struct held_lock *next) { … } struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; static DECLARE_BITMAP(lock_chains_in_use, MAX_LOCKDEP_CHAINS); static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS]; unsigned long nr_zapped_lock_chains; unsigned int nr_free_chain_hlocks; /* Free chain_hlocks in buckets */ unsigned int nr_lost_chain_hlocks; /* Lost chain_hlocks */ unsigned int nr_large_chain_blocks; /* size > MAX_CHAIN_BUCKETS */ /* * The first 2 chain_hlocks entries in the chain block in the bucket * list contains the following meta data: * * entry[0]: * Bit 15 - always set to 1 (it is not a class index) * Bits 0-14 - upper 15 bits of the next block index * entry[1] - lower 16 bits of next block index * * A next block index of all 1 bits means it is the end of the list. * * On the unsized bucket (bucket-0), the 3rd and 4th entries contain * the chain block size: * * entry[2] - upper 16 bits of the chain block size * entry[3] - lower 16 bits of the chain block size */ #define MAX_CHAIN_BUCKETS … #define CHAIN_BLK_FLAG … #define CHAIN_BLK_LIST_END … static int chain_block_buckets[MAX_CHAIN_BUCKETS]; static inline int size_to_bucket(int size) { … } /* * Iterate all the chain blocks in a bucket. */ #define for_each_chain_block(bucket, prev, curr) … /* * next block or -1 */ static inline int chain_block_next(int offset) { … } /* * bucket-0 only */ static inline int chain_block_size(int offset) { … } static inline void init_chain_block(int offset, int next, int bucket, int size) { … } static inline void add_chain_block(int offset, int size) { … } /* * Only the first block in the list can be deleted. * * For the variable size bucket[0], the first block (the largest one) is * returned, broken up and put back into the pool. So if a chain block of * length > MAX_CHAIN_BUCKETS is ever used and zapped, it will just be * queued up after the primordial chain block and never be used until the * hlock entries in the primordial chain block is almost used up. That * causes fragmentation and reduce allocation efficiency. That can be * monitored by looking at the "large chain blocks" number in lockdep_stats. */ static inline void del_chain_block(int bucket, int size, int next) { … } static void init_chain_block_buckets(void) { … } /* * Return offset of a chain block of the right size or -1 if not found. * * Fairly simple worst-fit allocator with the addition of a number of size * specific free lists. */ static int alloc_chain_hlocks(int req) { … } static inline void free_chain_hlocks(int base, int size) { … } struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i) { … } /* * Returns the index of the first held_lock of the current chain */ static inline int get_first_held_lock(struct task_struct *curr, struct held_lock *hlock) { … } #ifdef CONFIG_DEBUG_LOCKDEP /* * Returns the next chain_key iteration */ static u64 print_chain_key_iteration(u16 hlock_id, u64 chain_key) { … } static void print_chain_keys_held_locks(struct task_struct *curr, struct held_lock *hlock_next) { … } static void print_chain_keys_chain(struct lock_chain *chain) { … } static void print_collision(struct task_struct *curr, struct held_lock *hlock_next, struct lock_chain *chain) { … } #endif /* * Checks whether the chain and the current held locks are consistent * in depth and also in content. If they are not it most likely means * that there was a collision during the calculation of the chain_key. * Returns: 0 not passed, 1 passed */ static int check_no_collision(struct task_struct *curr, struct held_lock *hlock, struct lock_chain *chain) { … } /* * Given an index that is >= -1, return the index of the next lock chain. * Return -2 if there is no next lock chain. */ long lockdep_next_lockchain(long i) { … } unsigned long lock_chain_count(void) { … } /* Must be called with the graph lock held. */ static struct lock_chain *alloc_lock_chain(void) { … } /* * Adds a dependency chain into chain hashtable. And must be called with * graph_lock held. * * Return 0 if fail, and graph_lock is released. * Return 1 if succeed, with graph_lock held. */ static inline int add_chain_cache(struct task_struct *curr, struct held_lock *hlock, u64 chain_key) { … } /* * Look up a dependency chain. Must be called with either the graph lock or * the RCU read lock held. */ static inline struct lock_chain *lookup_chain_cache(u64 chain_key) { … } /* * If the key is not present yet in dependency chain cache then * add it and return 1 - in this case the new dependency chain is * validated. If the key is already hashed, return 0. * (On return with 1 graph_lock is held.) */ static inline int lookup_chain_cache_add(struct task_struct *curr, struct held_lock *hlock, u64 chain_key) { … } static int validate_chain(struct task_struct *curr, struct held_lock *hlock, int chain_head, u64 chain_key) { … } #else static inline int validate_chain(struct task_struct *curr, struct held_lock *hlock, int chain_head, u64 chain_key) { return 1; } static void init_chain_block_buckets(void) { } #endif /* CONFIG_PROVE_LOCKING */ /* * We are building curr_chain_key incrementally, so double-check * it from scratch, to make sure that it's done correctly: */ static void check_chain_key(struct task_struct *curr) { … } #ifdef CONFIG_PROVE_LOCKING static int mark_lock(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit new_bit); static void print_usage_bug_scenario(struct held_lock *lock) { … } static void print_usage_bug(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit) { … } /* * Print out an error if an invalid bit is set: */ static inline int valid_state(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit) { … } /* * print irq inversion bug: */ static void print_irq_inversion_bug(struct task_struct *curr, struct lock_list *root, struct lock_list *other, struct held_lock *this, int forwards, const char *irqclass) { … } /* * Prove that in the forwards-direction subgraph starting at <this> * there is no lock matching <mask>: */ static int check_usage_forwards(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit bit) { … } /* * Prove that in the backwards-direction subgraph starting at <this> * there is no lock matching <mask>: */ static int check_usage_backwards(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit bit) { … } void print_irqtrace_events(struct task_struct *curr) { … } static int HARDIRQ_verbose(struct lock_class *class) { … } static int SOFTIRQ_verbose(struct lock_class *class) { … } static int (*state_verbose_f[])(struct lock_class *class) = …; static inline int state_verbose(enum lock_usage_bit bit, struct lock_class *class) { … } check_usage_f; static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit new_bit) { … } /* * Mark all held locks with a usage bit: */ static int mark_held_locks(struct task_struct *curr, enum lock_usage_bit base_bit) { … } /* * Hardirqs will be enabled: */ static void __trace_hardirqs_on_caller(void) { … } /** * lockdep_hardirqs_on_prepare - Prepare for enabling interrupts * * Invoked before a possible transition to RCU idle from exit to user or * guest mode. This ensures that all RCU operations are done before RCU * stops watching. After the RCU transition lockdep_hardirqs_on() has to be * invoked to set the final state. */ void lockdep_hardirqs_on_prepare(void) { … } EXPORT_SYMBOL_GPL(…); void noinstr lockdep_hardirqs_on(unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); /* * Hardirqs were disabled: */ void noinstr lockdep_hardirqs_off(unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); /* * Softirqs will be enabled: */ void lockdep_softirqs_on(unsigned long ip) { … } /* * Softirqs were disabled: */ void lockdep_softirqs_off(unsigned long ip) { … } static int mark_usage(struct task_struct *curr, struct held_lock *hlock, int check) { … } static inline unsigned int task_irq_context(struct task_struct *task) { … } static int separate_irq_context(struct task_struct *curr, struct held_lock *hlock) { … } /* * Mark a lock with a usage bit, and validate the state transition: */ static int mark_lock(struct task_struct *curr, struct held_lock *this, enum lock_usage_bit new_bit) { … } static inline short task_wait_context(struct task_struct *curr) { … } static int print_lock_invalid_wait_context(struct task_struct *curr, struct held_lock *hlock) { … } /* * Verify the wait_type context. * * This check validates we take locks in the right wait-type order; that is it * ensures that we do not take mutexes inside spinlocks and do not attempt to * acquire spinlocks inside raw_spinlocks and the sort. * * The entire thing is slightly more complex because of RCU, RCU is a lock that * can be taken from (pretty much) any context but also has constraints. * However when taken in a stricter environment the RCU lock does not loosen * the constraints. * * Therefore we must look for the strictest environment in the lock stack and * compare that to the lock we're trying to acquire. */ static int check_wait_context(struct task_struct *curr, struct held_lock *next) { … } #else /* CONFIG_PROVE_LOCKING */ static inline int mark_usage(struct task_struct *curr, struct held_lock *hlock, int check) { return 1; } static inline unsigned int task_irq_context(struct task_struct *task) { return 0; } static inline int separate_irq_context(struct task_struct *curr, struct held_lock *hlock) { return 0; } static inline int check_wait_context(struct task_struct *curr, struct held_lock *next) { return 0; } #endif /* CONFIG_PROVE_LOCKING */ /* * Initialize a lock instance's lock-class mapping info: */ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, struct lock_class_key *key, int subclass, u8 inner, u8 outer, u8 lock_type) { … } EXPORT_SYMBOL_GPL(…); struct lock_class_key __lockdep_no_validate__; EXPORT_SYMBOL_GPL(…); struct lock_class_key __lockdep_no_track__; EXPORT_SYMBOL_GPL(…); #ifdef CONFIG_PROVE_LOCKING void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { … } EXPORT_SYMBOL_GPL(…); #endif static void print_lock_nested_lock_not_held(struct task_struct *curr, struct held_lock *hlock) { … } static int __lock_is_held(const struct lockdep_map *lock, int read); /* * This gets called for every mutex_lock*()/spin_lock*() operation. * We maintain the dependency maps and validate the locking attempt: * * The callers must make sure that IRQs are disabled before calling it, * otherwise we could get an interrupt which would want to take locks, * which would end up in lockdep again. */ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, int hardirqs_off, struct lockdep_map *nest_lock, unsigned long ip, int references, int pin_count, int sync) { … } static void print_unlock_imbalance_bug(struct task_struct *curr, struct lockdep_map *lock, unsigned long ip) { … } static noinstr int match_held_lock(const struct held_lock *hlock, const struct lockdep_map *lock) { … } /* @depth must not be zero */ static struct held_lock *find_held_lock(struct task_struct *curr, struct lockdep_map *lock, unsigned int depth, int *idx) { … } static int reacquire_held_locks(struct task_struct *curr, unsigned int depth, int idx, unsigned int *merged) { … } static int __lock_set_class(struct lockdep_map *lock, const char *name, struct lock_class_key *key, unsigned int subclass, unsigned long ip) { … } static int __lock_downgrade(struct lockdep_map *lock, unsigned long ip) { … } /* * Remove the lock from the list of currently held locks - this gets * called on mutex_unlock()/spin_unlock*() (or on a failed * mutex_lock_interruptible()). */ static int __lock_release(struct lockdep_map *lock, unsigned long ip) { … } static __always_inline int __lock_is_held(const struct lockdep_map *lock, int read) { … } static struct pin_cookie __lock_pin_lock(struct lockdep_map *lock) { … } static void __lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie) { … } static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) { … } /* * Check whether we follow the irq-flags state precisely: */ static noinstr void check_flags(unsigned long flags) { … } void lock_set_class(struct lockdep_map *lock, const char *name, struct lock_class_key *key, unsigned int subclass, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); void lock_downgrade(struct lockdep_map *lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); /* NMI context !!! */ static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass) { … } static bool lockdep_nmi(void) { … } /* * read_lock() is recursive if: * 1. We force lockdep think this way in selftests or * 2. The implementation is not queued read/write lock or * 3. The locker is at an in_interrupt() context. */ bool read_lock_is_recursive(void) { … } EXPORT_SYMBOL_GPL(…); /* * We are not always called with irqs disabled - do that here, * and also avoid lockdep recursion: */ void lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, struct lockdep_map *nest_lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); void lock_release(struct lockdep_map *lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); /* * lock_sync() - A special annotation for synchronize_{s,}rcu()-like API. * * No actual critical section is created by the APIs annotated with this: these * APIs are used to wait for one or multiple critical sections (on other CPUs * or threads), and it means that calling these APIs inside these critical * sections is potential deadlock. */ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read, int check, struct lockdep_map *nest_lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); noinstr int lock_is_held_type(const struct lockdep_map *lock, int read) { … } EXPORT_SYMBOL_GPL(…); NOKPROBE_SYMBOL(lock_is_held_type); struct pin_cookie lock_pin_lock(struct lockdep_map *lock) { … } EXPORT_SYMBOL_GPL(…); void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie) { … } EXPORT_SYMBOL_GPL(…); void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) { … } EXPORT_SYMBOL_GPL(…); #ifdef CONFIG_LOCK_STAT static void print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, unsigned long ip) { … } static void __lock_contended(struct lockdep_map *lock, unsigned long ip) { … } static void __lock_acquired(struct lockdep_map *lock, unsigned long ip) { … } void lock_contended(struct lockdep_map *lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); void lock_acquired(struct lockdep_map *lock, unsigned long ip) { … } EXPORT_SYMBOL_GPL(…); #endif /* * Used by the testsuite, sanitize the validator state * after a simulated failure: */ void lockdep_reset(void) { … } /* Remove a class from a lock chain. Must be called with the graph lock held. */ static void remove_class_from_lock_chain(struct pending_free *pf, struct lock_chain *chain, struct lock_class *class) { … } /* Must be called with the graph lock held. */ static void remove_class_from_lock_chains(struct pending_free *pf, struct lock_class *class) { … } /* * Remove all references to a lock class. The caller must hold the graph lock. */ static void zap_class(struct pending_free *pf, struct lock_class *class) { … } static void reinit_class(struct lock_class *class) { … } static inline int within(const void *addr, void *start, unsigned long size) { … } static bool inside_selftest(void) { … } /* The caller must hold the graph lock. */ static struct pending_free *get_pending_free(void) { … } static void free_zapped_rcu(struct rcu_head *cb); /* * Schedule an RCU callback if no RCU callback is pending. Must be called with * the graph lock held. */ static void call_rcu_zapped(struct pending_free *pf) { … } /* The caller must hold the graph lock. May be called from RCU context. */ static void __free_zapped_classes(struct pending_free *pf) { … } static void free_zapped_rcu(struct rcu_head *ch) { … } /* * Remove all lock classes from the class hash table and from the * all_lock_classes list whose key or name is in the address range [start, * start + size). Move these lock classes to the zapped_classes list. Must * be called with the graph lock held. */ static void __lockdep_free_key_range(struct pending_free *pf, void *start, unsigned long size) { … } /* * Used in module.c to remove lock classes from memory that is going to be * freed; and possibly re-used by other modules. * * We will have had one synchronize_rcu() before getting here, so we're * guaranteed nobody will look up these exact classes -- they're properly dead * but still allocated. */ static void lockdep_free_key_range_reg(void *start, unsigned long size) { … } /* * Free all lockdep keys in the range [start, start+size). Does not sleep. * Ignores debug_locks. Must only be used by the lockdep selftests. */ static void lockdep_free_key_range_imm(void *start, unsigned long size) { … } void lockdep_free_key_range(void *start, unsigned long size) { … } /* * Check whether any element of the @lock->class_cache[] array refers to a * registered lock class. The caller must hold either the graph lock or the * RCU read lock. */ static bool lock_class_cache_is_registered(struct lockdep_map *lock) { … } /* The caller must hold the graph lock. Does not sleep. */ static void __lockdep_reset_lock(struct pending_free *pf, struct lockdep_map *lock) { … } /* * Remove all information lockdep has about a lock if debug_locks == 1. Free * released data structures from RCU context. */ static void lockdep_reset_lock_reg(struct lockdep_map *lock) { … } /* * Reset a lock. Does not sleep. Ignores debug_locks. Must only be used by the * lockdep selftests. */ static void lockdep_reset_lock_imm(struct lockdep_map *lock) { … } void lockdep_reset_lock(struct lockdep_map *lock) { … } /* * Unregister a dynamically allocated key. * * Unlike lockdep_register_key(), a search is always done to find a matching * key irrespective of debug_locks to avoid potential invalid access to freed * memory in lock_class entry. */ void lockdep_unregister_key(struct lock_class_key *key) { … } EXPORT_SYMBOL_GPL(…); void __init lockdep_init(void) { … } static void print_freed_lock_bug(struct task_struct *curr, const void *mem_from, const void *mem_to, struct held_lock *hlock) { … } static inline int not_in_range(const void* mem_from, unsigned long mem_len, const void* lock_from, unsigned long lock_len) { … } /* * Called when kernel memory is freed (or unmapped), or if a lock * is destroyed or reinitialized - this code checks whether there is * any held lock in the memory range of <from> to <to>: */ void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len) { … } EXPORT_SYMBOL_GPL(…); static void print_held_locks_bug(void) { … } void debug_check_no_locks_held(void) { … } EXPORT_SYMBOL_GPL(…); #ifdef __KERNEL__ void debug_show_all_locks(void) { … } EXPORT_SYMBOL_GPL(…); #endif /* * Careful: only use this function if you are sure that * the task cannot run in parallel! */ void debug_show_held_locks(struct task_struct *task) { … } EXPORT_SYMBOL_GPL(…); asmlinkage __visible void lockdep_sys_exit(void) { … } void lockdep_rcu_suspicious(const char *file, const int line, const char *s) { … } EXPORT_SYMBOL_GPL(…);
Codebase Browser
linux