// SPDX-License-Identifier: GPL-2.0+ /* * Read-Copy Update module-based torture test facility * * Copyright (C) IBM Corporation, 2005, 2006 * * Authors: Paul E. McKenney <[email protected]> * Josh Triplett <[email protected]> * * See also: Documentation/RCU/torture.rst */ #define pr_fmt(fmt) … #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/kthread.h> #include <linux/err.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate_wait.h> #include <linux/rcu_notifier.h> #include <linux/interrupt.h> #include <linux/sched/signal.h> #include <uapi/linux/sched/types.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/reboot.h> #include <linux/freezer.h> #include <linux/cpu.h> #include <linux/delay.h> #include <linux/stat.h> #include <linux/srcu.h> #include <linux/slab.h> #include <linux/trace_clock.h> #include <asm/byteorder.h> #include <linux/torture.h> #include <linux/vmalloc.h> #include <linux/sched/debug.h> #include <linux/sched/sysctl.h> #include <linux/oom.h> #include <linux/tick.h> #include <linux/rcupdate_trace.h> #include <linux/nmi.h> #include "rcu.h" MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_AUTHOR(…) …; /* Bits for ->extendables field, extendables param, and related definitions. */ #define RCUTORTURE_RDR_SHIFT_1 … #define RCUTORTURE_RDR_MASK_1 … #define RCUTORTURE_RDR_SHIFT_2 … #define RCUTORTURE_RDR_MASK_2 … #define RCUTORTURE_RDR_BH … #define RCUTORTURE_RDR_IRQ … #define RCUTORTURE_RDR_PREEMPT … #define RCUTORTURE_RDR_RBH … #define RCUTORTURE_RDR_SCHED … #define RCUTORTURE_RDR_RCU_1 … #define RCUTORTURE_RDR_RCU_2 … #define RCUTORTURE_RDR_NBITS … #define RCUTORTURE_MAX_EXTEND … #define RCUTORTURE_RDR_MAX_LOOPS … /* Must be power of two minus one. */ #define RCUTORTURE_RDR_MAX_SEGS … torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); torture_param(bool, gp_cond_exp_full, false, "Use conditional/async full-stateexpedited GP wait primitives"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); torture_param(int, stutter, 5, "Number of seconds to run/halt test"); torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); static char *torture_type = …; module_param(torture_type, charp, 0444); MODULE_PARM_DESC(…) …; static int nrealnocbers; static int nrealreaders; static struct task_struct *writer_task; static struct task_struct **fakewriter_tasks; static struct task_struct **reader_tasks; static struct task_struct **nocb_tasks; static struct task_struct *stats_task; static struct task_struct *fqs_task; static struct task_struct *boost_tasks[NR_CPUS]; static struct task_struct *stall_task; static struct task_struct **fwd_prog_tasks; static struct task_struct **barrier_cbs_tasks; static struct task_struct *barrier_task; static struct task_struct *read_exit_task; #define RCU_TORTURE_PIPE_LEN … // Mailbox-like structure to check RCU global memory ordering. struct rcu_torture_reader_check { … } ____cacheline_internodealigned_in_smp; // Update-side data structure used to check RCU readers. struct rcu_torture { … }; static LIST_HEAD(rcu_torture_freelist); static struct rcu_torture __rcu *rcu_torture_current; static unsigned long rcu_torture_current_version; static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; static DEFINE_SPINLOCK(rcu_torture_lock); static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; static atomic_t n_rcu_torture_alloc; static atomic_t n_rcu_torture_alloc_fail; static atomic_t n_rcu_torture_free; static atomic_t n_rcu_torture_mberror; static atomic_t n_rcu_torture_mbchk_fail; static atomic_t n_rcu_torture_mbchk_tries; static atomic_t n_rcu_torture_error; static long n_rcu_torture_barrier_error; static long n_rcu_torture_boost_ktrerror; static long n_rcu_torture_boost_failure; static long n_rcu_torture_boosts; static atomic_long_t n_rcu_torture_timers; static long n_barrier_attempts; static long n_barrier_successes; /* did rcu_barrier test succeed? */ static unsigned long n_read_exits; static struct list_head rcu_torture_removed; static unsigned long shutdown_jiffies; static unsigned long start_gp_seq; static atomic_long_t n_nocb_offload; static atomic_long_t n_nocb_deoffload; static int rcu_torture_writer_state; #define RTWS_FIXED_DELAY … #define RTWS_DELAY … #define RTWS_REPLACE … #define RTWS_DEF_FREE … #define RTWS_EXP_SYNC … #define RTWS_COND_GET … #define RTWS_COND_GET_FULL … #define RTWS_COND_GET_EXP … #define RTWS_COND_GET_EXP_FULL … #define RTWS_COND_SYNC … #define RTWS_COND_SYNC_FULL … #define RTWS_COND_SYNC_EXP … #define RTWS_COND_SYNC_EXP_FULL … #define RTWS_POLL_GET … #define RTWS_POLL_GET_FULL … #define RTWS_POLL_GET_EXP … #define RTWS_POLL_GET_EXP_FULL … #define RTWS_POLL_WAIT … #define RTWS_POLL_WAIT_FULL … #define RTWS_POLL_WAIT_EXP … #define RTWS_POLL_WAIT_EXP_FULL … #define RTWS_SYNC … #define RTWS_STUTTER … #define RTWS_STOPPING … static const char * const rcu_torture_writer_state_names[] = …; /* Record reader segment types and duration for first failing read. */ struct rt_read_seg { … }; static int err_segs_recorded; static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; static int rt_read_nsegs; static const char *rcu_torture_writer_state_getname(void) { … } #ifdef CONFIG_RCU_TRACE static u64 notrace rcu_trace_clock_local(void) { … } #else /* #ifdef CONFIG_RCU_TRACE */ static u64 notrace rcu_trace_clock_local(void) { return 0ULL; } #endif /* #else #ifdef CONFIG_RCU_TRACE */ /* * Stop aggressive CPU-hog tests a bit before the end of the test in order * to avoid interfering with test shutdown. */ static bool shutdown_time_arrived(void) { … } static unsigned long boost_starttime; /* jiffies of next boost test start. */ static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ /* and boost task create/destroy. */ static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ static bool barrier_phase; /* Test phase. */ static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ /* * Allocate an element from the rcu_tortures pool. */ static struct rcu_torture * rcu_torture_alloc(void) { … } /* * Free an element to the rcu_tortures pool. */ static void rcu_torture_free(struct rcu_torture *p) { … } /* * Operations vector for selecting different types of tests. */ struct rcu_torture_ops { … }; static struct rcu_torture_ops *cur_ops; /* * Definitions for rcu torture testing. */ static int torture_readlock_not_held(void) { … } static int rcu_torture_read_lock(void) { … } static void rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) { … } static void rcu_torture_read_unlock(int idx) { … } /* * Update callback in the pipe. This should be invoked after a grace period. */ static bool rcu_torture_pipe_update_one(struct rcu_torture *rp) { … } /* * Update all callbacks in the pipe. Suitable for synchronous grace-period * primitives. */ static void rcu_torture_pipe_update(struct rcu_torture *old_rp) { … } static void rcu_torture_cb(struct rcu_head *p) { … } static unsigned long rcu_no_completed(void) { … } static void rcu_torture_deferred_free(struct rcu_torture *p) { … } static void rcu_sync_torture_init(void) { … } static bool rcu_poll_need_2gp(bool poll, bool poll_full) { … } static struct rcu_torture_ops rcu_ops = …; /* * Don't even think about trying any of these in real life!!! * The names includes "busted", and they really means it! * The only purpose of these functions is to provide a buggy RCU * implementation to make sure that rcutorture correctly emits * buggy-RCU error messages. */ static void rcu_busted_torture_deferred_free(struct rcu_torture *p) { … } static void synchronize_rcu_busted(void) { … } static void call_rcu_busted(struct rcu_head *head, rcu_callback_t func) { … } static struct rcu_torture_ops rcu_busted_ops = …; /* * Definitions for srcu torture testing. */ DEFINE_STATIC_SRCU(…); static struct srcu_struct srcu_ctld; static struct srcu_struct *srcu_ctlp = …; static struct rcu_torture_ops srcud_ops; static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) { … } static int srcu_torture_read_lock(void) { … } static void srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) { … } static void srcu_torture_read_unlock(int idx) { … } static int torture_srcu_read_lock_held(void) { … } static unsigned long srcu_torture_completed(void) { … } static void srcu_torture_deferred_free(struct rcu_torture *rp) { … } static void srcu_torture_synchronize(void) { … } static unsigned long srcu_torture_get_gp_state(void) { … } static unsigned long srcu_torture_start_gp_poll(void) { … } static bool srcu_torture_poll_gp_state(unsigned long oldstate) { … } static void srcu_torture_call(struct rcu_head *head, rcu_callback_t func) { … } static void srcu_torture_barrier(void) { … } static void srcu_torture_stats(void) { … } static void srcu_torture_synchronize_expedited(void) { … } static struct rcu_torture_ops srcu_ops = …; static void srcu_torture_init(void) { … } static void srcu_torture_cleanup(void) { … } /* As above, but dynamically allocated. */ static struct rcu_torture_ops srcud_ops = …; /* As above, but broken due to inappropriate reader extension. */ static struct rcu_torture_ops busted_srcud_ops = …; /* * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. * This implementation does not necessarily work well with CPU hotplug. */ static void synchronize_rcu_trivial(void) { … } static int rcu_torture_read_lock_trivial(void) { … } static void rcu_torture_read_unlock_trivial(int idx) { … } static struct rcu_torture_ops trivial_ops = …; #ifdef CONFIG_TASKS_RCU /* * Definitions for RCU-tasks torture testing. */ static int tasks_torture_read_lock(void) { … } static void tasks_torture_read_unlock(int idx) { … } static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) { … } static void synchronize_rcu_mult_test(void) { … } static struct rcu_torture_ops tasks_ops = …; #define TASKS_OPS … #else // #ifdef CONFIG_TASKS_RCU #define TASKS_OPS #endif // #else #ifdef CONFIG_TASKS_RCU #ifdef CONFIG_TASKS_RUDE_RCU /* * Definitions for rude RCU-tasks torture testing. */ static void rcu_tasks_rude_torture_deferred_free(struct rcu_torture *p) { … } static struct rcu_torture_ops tasks_rude_ops = …; #define TASKS_RUDE_OPS … #else // #ifdef CONFIG_TASKS_RUDE_RCU #define TASKS_RUDE_OPS #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU #ifdef CONFIG_TASKS_TRACE_RCU /* * Definitions for tracing RCU-tasks torture testing. */ static int tasks_tracing_torture_read_lock(void) { … } static void tasks_tracing_torture_read_unlock(int idx) { … } static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) { … } static struct rcu_torture_ops tasks_tracing_ops = …; #define TASKS_TRACING_OPS … #else // #ifdef CONFIG_TASKS_TRACE_RCU #define TASKS_TRACING_OPS #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) { … } /* * RCU torture priority-boost testing. Runs one real-time thread per * CPU for moderate bursts, repeatedly starting grace periods and waiting * for them to complete. If a given grace period takes too long, we assume * that priority inversion has occurred. */ static int old_rt_runtime = …; static void rcu_torture_disable_rt_throttle(void) { … } static void rcu_torture_enable_rt_throttle(void) { … } static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) { … } static int rcu_torture_boost(void *arg) { … } /* * RCU torture force-quiescent-state kthread. Repeatedly induces * bursts of calls to force_quiescent_state(), increasing the probability * of occurrence of some important types of race conditions. */ static int rcu_torture_fqs(void *arg) { … } // Used by writers to randomly choose from the available grace-period primitives. static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = …; static int nsynctypes; /* * Determine which grace-period primitives are available. */ static void rcu_torture_write_types(void) { … } /* * Do the specified rcu_torture_writer() synchronous grace period, * while also testing out the polled APIs. Note well that the single-CPU * grace-period optimizations must be accounted for. */ static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) { … } /* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure * after a series of grace periods (the "pipeline"). */ static int rcu_torture_writer(void *arg) { … } /* * RCU torture fake writer kthread. Repeatedly calls sync, with a random * delay between calls. */ static int rcu_torture_fakewriter(void *arg) { … } static void rcu_torture_timer_cb(struct rcu_head *rhp) { … } // Set up and carry out testing of RCU's global memory ordering static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, struct torture_random_state *trsp) { … } /* * Do one extension of an RCU read-side critical section using the * current reader state in readstate (set to zero for initial entry * to extended critical section), set the new state as specified by * newstate (set to zero for final exit from extended critical section), * and random-number-generator state in trsp. If this is neither the * beginning or end of the critical section and if there was actually a * change, do a ->read_delay(). */ static void rcutorture_one_extend(int *readstate, int newstate, struct torture_random_state *trsp, struct rt_read_seg *rtrsp) { … } /* Return the biggest extendables mask given current RCU and boot parameters. */ static int rcutorture_extend_mask_max(void) { … } /* Return a random protection state mask, but with at least one bit set. */ static int rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) { … } /* * Do a randomly selected number of extensions of an existing RCU read-side * critical section. */ static struct rt_read_seg * rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct rt_read_seg *rtrsp) { … } /* * Do one read-side critical section, returning false if there was * no data to read. Can be invoked both from process context and * from a timer handler. */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { … } static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); /* * RCU torture reader from timer handler. Dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The * counter in the element should never be greater than 1, otherwise, the * RCU implementation is broken. */ static void rcu_torture_timer(struct timer_list *unused) { … } /* * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The * counter in the element should never be greater than 1, otherwise, the * RCU implementation is broken. */ static int rcu_torture_reader(void *arg) { … } /* * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to * increase race probabilities and fuzzes the interval between toggling. */ static int rcu_nocb_toggle(void *arg) { … } /* * Print torture statistics. Caller must ensure that there is only * one call to this function at a given time!!! This is normally * accomplished by relying on the module system to only have one copy * of the module loaded, and then by giving the rcu_torture_stats * kthread full control (or the init/cleanup functions when rcu_torture_stats * thread is not running). */ static void rcu_torture_stats_print(void) { … } /* * Periodically prints torture statistics, if periodic statistics printing * was specified via the stat_interval module parameter. */ static int rcu_torture_stats(void *arg) { … } /* Test mem_dump_obj() and friends. */ static void rcu_torture_mem_dump_obj(void) { … } static void rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) { … } static int rcutorture_booster_cleanup(unsigned int cpu) { … } static int rcutorture_booster_init(unsigned int cpu) { … } static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) { … } static struct notifier_block rcu_torture_stall_block = …; /* * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then * induces a CPU stall for the time specified by stall_cpu. If a new * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. */ static int rcu_torture_stall(void *args) { … } /* Spawn CPU-stall kthread, if stall_cpu specified. */ static int __init rcu_torture_stall_init(void) { … } /* State structure for forward-progress self-propagating RCU callback. */ struct fwd_cb_state { … }; /* * Forward-progress self-propagating RCU callback function. Because * callbacks run from softirq, this function is an implicit RCU read-side * critical section. */ static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) { … } /* State for continuous-flood RCU callbacks. */ struct rcu_fwd_cb { … }; #define MAX_FWD_CB_JIFFIES … #define MIN_FWD_CB_LAUNDERS … #define MIN_FWD_CBS_LAUNDERED … #define FWD_CBS_HIST_DIV … #define N_LAUNDERS_HIST … struct rcu_launder_hist { … }; struct rcu_fwd { … }; static DEFINE_MUTEX(rcu_fwd_mutex); static struct rcu_fwd *rcu_fwds; static unsigned long rcu_fwd_seq; static atomic_long_t rcu_fwd_max_cbs; static bool rcu_fwd_emergency_stop; static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) { … } /* Callback function for continuous-flood RCU callbacks. */ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) { … } // Give the scheduler a chance, even on nohz_full CPUs. static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) { … } /* * Free all callbacks on the rcu_fwd_cb_head list, either because the * test is over or because we hit an OOM event. */ static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) { … } /* Carry out need_resched()/cond_resched() forward-progress testing. */ static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, int *tested, int *tested_tries) { … } /* Carry out call_rcu() forward-progress testing. */ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) { … } /* * OOM notifier, but this only prints diagnostic information for the * current forward-progress test. */ static int rcutorture_oom_notify(struct notifier_block *self, unsigned long notused, void *nfreed) { … } static struct notifier_block rcutorture_oom_nb = …; /* Carry out grace-period forward-progress testing. */ static int rcu_torture_fwd_prog(void *args) { … } /* If forward-progress checking is requested and feasible, spawn the thread. */ static int __init rcu_torture_fwd_prog_init(void) { … } static void rcu_torture_fwd_prog_cleanup(void) { … } /* Callback function for RCU barrier testing. */ static void rcu_torture_barrier_cbf(struct rcu_head *rcu) { … } /* IPI handler to get callback posted on desired CPU, if online. */ static int rcu_torture_barrier1cb(void *rcu_void) { … } /* kthread function to register callbacks used to test RCU barriers. */ static int rcu_torture_barrier_cbs(void *arg) { … } /* kthread function to drive and coordinate RCU barrier testing. */ static int rcu_torture_barrier(void *arg) { … } /* Initialize RCU barrier testing. */ static int rcu_torture_barrier_init(void) { … } /* Clean up after RCU barrier testing. */ static void rcu_torture_barrier_cleanup(void) { … } static bool rcu_torture_can_boost(void) { … } static bool read_exit_child_stop; static bool read_exit_child_stopped; static wait_queue_head_t read_exit_wq; // Child kthread which just does an rcutorture reader and exits. static int rcu_torture_read_exit_child(void *trsp_in) { … } // Parent kthread which creates and destroys read-exit child kthreads. static int rcu_torture_read_exit(void *unused) { … } static int rcu_torture_read_exit_init(void) { … } static void rcu_torture_read_exit_cleanup(void) { … } static void rcutorture_test_nmis(int n) { … } static enum cpuhp_state rcutor_hp; static void rcu_torture_cleanup(void) { … } static void rcu_torture_leak_cb(struct rcu_head *rhp) { … } static void rcu_torture_err_cb(struct rcu_head *rhp) { … } /* * Verify that double-free causes debug-objects to complain, but only * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test * cannot be carried out. */ static void rcu_test_debug_objects(void) { … } static void rcutorture_sync(void) { … } static DEFINE_MUTEX(mut0); static DEFINE_MUTEX(mut1); static DEFINE_MUTEX(mut2); static DEFINE_MUTEX(mut3); static DEFINE_MUTEX(mut4); static DEFINE_MUTEX(mut5); static DEFINE_MUTEX(mut6); static DEFINE_MUTEX(mut7); static DEFINE_MUTEX(mut8); static DEFINE_MUTEX(mut9); static DECLARE_RWSEM(rwsem0); static DECLARE_RWSEM(rwsem1); static DECLARE_RWSEM(rwsem2); static DECLARE_RWSEM(rwsem3); static DECLARE_RWSEM(rwsem4); static DECLARE_RWSEM(rwsem5); static DECLARE_RWSEM(rwsem6); static DECLARE_RWSEM(rwsem7); static DECLARE_RWSEM(rwsem8); static DECLARE_RWSEM(rwsem9); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); DEFINE_STATIC_SRCU(…); static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, int cyclelen, int deadlock) { … } // Test lockdep on SRCU-based deadlock scenarios. static void rcu_torture_init_srcu_lockdep(void) { … } static int __init rcu_torture_init(void) { … } module_init(…) …; module_exit(rcu_torture_cleanup);
Codebase Browser
linux