// SPDX-License-Identifier: GPL-2.0+ /* * Read-Copy Update module-based scalability-test facility * * Copyright (C) IBM Corporation, 2015 * * Authors: Paul E. McKenney <[email protected]> */ #define pr_fmt(fmt) … #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/kthread.h> #include <linux/err.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <uapi/linux/sched/types.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/reboot.h> #include <linux/freezer.h> #include <linux/cpu.h> #include <linux/delay.h> #include <linux/stat.h> #include <linux/srcu.h> #include <linux/slab.h> #include <asm/byteorder.h> #include <linux/torture.h> #include <linux/vmalloc.h> #include <linux/rcupdate_trace.h> #include "rcu.h" MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_AUTHOR(…) …; #define SCALE_FLAG … #define SCALEOUT_STRING(s) … #define VERBOSE_SCALEOUT_STRING(s) … #define SCALEOUT_ERRSTRING(s) … /* * The intended use cases for the nreaders and nwriters module parameters * are as follows: * * 1. Specify only the nr_cpus kernel boot parameter. This will * set both nreaders and nwriters to the value specified by * nr_cpus for a mixed reader/writer test. * * 2. Specify the nr_cpus kernel boot parameter, but set * rcuscale.nreaders to zero. This will set nwriters to the * value specified by nr_cpus for an update-only test. * * 3. Specify the nr_cpus kernel boot parameter, but set * rcuscale.nwriters to zero. This will set nreaders to the * value specified by nr_cpus for a read-only test. * * Various other use cases may of course be specified. * * Note that this test's readers are intended only as a test load for * the writers. The reader scalability statistics will be overly * pessimistic due to the per-critical-section interrupt disabling, * test-end checks, and the pair of calls through pointers. */ #ifdef MODULE #define RCUSCALE_SHUTDOWN … #else #define RCUSCALE_SHUTDOWN … #endif torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); torture_param(int, minruntime, 0, "Minimum run time (s)"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(int, nwriters, -1, "Number of RCU updater threads"); torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, "Shutdown at end of scalability tests."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); static char *scale_type = …; module_param(scale_type, charp, 0444); MODULE_PARM_DESC(…) …; static int nrealreaders; static int nrealwriters; static struct task_struct **writer_tasks; static struct task_struct **reader_tasks; static struct task_struct *shutdown_task; static u64 **writer_durations; static int *writer_n_durations; static atomic_t n_rcu_scale_reader_started; static atomic_t n_rcu_scale_writer_started; static atomic_t n_rcu_scale_writer_finished; static wait_queue_head_t shutdown_wq; static u64 t_rcu_scale_writer_started; static u64 t_rcu_scale_writer_finished; static unsigned long b_rcu_gp_test_started; static unsigned long b_rcu_gp_test_finished; static DEFINE_PER_CPU(atomic_t, n_async_inflight); #define MAX_MEAS … #define MIN_MEAS … /* * Operations vector for selecting different types of tests. */ struct rcu_scale_ops { … }; static struct rcu_scale_ops *cur_ops; /* * Definitions for rcu scalability testing. */ static int rcu_scale_read_lock(void) __acquires(RCU) { … } static void rcu_scale_read_unlock(int idx) __releases(RCU) { … } static unsigned long __maybe_unused rcu_no_completed(void) { … } static void rcu_sync_scale_init(void) { … } static struct rcu_scale_ops rcu_ops = …; /* * Definitions for srcu scalability testing. */ DEFINE_STATIC_SRCU(…); static struct srcu_struct *srcu_ctlp = …; static int srcu_scale_read_lock(void) __acquires(srcu_ctlp) { … } static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp) { … } static unsigned long srcu_scale_completed(void) { … } static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func) { … } static void srcu_rcu_barrier(void) { … } static void srcu_scale_synchronize(void) { … } static void srcu_scale_synchronize_expedited(void) { … } static struct rcu_scale_ops srcu_ops = …; static struct srcu_struct srcud; static void srcu_sync_scale_init(void) { … } static void srcu_sync_scale_cleanup(void) { … } static struct rcu_scale_ops srcud_ops = …; #ifdef CONFIG_TASKS_RCU /* * Definitions for RCU-tasks scalability testing. */ static int tasks_scale_read_lock(void) { … } static void tasks_scale_read_unlock(int idx) { … } static struct rcu_scale_ops tasks_ops = …; #define TASKS_OPS … #else // #ifdef CONFIG_TASKS_RCU #define TASKS_OPS #endif // #else // #ifdef CONFIG_TASKS_RCU #ifdef CONFIG_TASKS_RUDE_RCU /* * Definitions for RCU-tasks-rude scalability testing. */ static int tasks_rude_scale_read_lock(void) { … } static void tasks_rude_scale_read_unlock(int idx) { … } static struct rcu_scale_ops tasks_rude_ops = …; #define TASKS_RUDE_OPS … #else // #ifdef CONFIG_TASKS_RUDE_RCU #define TASKS_RUDE_OPS #endif // #else // #ifdef CONFIG_TASKS_RUDE_RCU #ifdef CONFIG_TASKS_TRACE_RCU /* * Definitions for RCU-tasks-trace scalability testing. */ static int tasks_trace_scale_read_lock(void) { … } static void tasks_trace_scale_read_unlock(int idx) { … } static struct rcu_scale_ops tasks_tracing_ops = …; #define TASKS_TRACING_OPS … #else // #ifdef CONFIG_TASKS_TRACE_RCU #define TASKS_TRACING_OPS #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old) { … } /* * If scalability tests complete, wait for shutdown to commence. */ static void rcu_scale_wait_shutdown(void) { … } /* * RCU scalability reader kthread. Repeatedly does empty RCU read-side * critical section, minimizing update-side interference. However, the * point of this test is not to evaluate reader scalability, but instead * to serve as a test load for update-side scalability testing. */ static int rcu_scale_reader(void *arg) { … } /* * Callback function for asynchronous grace periods from rcu_scale_writer(). */ static void rcu_scale_async_cb(struct rcu_head *rhp) { … } /* * RCU scale writer kthread. Repeatedly does a grace period. */ static int rcu_scale_writer(void *arg) { … } static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { … } /* * Return the number if non-negative. If -1, the number of CPUs. * If less than -1, that much less than the number of CPUs, but * at least one. */ static int compute_real(int n) { … } /* * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number * of iterations and measure total time and number of GP for all iterations to complete. */ torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu()."); torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration."); torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees."); torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?"); torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?"); static struct task_struct **kfree_reader_tasks; static int kfree_nrealthreads; static atomic_t n_kfree_scale_thread_started; static atomic_t n_kfree_scale_thread_ended; static struct task_struct *kthread_tp; static u64 kthread_stime; struct kfree_obj { … }; /* Used if doing RCU-kfree'ing via call_rcu(). */ static void kfree_call_rcu(struct rcu_head *rh) { … } static int kfree_scale_thread(void *arg) { … } static void kfree_scale_cleanup(void) { … } /* * shutdown kthread. Just waits to be awakened, then shuts down system. */ static int kfree_scale_shutdown(void *arg) { … } // Used if doing RCU-kfree'ing via call_rcu(). static unsigned long jiffies_at_lazy_cb; static struct rcu_head lazy_test1_rh; static int rcu_lazy_test1_cb_called; static void call_rcu_lazy_test1(struct rcu_head *rh) { … } static int __init kfree_scale_init(void) { … } static void rcu_scale_cleanup(void) { … } /* * RCU scalability shutdown kthread. Just waits to be awakened, then shuts * down system. */ static int rcu_scale_shutdown(void *arg) { … } static int __init rcu_scale_init(void) { … } module_init(…) …; module_exit(rcu_scale_cleanup);
Codebase Browser
linux