/* SPDX-License-Identifier: GPL-2.0+ */ /* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * Internal non-public definitions that provide either classic * or preemptible semantics. * * Copyright Red Hat, 2009 * Copyright IBM Corporation, 2009 * Copyright SUSE, 2021 * * Author: Ingo Molnar <[email protected]> * Paul E. McKenney <[email protected]> * Frederic Weisbecker <[email protected]> */ #ifdef CONFIG_RCU_NOCB_CPU static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */ static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) { … } static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) { … } /* * Offload callback processing from the boot-time-specified set of CPUs * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads * created that pull the callbacks from the corresponding CPU, wait for * a grace period to elapse, and invoke the callbacks. These kthreads * are organized into GP kthreads, which manage incoming callbacks, wait for * grace periods, and awaken CB kthreads, and the CB kthreads, which only * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs * do a wake_up() on their GP kthread when they insert a callback into any * empty list, unless the rcu_nocb_poll boot parameter has been specified, * in which case each kthread actively polls its CPU. (Which isn't so great * for energy efficiency, but which does reduce RCU's overhead on that CPU.) * * This is intended to be used in conjunction with Frederic Weisbecker's * adaptive-idle work, which would seriously reduce OS jitter on CPUs * running CPU-bound user-mode computations. * * Offloading of callbacks can also be used as an energy-efficiency * measure because CPUs with no RCU callbacks queued are more aggressive * about entering dyntick-idle mode. */ /* * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. * If the list is invalid, a warning is emitted and all CPUs are offloaded. */ static int __init rcu_nocb_setup(char *str) { … } __setup(…); static int __init parse_rcu_nocb_poll(char *arg) { … } __setup(…); /* * Don't bother bypassing ->cblist if the call_rcu() rate is low. * After all, the main point of bypassing is to avoid lock contention * on ->nocb_lock, which only can happen at high call_rcu() rates. */ static int nocb_nobypass_lim_per_jiffy = …; module_param(nocb_nobypass_lim_per_jiffy, int, 0); /* * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the * lock isn't immediately available, perform minimal sanity check. */ static void rcu_nocb_bypass_lock(struct rcu_data *rdp) __acquires(&rdp->nocb_bypass_lock) { … } /* * Conditionally acquire the specified rcu_data structure's * ->nocb_bypass_lock. */ static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) { … } /* * Release the specified rcu_data structure's ->nocb_bypass_lock. */ static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) __releases(&rdp->nocb_bypass_lock) { … } /* * Acquire the specified rcu_data structure's ->nocb_lock, but only * if it corresponds to a no-CBs CPU. */ static void rcu_nocb_lock(struct rcu_data *rdp) { … } /* * Release the specified rcu_data structure's ->nocb_lock, but only * if it corresponds to a no-CBs CPU. */ static void rcu_nocb_unlock(struct rcu_data *rdp) { … } /* * Release the specified rcu_data structure's ->nocb_lock and restore * interrupts, but only if it corresponds to a no-CBs CPU. */ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, unsigned long flags) { … } /* Lockdep check that ->cblist may be safely accessed. */ static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) { … } /* * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended * grace period. */ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) { … } static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) { … } static void rcu_init_one_nocb(struct rcu_node *rnp) { … } static bool __wake_nocb_gp(struct rcu_data *rdp_gp, struct rcu_data *rdp, bool force, unsigned long flags) __releases(rdp_gp->nocb_gp_lock) { … } /* * Kick the GP kthread for this NOCB group. */ static bool wake_nocb_gp(struct rcu_data *rdp, bool force) { … } #ifdef CONFIG_RCU_LAZY /* * LAZY_FLUSH_JIFFIES decides the maximum amount of time that * can elapse before lazy callbacks are flushed. Lazy callbacks * could be flushed much earlier for a number of other reasons * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are * left unsubmitted to RCU after those many jiffies. */ #define LAZY_FLUSH_JIFFIES … static unsigned long jiffies_lazy_flush = …; // To be called only from test code. void rcu_set_jiffies_lazy_flush(unsigned long jif) { … } EXPORT_SYMBOL(…); unsigned long rcu_get_jiffies_lazy_flush(void) { … } EXPORT_SYMBOL(…); #endif /* * Arrange to wake the GP kthread for this NOCB group at some future * time when it is safe to do so. */ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, const char *reason) { … } /* * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL. * However, if there is a callback to be enqueued and if ->nocb_bypass * proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * * Return true if there was something to be flushed and it succeeded, otherwise * false. * * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in, unsigned long j, bool lazy) { … } /* * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL. * However, if there is a callback to be enqueued and if ->nocb_bypass * proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy) { … } /* * If the ->nocb_bypass_lock is immediately available, flush the * ->nocb_bypass queue into ->cblist. */ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) { … } /* * See whether it is appropriate to use the ->nocb_bypass list in order * to control contention on ->nocb_lock. A limited number of direct * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass * is non-empty, further callbacks must be placed into ->nocb_bypass, * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch * back to direct use of ->cblist. However, ->nocb_bypass should not be * used if ->cblist is empty, because otherwise callbacks can be stranded * on ->nocb_bypass because we cannot count on the current CPU ever again * invoking call_rcu(). The general rule is that if ->nocb_bypass is * non-empty, the corresponding no-CBs grace-period kthread must not be * in an indefinite sleep state. * * Finally, it is not permitted to use the bypass during early boot, * as doing so would confuse the auto-initialization code. Besides * which, there is no point in worrying about lock contention while * there is only one CPU in operation. */ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool *was_alldone, unsigned long flags, bool lazy) { … } /* * Awaken the no-CBs grace-period kthread if needed, either due to it * legitimately being asleep or due to overload conditions. * * If warranted, also wake up the kthread servicing this CPUs queues. */ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, unsigned long flags) __releases(rdp->nocb_lock) { … } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func, unsigned long flags, bool lazy) { … } static int nocb_gp_toggle_rdp(struct rcu_data *rdp) { … } static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) { … } /* * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. */ static void nocb_gp_wait(struct rcu_data *my_rdp) { … } /* * No-CBs grace-period-wait kthread. There is one of these per group * of CPUs, but only once at least one CPU in that group has come online * at least once since boot. This kthread checks for newly posted * callbacks from any of the CPUs it is responsible for, waits for a * grace period, then awakens all of the rcu_nocb_cb_kthread() instances * that then have callback-invocation work to do. */ static int rcu_nocb_gp_kthread(void *arg) { … } static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) { … } /* * Invoke any ready callbacks from the corresponding no-CBs CPU, * then, if there are no more, wait for more to appear. */ static void nocb_cb_wait(struct rcu_data *rdp) { … } /* * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke * nocb_cb_wait() to do the dirty work. */ static int rcu_nocb_cb_kthread(void *arg) { … } /* Is a deferred wakeup of rcu_nocb_kthread() required? */ static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) { … } /* Do a deferred wakeup of rcu_nocb_kthread(). */ static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp, struct rcu_data *rdp, int level, unsigned long flags) __releases(rdp_gp->nocb_gp_lock) { … } /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */ static void do_nocb_deferred_wakeup_timer(struct timer_list *t) { … } /* * Do a deferred wakeup of rcu_nocb_kthread() from fastpath. * This means we do an inexact common-case check. Note that if * we miss, ->nocb_timer will eventually clean things up. */ static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) { … } void rcu_nocb_flush_deferred_wakeup(void) { … } EXPORT_SYMBOL_GPL(…); static int rdp_offload_toggle(struct rcu_data *rdp, bool offload, unsigned long flags) __releases(rdp->nocb_lock) { … } static long rcu_nocb_rdp_deoffload(void *arg) { … } int rcu_nocb_cpu_deoffload(int cpu) { … } EXPORT_SYMBOL_GPL(…); static long rcu_nocb_rdp_offload(void *arg) { … } int rcu_nocb_cpu_offload(int cpu) { … } EXPORT_SYMBOL_GPL(…); #ifdef CONFIG_RCU_LAZY static unsigned long lazy_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { … } static unsigned long lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { … } #endif // #ifdef CONFIG_RCU_LAZY void __init rcu_init_nohz(void) { … } /* Initialize per-rcu_data variables for no-CBs CPUs. */ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) { … } /* * If the specified CPU is a no-CBs CPU that does not already have its * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread * for this CPU's group has not yet been created, spawn it as well. */ static void rcu_spawn_cpu_nocb_kthread(int cpu) { … } /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */ static int rcu_nocb_gp_stride = …; module_param(rcu_nocb_gp_stride, int, 0444); /* * Initialize GP-CB relationships for all no-CBs CPU. */ static void __init rcu_organize_nocb_kthreads(void) { … } /* * Bind the current task to the offloaded CPUs. If there are no offloaded * CPUs, leave the task unbound. Splat if the bind attempt fails. */ void rcu_bind_current_to_nocb(void) { … } EXPORT_SYMBOL_GPL(…); // The ->on_cpu field is available only in CONFIG_SMP=y, so... #ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu(struct task_struct *tsp) { … } #else // #ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu(struct task_struct *tsp) { return ""; } #endif // #else #ifdef CONFIG_SMP /* * Dump out nocb grace-period kthread state for the specified rcu_data * structure. */ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) { … } /* Dump out nocb kthread state for the specified rcu_data structure. */ static void show_rcu_nocb_state(struct rcu_data *rdp) { … } #else /* #ifdef CONFIG_RCU_NOCB_CPU */ static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) { return 0; } static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) { return false; } /* No ->nocb_lock to acquire. */ static void rcu_nocb_lock(struct rcu_data *rdp) { } /* No ->nocb_lock to release. */ static void rcu_nocb_unlock(struct rcu_data *rdp) { } /* No ->nocb_lock to release. */ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, unsigned long flags) { local_irq_restore(flags); } /* Lockdep check that ->cblist may be safely accessed. */ static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) { lockdep_assert_irqs_disabled(); } static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) { } static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) { return NULL; } static void rcu_init_one_nocb(struct rcu_node *rnp) { } static bool wake_nocb_gp(struct rcu_data *rdp, bool force) { return false; } static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy) { return true; } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func, unsigned long flags, bool lazy) { WARN_ON_ONCE(1); /* Should be dead code! */ } static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags) { WARN_ON_ONCE(1); /* Should be dead code! */ } static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) { } static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) { return false; } static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) { return false; } static void rcu_spawn_cpu_nocb_kthread(int cpu) { } static void show_rcu_nocb_state(struct rcu_data *rdp) { } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
Codebase Browser
linux