/* SPDX-License-Identifier: GPL-2.0+ */ /* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * Internal non-public definitions. * * Copyright IBM Corporation, 2008 * * Author: Ingo Molnar <[email protected]> * Paul E. McKenney <[email protected]> */ #include <linux/cache.h> #include <linux/kthread.h> #include <linux/spinlock.h> #include <linux/rtmutex.h> #include <linux/threads.h> #include <linux/cpumask.h> #include <linux/seqlock.h> #include <linux/swait.h> #include <linux/rcu_node_tree.h> #include "rcu_segcblist.h" /* Communicate arguments to a kthread worker handler. */ struct rcu_exp_work { … }; /* RCU's kthread states for tracing. */ #define RCU_KTHREAD_STOPPED … #define RCU_KTHREAD_RUNNING … #define RCU_KTHREAD_WAITING … #define RCU_KTHREAD_OFFCPU … #define RCU_KTHREAD_YIELDING … #define RCU_KTHREAD_MAX … /* * Definition for node within the RCU grace-period-detection hierarchy. */ struct rcu_node { … } ____cacheline_internodealigned_in_smp; /* * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and * are indexed relative to this interval rather than the global CPU ID space. * This generates the bit for a CPU in node-local masks. */ #define leaf_node_cpu_bit(rnp, cpu) … /* * Union to allow "aggregate OR" operation on the need for a quiescent * state by the normal and expedited grace periods. */ rcu_noqs; /* * Record the snapshot of the core stats at half of the first RCU stall timeout. * The member gp_seq is used to ensure that all members are updated only once * during the sampling period. The snapshot is taken only if this gp_seq is not * equal to rdp->gp_seq. */ struct rcu_snap_record { … }; /* Per-CPU data for read-copy update. */ struct rcu_data { … }; /* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT … #define RCU_NOCB_WAKE_BYPASS … #define RCU_NOCB_WAKE_LAZY … #define RCU_NOCB_WAKE … #define RCU_NOCB_WAKE_FORCE … #define RCU_JIFFIES_TILL_FORCE_QS … /* For jiffies_till_first_fqs and */ /* and jiffies_till_next_fqs. */ #define RCU_JIFFIES_FQS_DIV … /* delay between bouts of */ /* quiescent-state forcing. */ #define RCU_STALL_RAT_DELAY … /* at least one scheduling clock */ /* irq before ratting on them. */ #define rcu_wait(cond) … /* * A max threshold for synchronize_rcu() users which are * awaken directly by the rcu_gp_kthread(). Left part is * deferred to the main worker. */ #define SR_MAX_USERS_WAKE_FROM_GP … #define SR_NORMAL_GP_WAIT_HEAD_MAX … struct sr_wait_node { … }; /* * RCU global state, including node hierarchy. This hierarchy is * represented in "heap" form in a dense array. The root (first level) * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]), * and the third level in ->node[m+1] and following (->node[m+1] referenced * by ->level[2]). The number of levels is determined by the number of * CPUs and by CONFIG_RCU_FANOUT. Small systems will have a "hierarchy" * consisting of a single rcu_node. */ struct rcu_state { … }; /* Values for rcu_state structure's gp_flags field. */ #define RCU_GP_FLAG_INIT … #define RCU_GP_FLAG_FQS … #define RCU_GP_FLAG_OVLD … /* Values for rcu_state structure's gp_state field. */ #define RCU_GP_IDLE … #define RCU_GP_WAIT_GPS … #define RCU_GP_DONE_GPS … #define RCU_GP_ONOFF … #define RCU_GP_INIT … #define RCU_GP_WAIT_FQS … #define RCU_GP_DOING_FQS … #define RCU_GP_CLEANUP … #define RCU_GP_CLEANED … /* * In order to export the rcu_state name to the tracing tools, it * needs to be added in the __tracepoint_string section. * This requires defining a separate variable tp_<sname>_varname * that points to the string being used, and this will allow * the tracing userspace tools to be able to decipher the string * address to the matching string. */ #ifdef CONFIG_PREEMPT_RCU #define RCU_ABBR … #define RCU_NAME_RAW … #else /* #ifdef CONFIG_PREEMPT_RCU */ #define RCU_ABBR … #define RCU_NAME_RAW … #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ #ifndef CONFIG_TRACING #define RCU_NAME … #else /* #ifdef CONFIG_TRACING */ static char rcu_name[] = …; static const char *tp_rcu_varname __used __tracepoint_string = …; #define RCU_NAME … #endif /* #else #ifdef CONFIG_TRACING */ /* Forward declarations for tree_plugin.h */ static void rcu_bootup_announce(void); static void rcu_qs(void); static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); #ifdef CONFIG_HOTPLUG_CPU static bool rcu_preempt_has_tasks(struct rcu_node *rnp); #endif /* #ifdef CONFIG_HOTPLUG_CPU */ static int rcu_print_task_exp_stall(struct rcu_node *rnp); static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); static void rcu_flavor_sched_clock_irq(int user); static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); static bool rcu_is_callbacks_kthread(struct rcu_data *rdp); static void rcu_cpu_kthread_setup(unsigned int cpu); static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp); static bool rcu_preempt_has_tasks(struct rcu_node *rnp); static bool rcu_preempt_need_deferred_qs(struct task_struct *t); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); static void rcu_init_one_nocb(struct rcu_node *rnp); static bool wake_nocb_gp(struct rcu_data *rdp, bool force); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func, unsigned long flags, bool lazy); static void __maybe_unused __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level); static bool do_nocb_deferred_wakeup(struct rcu_data *rdp); static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp); static void rcu_spawn_cpu_nocb_kthread(int cpu); static void show_rcu_nocb_state(struct rcu_data *rdp); static void rcu_nocb_lock(struct rcu_data *rdp); static void rcu_nocb_unlock(struct rcu_data *rdp); static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, unsigned long flags); static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp); #ifdef CONFIG_RCU_NOCB_CPU static void __init rcu_organize_nocb_kthreads(void); /* * Disable IRQs before checking offloaded state so that local * locking is safe against concurrent de-offloading. */ #define rcu_nocb_lock_irqsave(rdp, flags) … #else /* #ifdef CONFIG_RCU_NOCB_CPU */ #define rcu_nocb_lock_irqsave … #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ static void rcu_bind_gp_kthread(void); static bool rcu_nohz_full_cpu(void); /* Forward declarations for tree_stall.h */ static void record_gp_stall_check_time(void); static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, const unsigned long gpssdelay); /* Forward declarations for tree_exp.h. */ static void sync_rcu_do_polled_gp(struct work_struct *wp);
Codebase Browser
linux