/* SPDX-License-Identifier: GPL-2.0+ */ /* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * Internal non-public definitions that provide either classic * or preemptible semantics. * * Copyright Red Hat, 2009 * Copyright IBM Corporation, 2009 * * Author: Ingo Molnar <[email protected]> * Paul E. McKenney <[email protected]> */ #include "../locking/rtmutex_common.h" static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) { … } /* * Check the RCU kernel configuration parameters and print informative * messages about anything out of the ordinary. */ static void __init rcu_bootup_announce_oddness(void) { … } #ifdef CONFIG_PREEMPT_RCU static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake); static void rcu_read_unlock_special(struct task_struct *t); /* * Tell them what RCU they are running. */ static void __init rcu_bootup_announce(void) { … } /* Flags for rcu_preempt_ctxt_queue() decision table. */ #define RCU_GP_TASKS … #define RCU_EXP_TASKS … #define RCU_GP_BLKD … #define RCU_EXP_BLKD … /* * Queues a task preempted within an RCU-preempt read-side critical * section into the appropriate location within the ->blkd_tasks list, * depending on the states of any ongoing normal and expedited grace * periods. The ->gp_tasks pointer indicates which element the normal * grace period is waiting on (NULL if none), and the ->exp_tasks pointer * indicates which element the expedited grace period is waiting on (again, * NULL if none). If a grace period is waiting on a given element in the * ->blkd_tasks list, it also waits on all subsequent elements. Thus, * adding a task to the tail of the list blocks any grace period that is * already waiting on one of the elements. In contrast, adding a task * to the head of the list won't block any grace period that is already * waiting on one of the elements. * * This queuing is imprecise, and can sometimes make an ongoing grace * period wait for a task that is not strictly speaking blocking it. * Given the choice, we needlessly block a normal grace period rather than * blocking an expedited grace period. * * Note that an endless sequence of expedited grace periods still cannot * indefinitely postpone a normal grace period. Eventually, all of the * fixed number of preempted tasks blocking the normal grace period that are * not also blocking the expedited grace period will resume and complete * their RCU read-side critical sections. At that point, the ->gp_tasks * pointer will equal the ->exp_tasks pointer, at which point the end of * the corresponding expedited grace period will also be the end of the * normal grace period. */ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) __releases(rnp->lock) /* But leaves rrupts disabled. */ { … } /* * Record a preemptible-RCU quiescent state for the specified CPU. * Note that this does not necessarily mean that the task currently running * on the CPU is in a quiescent state: Instead, it means that the current * grace period need not wait on any RCU read-side critical section that * starts later on this CPU. It also means that if the current task is * in an RCU read-side critical section, it has already added itself to * some leaf rcu_node structure's ->blkd_tasks list. In addition to the * current task, there might be any number of other tasks blocked while * in an RCU read-side critical section. * * Unlike non-preemptible-RCU, quiescent state reports for expedited * grace periods are handled separately via deferred quiescent states * and context switch events. * * Callers to this function must disable preemption. */ static void rcu_qs(void) { … } /* * We have entered the scheduler, and the current task might soon be * context-switched away from. If this task is in an RCU read-side * critical section, we will no longer be able to rely on the CPU to * record that fact, so we enqueue the task on the blkd_tasks list. * The task will dequeue itself when it exits the outermost enclosing * RCU read-side critical section. Therefore, the current grace period * cannot be permitted to complete until the blkd_tasks list entries * predating the current grace period drain, in other words, until * rnp->gp_tasks becomes NULL. * * Caller must disable interrupts. */ void rcu_note_context_switch(bool preempt) { … } EXPORT_SYMBOL_GPL(…); /* * Check for preempted RCU readers blocking the current grace period * for the specified rcu_node structure. If the caller needs a reliable * answer, it must hold the rcu_node's ->lock. */ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) { … } /* limit value for ->rcu_read_lock_nesting. */ #define RCU_NEST_PMAX … static void rcu_preempt_read_enter(void) { … } static int rcu_preempt_read_exit(void) { … } static void rcu_preempt_depth_set(int val) { … } /* * Preemptible RCU implementation for rcu_read_lock(). * Just increment ->rcu_read_lock_nesting, shared state will be updated * if we block. */ void __rcu_read_lock(void) { … } EXPORT_SYMBOL_GPL(…); /* * Preemptible RCU implementation for rcu_read_unlock(). * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then * invoke rcu_read_unlock_special() to clean up after a context switch * in an RCU read-side critical section and other special cases. */ void __rcu_read_unlock(void) { … } EXPORT_SYMBOL_GPL(…); /* * Advance a ->blkd_tasks-list pointer to the next entry, instead * returning NULL if at the end of the list. */ static struct list_head *rcu_next_node_entry(struct task_struct *t, struct rcu_node *rnp) { … } /* * Return true if the specified rcu_node structure has tasks that were * preempted within an RCU read-side critical section. */ static bool rcu_preempt_has_tasks(struct rcu_node *rnp) { … } /* * Report deferred quiescent states. The deferral time can * be quite short, for example, in the case of the call from * rcu_read_unlock_special(). */ static notrace void rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) { … } /* * Is a deferred quiescent-state pending, and are we also not in * an RCU read-side critical section? It is the caller's responsibility * to ensure it is otherwise safe to report any deferred quiescent * states. The reason for this is that it is safe to report a * quiescent state during context switch even though preemption * is disabled. This function cannot be expected to understand these * nuances, so the caller must handle them. */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { … } /* * Report a deferred quiescent state if needed and safe to do so. * As with rcu_preempt_need_deferred_qs(), "safe" involves only * not being in an RCU read-side critical section. The caller must * evaluate safety in terms of interrupt, softirq, and preemption * disabling. */ notrace void rcu_preempt_deferred_qs(struct task_struct *t) { … } /* * Minimal handler to give the scheduler a chance to re-evaluate. */ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) { … } /* * Handle special cases during rcu_read_unlock(), such as needing to * notify RCU core processing or task having blocked during the RCU * read-side critical section. */ static void rcu_read_unlock_special(struct task_struct *t) { … } /* * Check that the list of blocked tasks for the newly completed grace * period is in fact empty. It is a serious bug to complete a grace * period that still has RCU readers blocked! This function must be * invoked -before- updating this rnp's ->gp_seq. * * Also, if there are blocked tasks on the list, they automatically * block the newly created grace period, so set up ->gp_tasks accordingly. */ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) { … } /* * Check for a quiescent state from the current CPU, including voluntary * context switches for Tasks RCU. When a task blocks, the task is * recorded in the corresponding CPU's rcu_node structure, which is checked * elsewhere, hence this function need only check for quiescent states * related to the current CPU, not to those related to tasks. */ static void rcu_flavor_sched_clock_irq(int user) { … } /* * Check for a task exiting while in a preemptible-RCU read-side * critical section, clean up if so. No need to issue warnings, as * debug_check_no_locks_held() already does this if lockdep is enabled. * Besides, if this function does anything other than just immediately * return, there was a bug of some sort. Spewing warnings from this * function is like as not to simply obscure important prior warnings. */ void exit_rcu(void) { … } /* * Dump the blocked-tasks state, but limit the list dump to the * specified number of elements. */ static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck) { … } #else /* #ifdef CONFIG_PREEMPT_RCU */ /* * If strict grace periods are enabled, and if the calling * __rcu_read_unlock() marks the beginning of a quiescent state, immediately * report that quiescent state and, if requested, spin for a bit. */ void rcu_read_unlock_strict(void) { struct rcu_data *rdp; if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) return; rdp = this_cpu_ptr(&rcu_data); rdp->cpu_no_qs.b.norm = false; rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } EXPORT_SYMBOL_GPL(rcu_read_unlock_strict); /* * Tell them what RCU they are running. */ static void __init rcu_bootup_announce(void) { pr_info("Hierarchical RCU implementation.\n"); rcu_bootup_announce_oddness(); } /* * Note a quiescent state for PREEMPTION=n. Because we do not need to know * how many quiescent states passed, just if there was at least one since * the start of the grace period, this just sets a flag. The caller must * have disabled preemption. */ static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!"); if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } /* * Register an urgently needed quiescent state. If there is an * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight * dyntick-idle quiescent state visible to other CPUs, which will in * some cases serve for expedited as well as normal grace periods. * Either way, register a lightweight quiescent state. */ void rcu_all_qs(void) { unsigned long flags; if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { preempt_enable(); return; } this_cpu_write(rcu_data.rcu_urgent_qs, false); if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { local_irq_save(flags); rcu_momentary_dyntick_idle(); local_irq_restore(flags); } rcu_qs(); preempt_enable(); } EXPORT_SYMBOL_GPL(rcu_all_qs); /* * Note a PREEMPTION=n context switch. The caller must have disabled interrupts. */ void rcu_note_context_switch(bool preempt) { trace_rcu_utilization(TPS("Start context switch")); rcu_qs(); /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; this_cpu_write(rcu_data.rcu_urgent_qs, false); if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) rcu_momentary_dyntick_idle(); out: rcu_tasks_qs(current, preempt); trace_rcu_utilization(TPS("End context switch")); } EXPORT_SYMBOL_GPL(rcu_note_context_switch); /* * Because preemptible RCU does not exist, there are never any preempted * RCU readers. */ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) { return 0; } /* * Because there is no preemptible RCU, there can be no readers blocked. */ static bool rcu_preempt_has_tasks(struct rcu_node *rnp) { return false; } /* * Because there is no preemptible RCU, there can be no deferred quiescent * states. */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { return false; } // Except that we do need to respond to a request by an expedited // grace period for a quiescent state from this CPU. Note that in // non-preemptible kernels, there can be no context switches within RCU // read-side critical sections, which in turn means that the leaf rcu_node // structure's blocked-tasks list is always empty. is therefore no need to // actually check it. Instead, a quiescent state from this CPU suffices, // and this function is only called from such a quiescent state. notrace void rcu_preempt_deferred_qs(struct task_struct *t) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); if (READ_ONCE(rdp->cpu_no_qs.b.exp)) rcu_report_exp_rdp(rdp); } /* * Because there is no preemptible RCU, there can be no readers blocked, * so there is no need to check for blocked tasks. So check only for * bogus qsmask values. */ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) { WARN_ON_ONCE(rnp->qsmask); } /* * Check to see if this CPU is in a non-context-switch quiescent state, * namely user mode and idle loop. */ static void rcu_flavor_sched_clock_irq(int user) { if (user || rcu_is_cpu_rrupt_from_idle()) { /* * Get here if this CPU took its interrupt from user * mode or from the idle loop, and if this is not a * nested interrupt. In this case, the CPU is in * a quiescent state, so note it. * * No memory barrier is required here because rcu_qs() * references only CPU-local variables that other CPUs * neither access nor modify, at least not while the * corresponding CPU is online. */ rcu_qs(); } } /* * Because preemptible RCU does not exist, tasks cannot possibly exit * while in preemptible RCU read-side critical sections. */ void exit_rcu(void) { } /* * Dump the guaranteed-empty blocked-tasks state. Trust but verify. */ static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck) { WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks)); } #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* * If boosting, set rcuc kthreads to realtime priority. */ static void rcu_cpu_kthread_setup(unsigned int cpu) { … } static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) { … } /* * Is the current CPU running the RCU-callbacks kthread? * Caller must have preemption disabled. */ static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) { … } #ifdef CONFIG_RCU_BOOST /* * Carry out RCU priority boosting on the task indicated by ->exp_tasks * or ->boost_tasks, advancing the pointer to the next task in the * ->blkd_tasks list. * * Note that irqs must be enabled: boosting the task can block. * Returns 1 if there are more tasks needing to be boosted. */ static int rcu_boost(struct rcu_node *rnp) { … } /* * Priority-boosting kthread, one per leaf rcu_node. */ static int rcu_boost_kthread(void *arg) { … } /* * Check to see if it is time to start boosting RCU readers that are * blocking the current grace period, and, if so, tell the per-rcu_node * kthread to start boosting them. If there is an expedited grace * period in progress, it is always time to boost. * * The caller must hold rnp->lock, which this function releases. * The ->boost_kthread_task is immortal, so we don't need to worry * about it going away. */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock) { … } #define RCU_BOOST_DELAY_JIFFIES … /* * Do priority-boost accounting for the start of a new grace period. */ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) { … } /* * Create an RCU-boost kthread for the specified node if one does not * already exist. We only create this kthread for preemptible RCU. */ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) { … } static struct task_struct *rcu_boost_task(struct rcu_node *rnp) { … } #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) { } static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) { } static struct task_struct *rcu_boost_task(struct rcu_node *rnp) { return NULL; } #endif /* #else #ifdef CONFIG_RCU_BOOST */ /* * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the * grace-period kthread will do force_quiescent_state() processing? * The idea is to avoid waking up RCU core processing on such a * CPU unless the grace period has extended for too long. * * This code relies on the fact that all NO_HZ_FULL CPUs are also * RCU_NOCB_CPU CPUs. */ static bool rcu_nohz_full_cpu(void) { … } /* * Bind the RCU grace-period kthreads to the housekeeping CPU. */ static void rcu_bind_gp_kthread(void) { … }
Codebase Browser
linux