// SPDX-License-Identifier: GPL-2.0+ /* * RCU segmented callback lists, function definitions * * Copyright IBM Corporation, 2017 * * Authors: Paul E. McKenney <[email protected]> */ #include <linux/cpu.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/types.h> #include "rcu_segcblist.h" /* Initialize simple callback list. */ void rcu_cblist_init(struct rcu_cblist *rclp) { … } /* * Enqueue an rcu_head structure onto the specified callback list. */ void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp) { … } /* * Flush the second rcu_cblist structure onto the first one, obliterating * any contents of the first. If rhp is non-NULL, enqueue it as the sole * element of the second rcu_cblist structure, but ensuring that the second * rcu_cblist structure, if initially non-empty, always appears non-empty * throughout the process. If rdp is NULL, the second rcu_cblist structure * is instead initialized to empty. */ void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, struct rcu_cblist *srclp, struct rcu_head *rhp) { … } /* * Dequeue the oldest rcu_head structure from the specified callback * list. */ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp) { … } /* Set the length of an rcu_segcblist structure. */ static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) { … } /* Get the length of a segment of the rcu_segcblist structure. */ long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) { … } /* Return number of callbacks in segmented callback list by summing seglen. */ long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp) { … } /* Set the length of a segment of the rcu_segcblist structure. */ static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v) { … } /* Increase the numeric length of a segment by a specified amount. */ static void rcu_segcblist_add_seglen(struct rcu_segcblist *rsclp, int seg, long v) { … } /* Move from's segment length to to's segment. */ static void rcu_segcblist_move_seglen(struct rcu_segcblist *rsclp, int from, int to) { … } /* Increment segment's length. */ static void rcu_segcblist_inc_seglen(struct rcu_segcblist *rsclp, int seg) { … } /* * Increase the numeric length of an rcu_segcblist structure by the * specified amount, which can be negative. This can cause the ->len * field to disagree with the actual number of callbacks on the structure. * This increase is fully ordered with respect to the callers accesses * both before and after. * * So why on earth is a memory barrier required both before and after * the update to the ->len field??? * * The reason is that rcu_barrier() locklessly samples each CPU's ->len * field, and if a given CPU's field is zero, avoids IPIing that CPU. * This can of course race with both queuing and invoking of callbacks. * Failing to correctly handle either of these races could result in * rcu_barrier() failing to IPI a CPU that actually had callbacks queued * which rcu_barrier() was obligated to wait on. And if rcu_barrier() * failed to wait on such a callback, unloading certain kernel modules * would result in calls to functions whose code was no longer present in * the kernel, for but one example. * * Therefore, ->len transitions from 1->0 and 0->1 have to be carefully * ordered with respect with both list modifications and the rcu_barrier(). * * The queuing case is CASE 1 and the invoking case is CASE 2. * * CASE 1: Suppose that CPU 0 has no callbacks queued, but invokes * call_rcu() just as CPU 1 invokes rcu_barrier(). CPU 0's ->len field * will transition from 0->1, which is one of the transitions that must * be handled carefully. Without the full memory barriers after the ->len * update and at the beginning of rcu_barrier(), the following could happen: * * CPU 0 CPU 1 * * call_rcu(). * rcu_barrier() sees ->len as 0. * set ->len = 1. * rcu_barrier() does nothing. * module is unloaded. * callback invokes unloaded function! * * With the full barriers, any case where rcu_barrier() sees ->len as 0 will * have unambiguously preceded the return from the racing call_rcu(), which * means that this call_rcu() invocation is OK to not wait on. After all, * you are supposed to make sure that any problematic call_rcu() invocations * happen before the rcu_barrier(). * * * CASE 2: Suppose that CPU 0 is invoking its last callback just as * CPU 1 invokes rcu_barrier(). CPU 0's ->len field will transition from * 1->0, which is one of the transitions that must be handled carefully. * Without the full memory barriers before the ->len update and at the * end of rcu_barrier(), the following could happen: * * CPU 0 CPU 1 * * start invoking last callback * set ->len = 0 (reordered) * rcu_barrier() sees ->len as 0 * rcu_barrier() does nothing. * module is unloaded * callback executing after unloaded! * * With the full barriers, any case where rcu_barrier() sees ->len as 0 * will be fully ordered after the completion of the callback function, * so that the module unloading operation is completely safe. * */ void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v) { … } /* * Increase the numeric length of an rcu_segcblist structure by one. * This can cause the ->len field to disagree with the actual number of * callbacks on the structure. This increase is fully ordered with respect * to the callers accesses both before and after. */ void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp) { … } /* * Initialize an rcu_segcblist structure. */ void rcu_segcblist_init(struct rcu_segcblist *rsclp) { … } /* * Disable the specified rcu_segcblist structure, so that callbacks can * no longer be posted to it. This structure must be empty. */ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) { … } /* * Mark the specified rcu_segcblist structure as offloaded (or not) */ void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload) { … } /* * Does the specified rcu_segcblist structure contain callbacks that * are ready to be invoked? */ bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp) { … } /* * Does the specified rcu_segcblist structure contain callbacks that * are still pending, that is, not yet ready to be invoked? */ bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp) { … } /* * Return a pointer to the first callback in the specified rcu_segcblist * structure. This is useful for diagnostics. */ struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp) { … } /* * Return a pointer to the first pending callback in the specified * rcu_segcblist structure. This is useful just after posting a given * callback -- if that callback is the first pending callback, then * you cannot rely on someone else having already started up the required * grace period. */ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) { … } /* * Return false if there are no CBs awaiting grace periods, otherwise, * return true and store the nearest waited-upon grace period into *lp. */ bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) { … } /* * Enqueue the specified callback onto the specified rcu_segcblist * structure, updating accounting as needed. Note that the ->len * field may be accessed locklessly, hence the WRITE_ONCE(). * The ->len field is used by rcu_barrier() and friends to determine * if it must post a callback on this structure, and it is OK * for rcu_barrier() to sometimes post callbacks needlessly, but * absolutely not OK for it to ever miss posting a callback. */ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, struct rcu_head *rhp) { … } /* * Entrain the specified callback onto the specified rcu_segcblist at * the end of the last non-empty segment. If the entire rcu_segcblist * is empty, make no change, but return false. * * This is intended for use by rcu_barrier()-like primitives, -not- * for normal grace-period use. IMPORTANT: The callback you enqueue * will wait for all prior callbacks, NOT necessarily for a grace * period. You have been warned. */ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, struct rcu_head *rhp) { … } /* * Extract only those callbacks ready to be invoked from the specified * rcu_segcblist structure and place them in the specified rcu_cblist * structure. */ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { … } /* * Extract only those callbacks still pending (not yet ready to be * invoked) from the specified rcu_segcblist structure and place them in * the specified rcu_cblist structure. Note that this loses information * about any callbacks that might have been partway done waiting for * their grace period. Too bad! They will have to start over. */ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { … } /* * Insert counts from the specified rcu_cblist structure in the * specified rcu_segcblist structure. */ void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { … } /* * Move callbacks from the specified rcu_cblist to the beginning of the * done-callbacks segment of the specified rcu_segcblist. */ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { … } /* * Move callbacks from the specified rcu_cblist to the end of the * new-callbacks segment of the specified rcu_segcblist. */ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { … } /* * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { … } /* * "Accelerate" callbacks based on more-accurate grace-period information. * The reason for this is that RCU does not synchronize the beginnings and * ends of grace periods, and that callbacks are posted locally. This in * turn means that the callbacks must be labelled conservatively early * on, as getting exact information would degrade both performance and * scalability. When more accurate grace-period information becomes * available, previously posted callbacks can be "accelerated", marking * them to complete at the end of the earlier grace period. * * This function operates on an rcu_segcblist structure, and also the * grace-period sequence number seq at which new callbacks would become * ready to invoke. Returns true if there are callbacks that won't be * ready to invoke until seq, false otherwise. */ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { … } /* * Merge the source rcu_segcblist structure into the destination * rcu_segcblist structure, then initialize the source. Any pending * callbacks from the source get to start over. It is best to * advance and accelerate both the destination and the source * before merging. */ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp) { … }
Codebase Browser
linux