linux/kernel/locking/percpu-rwsem.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/atomic.h>
#include <linux/percpu.h>
#include <linux/wait.h>
#include <linux/lockdep.h>
#include <linux/percpu-rwsem.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/sched/debug.h>
#include <linux/errno.h>
#include <trace/events/lock.h>

int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
			const char *name, struct lock_class_key *key)
{ … }
EXPORT_SYMBOL_GPL(…);

void percpu_free_rwsem(struct percpu_rw_semaphore *sem)
{ … }
EXPORT_SYMBOL_GPL(…);

static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
{ … }

static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem)
{ … }

static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader)
{ … }

/*
 * The return value of wait_queue_entry::func means:
 *
 *  <0 - error, wakeup is terminated and the error is returned
 *   0 - no wakeup, a next waiter is tried
 *  >0 - woken, if EXCLUSIVE, counted towards @nr_exclusive.
 *
 * We use EXCLUSIVE for both readers and writers to preserve FIFO order,
 * and play games with the return value to allow waking multiple readers.
 *
 * Specifically, we wake readers until we've woken a single writer, or until a
 * trylock fails.
 */
static int percpu_rwsem_wake_function(struct wait_queue_entry *wq_entry,
				      unsigned int mode, int wake_flags,
				      void *key)
{ … }

static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader)
{ … }

bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
{ … }
EXPORT_SYMBOL_GPL(…);

#define per_cpu_sum(var) …

bool percpu_is_read_locked(struct percpu_rw_semaphore *sem)
{ … }
EXPORT_SYMBOL_GPL(…);

/*
 * Return true if the modular sum of the sem->read_count per-CPU variable is
 * zero.  If this sum is zero, then it is stable due to the fact that if any
 * newly arriving readers increment a given counter, they will immediately
 * decrement that same counter.
 *
 * Assumes sem->block is set.
 */
static bool readers_active_check(struct percpu_rw_semaphore *sem)
{ … }

void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
{ … }
EXPORT_SYMBOL_GPL(…);

void percpu_up_write(struct percpu_rw_semaphore *sem)
{ … }
EXPORT_SYMBOL_GPL(…);