linux/mm/swap_slots.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Manage cache of swap slots to be used for and returned from
 * swap.
 *
 * Copyright(c) 2016 Intel Corporation.
 *
 * Author: Tim Chen <[email protected]>
 *
 * We allocate the swap slots from the global pool and put
 * it into local per cpu caches.  This has the advantage
 * of no needing to acquire the swap_info lock every time
 * we need a new slot.
 *
 * There is also opportunity to simply return the slot
 * to local caches without needing to acquire swap_info
 * lock.  We do not reuse the returned slots directly but
 * move them back to the global pool in a batch.  This
 * allows the slots to coalesce and reduce fragmentation.
 *
 * The swap entry allocated is marked with SWAP_HAS_CACHE
 * flag in map_count that prevents it from being allocated
 * again from the global pool.
 *
 * The swap slots cache is protected by a mutex instead of
 * a spin lock as when we search for slots with scan_swap_map,
 * we can possibly sleep.
 */

#include <linux/swap_slots.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mutex.h>
#include <linux/mm.h>

static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
static bool	swap_slot_cache_active;
bool	swap_slot_cache_enabled;
static bool	swap_slot_cache_initialized;
static DEFINE_MUTEX(swap_slots_cache_mutex);
/* Serialize swap slots cache enable/disable operations */
static DEFINE_MUTEX(swap_slots_cache_enable_mutex);

static void __drain_swap_slots_cache(unsigned int type);

#define use_swap_slot_cache …
#define SLOTS_CACHE …
#define SLOTS_CACHE_RET …

static void deactivate_swap_slots_cache(void)
{ … }

static void reactivate_swap_slots_cache(void)
{ … }

/* Must not be called with cpu hot plug lock */
void disable_swap_slots_cache_lock(void)
{ … }

static void __reenable_swap_slots_cache(void)
{ … }

void reenable_swap_slots_cache_unlock(void)
{ … }

static bool check_cache_active(void)
{ … }

static int alloc_swap_slot_cache(unsigned int cpu)
{ … }

static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
				  bool free_slots)
{ … }

static void __drain_swap_slots_cache(unsigned int type)
{ … }

static int free_slot_cache(unsigned int cpu)
{ … }

void enable_swap_slots_cache(void)
{ … }

/* called with swap slot cache's alloc lock held */
static int refill_swap_slots_cache(struct swap_slots_cache *cache)
{ … }

void free_swap_slot(swp_entry_t entry)
{ … }

swp_entry_t folio_alloc_swap(struct folio *folio)
{ … }