linux/arch/x86/kernel/setup_percpu.c

// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) …

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/memblock.h>
#include <linux/percpu.h>
#include <linux/kexec.h>
#include <linux/crash_dump.h>
#include <linux/smp.h>
#include <linux/topology.h>
#include <linux/pfn.h>
#include <linux/stackprotector.h>
#include <asm/sections.h>
#include <asm/processor.h>
#include <asm/desc.h>
#include <asm/setup.h>
#include <asm/mpspec.h>
#include <asm/apicdef.h>
#include <asm/highmem.h>
#include <asm/proto.h>
#include <asm/cpumask.h>
#include <asm/cpu.h>

#ifdef CONFIG_X86_64
#define BOOT_PERCPU_OFFSET …
#else
#define BOOT_PERCPU_OFFSET …
#endif

DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = …;
EXPORT_PER_CPU_SYMBOL(…);

unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init = …;
EXPORT_SYMBOL(…);

/*
 * On x86_64 symbols referenced from code should be reachable using
 * 32bit relocations.  Reserve space for static percpu variables in
 * modules so that they are always served from the first chunk which
 * is located at the percpu segment base.  On x86_32, anything can
 * address anywhere.  No need to reserve space in the first chunk.
 */
#ifdef CONFIG_X86_64
#define PERCPU_FIRST_CHUNK_RESERVE …
#else
#define PERCPU_FIRST_CHUNK_RESERVE …
#endif

#ifdef CONFIG_X86_32
/**
 * pcpu_need_numa - determine percpu allocation needs to consider NUMA
 *
 * If NUMA is not configured or there is only one NUMA node available,
 * there is no reason to consider NUMA.  This function determines
 * whether percpu allocation should consider NUMA or not.
 *
 * RETURNS:
 * true if NUMA should be considered; otherwise, false.
 */
static bool __init pcpu_need_numa(void)
{
#ifdef CONFIG_NUMA
	pg_data_t *last = NULL;
	unsigned int cpu;

	for_each_possible_cpu(cpu) {
		int node = early_cpu_to_node(cpu);

		if (node_online(node) && NODE_DATA(node) &&
		    last && last != NODE_DATA(node))
			return true;

		last = NODE_DATA(node);
	}
#endif
	return false;
}
#endif

static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
{ … }

static int __init pcpu_cpu_to_node(int cpu)
{ … }

void __init pcpu_populate_pte(unsigned long addr)
{ … }

static inline void setup_percpu_segment(int cpu)
{ … }

void __init setup_per_cpu_areas(void)
{ … }