linux/arch/x86/kernel/apic/ipi.c

// SPDX-License-Identifier: GPL-2.0

#include <linux/cpumask.h>
#include <linux/delay.h>
#include <linux/smp.h>

#include <asm/io_apic.h>

#include "local.h"

DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);

#ifdef CONFIG_SMP
static int apic_ipi_shorthand_off __ro_after_init;

static __init int apic_ipi_shorthand(char *str)
{ … }
__setup(…);

static int __init print_ipi_mode(void)
{ … }
late_initcall(print_ipi_mode);

void apic_smt_update(void)
{ … }

void apic_send_IPI_allbutself(unsigned int vector)
{ … }

/*
 * Send a 'reschedule' IPI to another CPU. It goes straight through and
 * wastes no time serializing anything. Worst case is that we lose a
 * reschedule ...
 */
void native_smp_send_reschedule(int cpu)
{ … }

void native_send_call_func_single_ipi(int cpu)
{ … }

void native_send_call_func_ipi(const struct cpumask *mask)
{ … }

void apic_send_nmi_to_offline_cpu(unsigned int cpu)
{ … }
#endif /* CONFIG_SMP */

static inline int __prepare_ICR2(unsigned int mask)
{ … }

u32 apic_mem_wait_icr_idle_timeout(void)
{ … }

void apic_mem_wait_icr_idle(void)
{ … }

/*
 * This is safe against interruption because it only writes the lower 32
 * bits of the APIC_ICR register. The destination field is ignored for
 * short hand IPIs.
 *
 *  wait_icr_idle()
 *  write(ICR2, dest)
 *  NMI
 *	wait_icr_idle()
 *	write(ICR)
 *	wait_icr_idle()
 *  write(ICR)
 *
 * This function does not need to disable interrupts as there is no ICR2
 * interaction. The memory write is direct except when the machine is
 * affected by the 11AP Pentium erratum, which turns the plain write into
 * an XCHG operation.
 */
static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
{ … }

/*
 * This is used to send an IPI with no shorthand notation (the destination is
 * specified in bits 56 to 63 of the ICR).
 */
void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
				   unsigned int dest_mode)
{ … }

void default_send_IPI_single_phys(int cpu, int vector)
{ … }

void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
{ … }

void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
						 int vector)
{ … }

/*
 * Helper function for APICs which insist on cpumasks
 */
void default_send_IPI_single(int cpu, int vector)
{ … }

void default_send_IPI_allbutself(int vector)
{ … }

void default_send_IPI_all(int vector)
{ … }

void default_send_IPI_self(int vector)
{ … }

#ifdef CONFIG_X86_32
void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
{
	unsigned long flags;
	unsigned int cpu;

	local_irq_save(flags);
	for_each_cpu(cpu, mask)
		__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
	local_irq_restore(flags);
}

void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
						 int vector)
{
	unsigned int cpu, this_cpu = smp_processor_id();
	unsigned long flags;

	local_irq_save(flags);
	for_each_cpu(cpu, mask) {
		if (cpu == this_cpu)
			continue;
		__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
	}
	local_irq_restore(flags);
}

void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
{
	unsigned long mask = cpumask_bits(cpumask)[0];
	unsigned long flags;

	if (!mask)
		return;

	local_irq_save(flags);
	WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
	__default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
	local_irq_restore(flags);
}

#ifdef CONFIG_SMP
static int convert_apicid_to_cpu(u32 apic_id)
{
	int i;

	for_each_possible_cpu(i) {
		if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
			return i;
	}
	return -1;
}

int safe_smp_processor_id(void)
{
	u32 apicid;
	int cpuid;

	if (!boot_cpu_has(X86_FEATURE_APIC))
		return 0;

	apicid = read_apic_id();
	if (apicid == BAD_APICID)
		return 0;

	cpuid = convert_apicid_to_cpu(apicid);

	return cpuid >= 0 ? cpuid : 0;
}
#endif
#endif