// SPDX-License-Identifier: GPL-2.0-only /* * Local APIC handling, local APIC timers * * (c) 1999, 2000, 2009 Ingo Molnar <[email protected]> * * Fixes * Maciej W. Rozycki : Bits for genuine 82489DX APICs; * thanks to Eric Gilmore * and Rolf G. Tews * for testing these extensively. * Maciej W. Rozycki : Various updates and fixes. * Mikael Pettersson : Power Management for UP-APIC. * Pavel Machek and * Mikael Pettersson : PM converted to driver model. */ #include <linux/perf_event.h> #include <linux/kernel_stat.h> #include <linux/mc146818rtc.h> #include <linux/acpi_pmtmr.h> #include <linux/bitmap.h> #include <linux/clockchips.h> #include <linux/interrupt.h> #include <linux/memblock.h> #include <linux/ftrace.h> #include <linux/ioport.h> #include <linux/export.h> #include <linux/syscore_ops.h> #include <linux/delay.h> #include <linux/timex.h> #include <linux/i8253.h> #include <linux/dmar.h> #include <linux/init.h> #include <linux/cpu.h> #include <linux/dmi.h> #include <linux/smp.h> #include <linux/mm.h> #include <xen/xen.h> #include <asm/trace/irq_vectors.h> #include <asm/irq_remapping.h> #include <asm/pc-conf-reg.h> #include <asm/perf_event.h> #include <asm/x86_init.h> #include <linux/atomic.h> #include <asm/barrier.h> #include <asm/mpspec.h> #include <asm/i8259.h> #include <asm/proto.h> #include <asm/traps.h> #include <asm/apic.h> #include <asm/acpi.h> #include <asm/io_apic.h> #include <asm/desc.h> #include <asm/hpet.h> #include <asm/mtrr.h> #include <asm/time.h> #include <asm/smp.h> #include <asm/mce.h> #include <asm/tsc.h> #include <asm/hypervisor.h> #include <asm/cpu_device_id.h> #include <asm/intel-family.h> #include <asm/irq_regs.h> #include <asm/cpu.h> #include "local.h" /* Processor that is doing the boot up */ u32 boot_cpu_physical_apicid __ro_after_init = …; EXPORT_SYMBOL_GPL(…); u8 boot_cpu_apic_version __ro_after_init; /* * This variable controls which CPUs receive external NMIs. By default, * external NMIs are delivered only to the BSP. */ static int apic_extnmi __ro_after_init = …; /* * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID */ static bool virt_ext_dest_id __ro_after_init; /* For parallel bootup. */ unsigned long apic_mmio_base __ro_after_init; static inline bool apic_accessible(void) { … } #ifdef CONFIG_X86_32 /* Local APIC was disabled by the BIOS and enabled by the kernel */ static int enabled_via_apicbase __ro_after_init; /* * Handle interrupt mode configuration register (IMCR). * This register controls whether the interrupt signals * that reach the BSP come from the master PIC or from the * local APIC. Before entering Symmetric I/O Mode, either * the BIOS or the operating system must switch out of * PIC Mode by changing the IMCR. */ static inline void imcr_pic_to_apic(void) { /* NMI and 8259 INTR go through APIC */ pc_conf_set(PC_CONF_MPS_IMCR, 0x01); } static inline void imcr_apic_to_pic(void) { /* NMI and 8259 INTR go directly to BSP */ pc_conf_set(PC_CONF_MPS_IMCR, 0x00); } #endif /* * Knob to control our willingness to enable the local APIC. * * +1=force-enable */ static int force_enable_local_apic __initdata; /* * APIC command line parameters */ static int __init parse_lapic(char *arg) { … } early_param(…); #ifdef CONFIG_X86_64 static int apic_calibrate_pmtmr __initdata; static __init int setup_apicpmtimer(char *s) { … } __setup(…); #endif static unsigned long mp_lapic_addr __ro_after_init; bool apic_is_disabled __ro_after_init; /* Disable local APIC timer from the kernel commandline or via dmi quirk */ static int disable_apic_timer __initdata; /* Local APIC timer works in C2 */ int local_apic_timer_c2_ok __ro_after_init; EXPORT_SYMBOL_GPL(…); /* * Debug level, exported for io_apic.c */ int apic_verbosity __ro_after_init; int pic_mode __ro_after_init; /* Have we found an MP table */ int smp_found_config __ro_after_init; static struct resource lapic_resource = …; unsigned int lapic_timer_period = …; static void apic_pm_activate(void); /* * Get the LAPIC version */ static inline int lapic_get_version(void) { … } /* * Check, if the APIC is integrated or a separate chip */ static inline int lapic_is_integrated(void) { … } /* * Check, whether this is a modern or a first generation APIC */ static int modern_apic(void) { … } /* * right after this call apic become NOOP driven * so apic->write/read doesn't do anything */ static void __init apic_disable(void) { … } void native_apic_icr_write(u32 low, u32 id) { … } u64 native_apic_icr_read(void) { … } /** * lapic_get_maxlvt - get the maximum number of local vector table entries */ int lapic_get_maxlvt(void) { … } /* * Local APIC timer */ /* Clock divisor */ #define APIC_DIVISOR … #define TSC_DIVISOR … /* i82489DX specific */ #define I82489DX_BASE_DIVIDER … /* * This function sets up the local APIC timer, with a timeout of * 'clocks' APIC bus clock. During calibration we actually call * this function twice on the boot CPU, once with a bogus timeout * value, second time for real. The other (noncalibrating) CPUs * call this function only once, with the real, calibrated value. * * We do reads before writes even if unnecessary, to get around the * P5 APIC double write bug. */ static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen) { … } /* * Setup extended LVT, AMD specific * * Software should use the LVT offsets the BIOS provides. The offsets * are determined by the subsystems using it like those for MCE * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts * are supported. Beginning with family 10h at least 4 offsets are * available. * * Since the offsets must be consistent for all cores, we keep track * of the LVT offsets in software and reserve the offset for the same * vector also to be used on other cores. An offset is freed by * setting the entry to APIC_EILVT_MASKED. * * If the BIOS is right, there should be no conflicts. Otherwise a * "[Firmware Bug]: ..." error message is generated. However, if * software does not properly determines the offsets, it is not * necessarily a BIOS bug. */ static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX]; static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new) { … } static unsigned int reserve_eilvt_offset(int offset, unsigned int new) { … } /* * If mask=1, the LVT entry does not generate interrupts while mask=0 * enables the vector. See also the BKDGs. Must be called with * preemption disabled. */ int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask) { … } EXPORT_SYMBOL_GPL(…); /* * Program the next event, relative to now */ static int lapic_next_event(unsigned long delta, struct clock_event_device *evt) { … } static int lapic_next_deadline(unsigned long delta, struct clock_event_device *evt) { … } static int lapic_timer_shutdown(struct clock_event_device *evt) { … } static inline int lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot) { … } static int lapic_timer_set_periodic(struct clock_event_device *evt) { … } static int lapic_timer_set_oneshot(struct clock_event_device *evt) { … } /* * Local APIC timer broadcast function */ static void lapic_timer_broadcast(const struct cpumask *mask) { … } /* * The local apic timer can be used for any function which is CPU local. */ static struct clock_event_device lapic_clockevent = …; static DEFINE_PER_CPU(struct clock_event_device, lapic_events); static const struct x86_cpu_id deadline_match[] __initconst = …; static __init bool apic_validate_deadline_timer(void) { … } /* * Setup the local APIC timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. */ static void setup_APIC_timer(void) { … } /* * Install the updated TSC frequency from recalibration at the TSC * deadline clockevent devices. */ static void __lapic_update_tsc_freq(void *info) { … } void lapic_update_tsc_freq(void) { … } /* * In this functions we calibrate APIC bus clocks to the external timer. * * We want to do the calibration only once since we want to have local timer * irqs synchronous. CPUs connected by the same APIC bus have the very same bus * frequency. * * This was previously done by reading the PIT/HPET and waiting for a wrap * around to find out, that a tick has elapsed. I have a box, where the PIT * readout is broken, so it never gets out of the wait loop again. This was * also reported by others. * * Monitoring the jiffies value is inaccurate and the clockevents * infrastructure allows us to do a simple substitution of the interrupt * handler. * * The calibration routine also uses the pm_timer when possible, as the PIT * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes * back to normal later in the boot process). */ #define LAPIC_CAL_LOOPS … static __initdata int lapic_cal_loops = …; static __initdata long lapic_cal_t1, lapic_cal_t2; static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2; static __initdata u32 lapic_cal_pm1, lapic_cal_pm2; static __initdata unsigned long lapic_cal_j1, lapic_cal_j2; /* * Temporary interrupt handler and polled calibration function. */ static void __init lapic_cal_handler(struct clock_event_device *dev) { … } static int __init calibrate_by_pmtimer(u32 deltapm, long *delta, long *deltatsc) { … } static int __init lapic_init_clockevent(void) { … } bool __init apic_needs_pit(void) { … } static int __init calibrate_APIC_clock(void) { … } /* * Setup the boot APIC * * Calibrate and verify the result. */ void __init setup_boot_APIC_clock(void) { … } void setup_secondary_APIC_clock(void) { … } /* * The guts of the apic timer interrupt */ static void local_apic_timer_interrupt(void) { … } /* * Local APIC timer interrupt. This is the most natural way for doing * local interrupts, but local timer interrupts can be emulated by * broadcast interrupts too. [in case the hw doesn't support APIC timers] * * [ if a single-CPU system runs an SMP kernel then we call the local * interrupt as well. Thus we cannot inline the local irq ... ] */ DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) { … } /* * Local APIC start and shutdown */ /** * clear_local_APIC - shutdown the local APIC * * This is called, when a CPU is disabled and before rebooting, so the state of * the local APIC has no dangling leftovers. Also used to cleanout any BIOS * leftovers during boot. */ void clear_local_APIC(void) { … } /** * apic_soft_disable - Clears and software disables the local APIC on hotplug * * Contrary to disable_local_APIC() this does not touch the enable bit in * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC * bus would require a hardware reset as the APIC would lose track of bus * arbitration. On systems with FSB delivery APICBASE could be disabled, * but it has to be guaranteed that no interrupt is sent to the APIC while * in that state and it's not clear from the SDM whether it still responds * to INIT/SIPI messages. Stay on the safe side and use software disable. */ void apic_soft_disable(void) { … } /** * disable_local_APIC - clear and disable the local APIC */ void disable_local_APIC(void) { … } /* * If Linux enabled the LAPIC against the BIOS default disable it down before * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and * not power-off. Additionally clear all LVT entries before disable_local_APIC * for the case where Linux didn't enable the LAPIC. */ void lapic_shutdown(void) { … } /** * sync_Arb_IDs - synchronize APIC bus arbitration IDs */ void __init sync_Arb_IDs(void) { … } enum apic_intr_mode_id apic_intr_mode __ro_after_init; static int __init __apic_intr_mode_select(void) { … } /* Select the interrupt delivery mode for the BSP */ void __init apic_intr_mode_select(void) { … } /* * An initial setup of the virtual wire mode. */ void __init init_bsp_APIC(void) { … } static void __init apic_bsp_setup(bool upmode); /* Init the interrupt delivery mode for the BSP */ void __init apic_intr_mode_init(void) { … } static void lapic_setup_esr(void) { … } #define APIC_IR_REGS … #define APIC_IR_BITS … #define APIC_IR_MAPSIZE … apic_ir; static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) { … } /* * After a crash, we no longer service the interrupts and a pending * interrupt from previous kernel might still have ISR bit set. * * Most probably by now the CPU has serviced that pending interrupt and it * might not have done the apic_eoi() because it thought, interrupt * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear * the ISR bit and cpu thinks it has already serviced the interrupt. Hence * a vector might get locked. It was noticed for timer irq (vector * 0x31). Issue an extra EOI to clear ISR. * * If there are pending IRR bits they turn into ISR bits after a higher * priority ISR bit has been acked. */ static void apic_pending_intr_clear(void) { … } /** * setup_local_APIC - setup the local APIC * * Used to setup local APIC while initializing BSP or bringing up APs. * Always called with preemption disabled. */ static void setup_local_APIC(void) { … } static void end_local_APIC_setup(void) { … } /* * APIC setup function for application processors. Called from smpboot.c */ void apic_ap_setup(void) { … } static __init void apic_read_boot_cpu_id(bool x2apic) { … } #ifdef CONFIG_X86_X2APIC int x2apic_mode; EXPORT_SYMBOL_GPL(…); enum { … }; static int x2apic_state; static bool x2apic_hw_locked(void) { … } static void __x2apic_disable(void) { … } static void __x2apic_enable(void) { … } static int __init setup_nox2apic(char *str) { … } early_param(…); /* Called from cpu_init() to enable x2apic on (secondary) cpus */ void x2apic_setup(void) { … } static __init void apic_set_fixmap(bool read_apic); static __init void x2apic_disable(void) { … } static __init void x2apic_enable(void) { … } static __init void try_to_enable_x2apic(int remap_mode) { … } void __init check_x2apic(void) { … } #else /* CONFIG_X86_X2APIC */ void __init check_x2apic(void) { if (!apic_is_x2apic_enabled()) return; /* * Checkme: Can we simply turn off x2APIC here instead of disabling the APIC? */ pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n"); pr_err("Disabling APIC, expect reduced performance and functionality.\n"); apic_is_disabled = true; setup_clear_cpu_cap(X86_FEATURE_APIC); } static inline void try_to_enable_x2apic(int remap_mode) { } static inline void __x2apic_enable(void) { } #endif /* !CONFIG_X86_X2APIC */ void __init enable_IR_x2apic(void) { … } #ifdef CONFIG_X86_64 /* * Detect and enable local APICs on non-SMP boards. * Original code written by Keir Fraser. * On AMD64 we trust the BIOS - if it says no APIC it is likely * not correctly set up (usually the APIC timer won't work etc.) */ static bool __init detect_init_APIC(void) { … } #else static bool __init apic_verify(unsigned long addr) { u32 features, h, l; /* * The APIC feature bit should now be enabled * in `cpuid' */ features = cpuid_edx(1); if (!(features & (1 << X86_FEATURE_APIC))) { pr_warn("Could not enable APIC!\n"); return false; } set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); /* The BIOS may have set up the APIC at some other address */ if (boot_cpu_data.x86 >= 6) { rdmsr(MSR_IA32_APICBASE, l, h); if (l & MSR_IA32_APICBASE_ENABLE) addr = l & MSR_IA32_APICBASE_BASE; } register_lapic_address(addr); pr_info("Found and enabled local APIC!\n"); return true; } bool __init apic_force_enable(unsigned long addr) { u32 h, l; if (apic_is_disabled) return false; /* * Some BIOSes disable the local APIC in the APIC_BASE * MSR. This can only be done in software for Intel P6 or later * and AMD K7 (Model > 1) or later. */ if (boot_cpu_data.x86 >= 6) { rdmsr(MSR_IA32_APICBASE, l, h); if (!(l & MSR_IA32_APICBASE_ENABLE)) { pr_info("Local APIC disabled by BIOS -- reenabling.\n"); l &= ~MSR_IA32_APICBASE_BASE; l |= MSR_IA32_APICBASE_ENABLE | addr; wrmsr(MSR_IA32_APICBASE, l, h); enabled_via_apicbase = 1; } } return apic_verify(addr); } /* * Detect and initialize APIC */ static bool __init detect_init_APIC(void) { /* Disabled by kernel option? */ if (apic_is_disabled) return false; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_AMD: if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) || (boot_cpu_data.x86 >= 15)) break; goto no_apic; case X86_VENDOR_HYGON: break; case X86_VENDOR_INTEL: if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC))) break; goto no_apic; default: goto no_apic; } if (!boot_cpu_has(X86_FEATURE_APIC)) { /* * Over-ride BIOS and try to enable the local APIC only if * "lapic" specified. */ if (!force_enable_local_apic) { pr_info("Local APIC disabled by BIOS -- " "you can enable it with \"lapic\"\n"); return false; } if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE)) return false; } else { if (!apic_verify(APIC_DEFAULT_PHYS_BASE)) return false; } apic_pm_activate(); return true; no_apic: pr_info("No local APIC present or hardware disabled\n"); return false; } #endif /** * init_apic_mappings - initialize APIC mappings */ void __init init_apic_mappings(void) { … } static __init void apic_set_fixmap(bool read_apic) { … } void __init register_lapic_address(unsigned long address) { … } /* * Local APIC interrupts */ /* * Common handling code for spurious_interrupt and spurious_vector entry * points below. No point in allowing the compiler to inline it twice. */ static noinline void handle_spurious_interrupt(u8 vector) { … } /** * spurious_interrupt - Catch all for interrupts raised on unused vectors * @regs: Pointer to pt_regs on stack * @vector: The vector number * * This is invoked from ASM entry code to catch all interrupts which * trigger on an entry which is routed to the common_spurious idtentry * point. */ DEFINE_IDTENTRY_IRQ(spurious_interrupt) { … } DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) { … } /* * This interrupt should never happen with our APIC/SMP architecture */ DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) { … } /** * connect_bsp_APIC - attach the APIC to the interrupt system */ static void __init connect_bsp_APIC(void) { … } /** * disconnect_bsp_APIC - detach the APIC from the interrupt system * @virt_wire_setup: indicates, whether virtual wire mode is selected * * Virtual wire mode is necessary to deliver legacy interrupts even when the * APIC is disabled. */ void disconnect_bsp_APIC(int virt_wire_setup) { … } void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, bool dmar) { … } u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) { … } EXPORT_SYMBOL_GPL(…); static void __init apic_bsp_up_setup(void) { … } /** * apic_bsp_setup - Setup function for local apic and io-apic * @upmode: Force UP mode (for APIC_init_uniprocessor) */ static void __init apic_bsp_setup(bool upmode) { … } #ifdef CONFIG_UP_LATE_INIT void __init up_late_init(void) { if (apic_intr_mode == APIC_PIC) return; /* Setup local timer */ x86_init.timers.setup_percpu_clockev(); } #endif /* * Power management */ #ifdef CONFIG_PM static struct { … } apic_pm_state; static int lapic_suspend(void) { … } static void lapic_resume(void) { … } /* * This device has no shutdown method - fully functioning local APICs * are needed on every CPU up until machine_halt/restart/poweroff. */ static struct syscore_ops lapic_syscore_ops = …; static void apic_pm_activate(void) { … } static int __init init_lapic_sysfs(void) { … } /* local apic needs to resume before other devices access its registers. */ core_initcall(init_lapic_sysfs); #else /* CONFIG_PM */ static void apic_pm_activate(void) { } #endif /* CONFIG_PM */ #ifdef CONFIG_X86_64 static int multi_checked; static int multi; static int set_multi(const struct dmi_system_id *d) { … } static const struct dmi_system_id multi_dmi_table[] = …; static void dmi_check_multi(void) { … } /* * apic_is_clustered_box() -- Check if we can expect good TSC * * Thus far, the major user of this is IBM's Summit2 series: * Clustered boxes may have unsynced TSC problems if they are * multi-chassis. * Use DMI to check them */ int apic_is_clustered_box(void) { … } #endif /* * APIC command line parameters */ static int __init setup_disableapic(char *arg) { … } early_param(…); /* same as disableapic, for compatibility */ static int __init setup_nolapic(char *arg) { … } early_param(…); static int __init parse_lapic_timer_c2_ok(char *arg) { … } early_param(…); static int __init parse_disable_apic_timer(char *arg) { … } early_param(…); static int __init parse_nolapic_timer(char *arg) { … } early_param(…); static int __init apic_set_verbosity(char *arg) { … } early_param(…); static int __init lapic_insert_resource(void) { … } /* * need call insert after e820__reserve_resources() * that is using request_resource */ late_initcall(lapic_insert_resource); static int __init apic_set_extnmi(char *arg) { … } early_param(…);
Codebase Browser
linux