linux/arch/x86/entry/common.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * common.c - C code for kernel entry and exit
 * Copyright (c) 2015 Andrew Lutomirski
 *
 * Based on asm and ptrace code by many authors.  The code here originated
 * in ptrace.c and signal.c.
 */

#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/entry-common.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/export.h>
#include <linux/nospec.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/init.h>

#ifdef CONFIG_XEN_PV
#include <xen/xen-ops.h>
#include <xen/events.h>
#endif

#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/traps.h>
#include <asm/vdso.h>
#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
#include <asm/nospec-branch.h>
#include <asm/io_bitmap.h>
#include <asm/syscall.h>
#include <asm/irq_stack.h>

#ifdef CONFIG_X86_64

static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
{ … }

static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
{ … }

/* Returns true to return using SYSRET, or false to use IRET */
__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
{ … }
#endif

#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
static __always_inline int syscall_32_enter(struct pt_regs *regs)
{ … }

#ifdef CONFIG_IA32_EMULATION
bool __ia32_enabled __ro_after_init = !IS_ENABLED(…);

static int ia32_emulation_override_cmdline(char *arg)
{ … }
early_param(…);
#endif

/*
 * Invoke a 32-bit syscall.  Called with IRQs on in CT_STATE_KERNEL.
 */
static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
{ … }

#ifdef CONFIG_IA32_EMULATION
static __always_inline bool int80_is_external(void)
{ … }

/**
 * do_int80_emulation - 32-bit legacy syscall C entry from asm
 *
 * This entry point can be used by 32-bit and 64-bit programs to perform
 * 32-bit system calls.  Instances of INT $0x80 can be found inline in
 * various programs and libraries.  It is also used by the vDSO's
 * __kernel_vsyscall fallback for hardware that doesn't support a faster
 * entry method.  Restarted 32-bit system calls also fall back to INT
 * $0x80 regardless of what instruction was originally used to do the
 * system call.
 *
 * This is considered a slow path.  It is not used by most libc
 * implementations on modern hardware except during process startup.
 *
 * The arguments for the INT $0x80 based syscall are on stack in the
 * pt_regs structure:
 *   eax:				system call number
 *   ebx, ecx, edx, esi, edi, ebp:	arg1 - arg 6
 */
__visible noinstr void do_int80_emulation(struct pt_regs *regs)
{ … }

#ifdef CONFIG_X86_FRED
/*
 * A FRED-specific INT80 handler is warranted for the follwing reasons:
 *
 * 1) As INT instructions and hardware interrupts are separate event
 *    types, FRED does not preclude the use of vector 0x80 for external
 *    interrupts. As a result, the FRED setup code does not reserve
 *    vector 0x80 and calling int80_is_external() is not merely
 *    suboptimal but actively incorrect: it could cause a system call
 *    to be incorrectly ignored.
 *
 * 2) It is called only for handling vector 0x80 of event type
 *    EVENT_TYPE_SWINT and will never be called to handle any external
 *    interrupt (event type EVENT_TYPE_EXTINT).
 *
 * 3) FRED has separate entry flows depending on if the event came from
 *    user space or kernel space, and because the kernel does not use
 *    INT insns, the FRED kernel entry handler fred_entry_from_kernel()
 *    falls through to fred_bad_type() if the event type is
 *    EVENT_TYPE_SWINT, i.e., INT insns. So if the kernel is handling
 *    an INT insn, it can only be from a user level.
 *
 * 4) int80_emulation() does a CLEAR_BRANCH_HISTORY. While FRED will
 *    likely take a different approach if it is ever needed: it
 *    probably belongs in either fred_intx()/ fred_other() or
 *    asm_fred_entrypoint_user(), depending on if this ought to be done
 *    for all entries from userspace or only system
 *    calls.
 *
 * 5) INT $0x80 is the fast path for 32-bit system calls under FRED.
 */
DEFINE_FREDENTRY_RAW(int80_emulation)
{ … }
#endif
#else /* CONFIG_IA32_EMULATION */

/* Handles int $0x80 on a 32bit kernel */
__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
{
	int nr = syscall_32_enter(regs);

	add_random_kstack_offset();
	/*
	 * Subtlety here: if ptrace pokes something larger than 2^31-1 into
	 * orig_ax, the int return value truncates it. This matches
	 * the semantics of syscall_get_nr().
	 */
	nr = syscall_enter_from_user_mode(regs, nr);
	instrumentation_begin();

	do_syscall_32_irqs_on(regs, nr);

	instrumentation_end();
	syscall_exit_to_user_mode(regs);
}
#endif /* !CONFIG_IA32_EMULATION */

static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
{ … }

/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
{ … }

/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
{ … }
#endif

SYSCALL_DEFINE0(…)
{ …

#ifdef CONFIG_XEN_PV
#ifndef CONFIG_PREEMPTION
/*
 * Some hypercalls issued by the toolstack can take many 10s of
 * seconds. Allow tasks running hypercalls via the privcmd driver to
 * be voluntarily preempted even if full kernel preemption is
 * disabled.
 *
 * Such preemptible hypercalls are bracketed by
 * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end()
 * calls.
 */
DEFINE_PER_CPU(bool, xen_in_preemptible_hcall);
EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall);

/*
 * In case of scheduling the flag must be cleared and restored after
 * returning from schedule as the task might move to a different CPU.
 */
static __always_inline bool get_and_clear_inhcall(void)
{
	bool inhcall = __this_cpu_read(xen_in_preemptible_hcall);

	__this_cpu_write(xen_in_preemptible_hcall, false);
	return inhcall;
}

static __always_inline void restore_inhcall(bool inhcall)
{
	__this_cpu_write(xen_in_preemptible_hcall, inhcall);
}
#else
static __always_inline bool get_and_clear_inhcall(void) { … }
static __always_inline void restore_inhcall(bool inhcall) { … }
#endif

static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs)
{ … }

__visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
{ … }
#endif /* CONFIG_XEN_PV */