linux/arch/x86/mm/pti.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright(c) 2017 Intel Corporation. All rights reserved.
 *
 * This code is based in part on work published here:
 *
 *	https://github.com/IAIK/KAISER
 *
 * The original work was written by and signed off by for the Linux
 * kernel by:
 *
 *   Signed-off-by: Richard Fellner <[email protected]>
 *   Signed-off-by: Moritz Lipp <[email protected]>
 *   Signed-off-by: Daniel Gruss <[email protected]>
 *   Signed-off-by: Michael Schwarz <[email protected]>
 *
 * Major changes to the original code by: Dave Hansen <[email protected]>
 * Mostly rewritten by Thomas Gleixner <[email protected]> and
 *		       Andy Lutomirsky <[email protected]>
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/bug.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/cpu.h>

#include <asm/cpufeature.h>
#include <asm/hypervisor.h>
#include <asm/vsyscall.h>
#include <asm/cmdline.h>
#include <asm/pti.h>
#include <asm/tlbflush.h>
#include <asm/desc.h>
#include <asm/sections.h>
#include <asm/set_memory.h>

#undef pr_fmt
#define pr_fmt(fmt) …

/* Backporting helper */
#ifndef __GFP_NOTRACK
#define __GFP_NOTRACK …
#endif

/*
 * Define the page-table levels we clone for user-space on 32
 * and 64 bit.
 */
#ifdef CONFIG_X86_64
#define PTI_LEVEL_KERNEL_IMAGE …
#else
#define PTI_LEVEL_KERNEL_IMAGE …
#endif

static void __init pti_print_if_insecure(const char *reason)
{ … }

static void __init pti_print_if_secure(const char *reason)
{ … }

/* Assume mode is auto unless overridden via cmdline below. */
static enum pti_mode { … } pti_mode;

void __init pti_check_boottime_disable(void)
{ … }

static int __init pti_parse_cmdline(char *arg)
{ … }
early_param(…);

static int __init pti_parse_cmdline_nopti(char *arg)
{ … }
early_param(…);

pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
{ … }

/*
 * Walk the user copy of the page tables (optionally) trying to allocate
 * page table pages on the way down.
 *
 * Returns a pointer to a P4D on success, or NULL on failure.
 */
static p4d_t *pti_user_pagetable_walk_p4d(unsigned long address)
{ … }

/*
 * Walk the user copy of the page tables (optionally) trying to allocate
 * page table pages on the way down.
 *
 * Returns a pointer to a PMD on success, or NULL on failure.
 */
static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address)
{ … }

/*
 * Walk the shadow copy of the page tables (optionally) trying to allocate
 * page table pages on the way down.  Does not support large pages.
 *
 * Note: this is only used when mapping *new* kernel data into the
 * user/shadow page tables.  It is never used for userspace data.
 *
 * Returns a pointer to a PTE on success, or NULL on failure.
 */
static pte_t *pti_user_pagetable_walk_pte(unsigned long address)
{ … }

#ifdef CONFIG_X86_VSYSCALL_EMULATION
static void __init pti_setup_vsyscall(void)
{ … }
#else
static void __init pti_setup_vsyscall(void) { }
#endif

enum pti_clone_level { … };

static void
pti_clone_pgtable(unsigned long start, unsigned long end,
		  enum pti_clone_level level)
{ … }

#ifdef CONFIG_X86_64
/*
 * Clone a single p4d (i.e. a top-level entry on 4-level systems and a
 * next-level entry on 5-level systems.
 */
static void __init pti_clone_p4d(unsigned long addr)
{ … }

/*
 * Clone the CPU_ENTRY_AREA and associated data into the user space visible
 * page table.
 */
static void __init pti_clone_user_shared(void)
{ … }

#else /* CONFIG_X86_64 */

/*
 * On 32 bit PAE systems with 1GB of Kernel address space there is only
 * one pgd/p4d for the whole kernel. Cloning that would map the whole
 * address space into the user page-tables, making PTI useless. So clone
 * the page-table on the PMD level to prevent that.
 */
static void __init pti_clone_user_shared(void)
{
	unsigned long start, end;

	start = CPU_ENTRY_AREA_BASE;
	end   = start + (PAGE_SIZE * CPU_ENTRY_AREA_PAGES);

	pti_clone_pgtable(start, end, PTI_CLONE_PMD);
}
#endif /* CONFIG_X86_64 */

/*
 * Clone the ESPFIX P4D into the user space visible page table
 */
static void __init pti_setup_espfix64(void)
{ … }

/*
 * Clone the populated PMDs of the entry text and force it RO.
 */
static void pti_clone_entry_text(void)
{ … }

/*
 * Global pages and PCIDs are both ways to make kernel TLB entries
 * live longer, reduce TLB misses and improve kernel performance.
 * But, leaving all kernel text Global makes it potentially accessible
 * to Meltdown-style attacks which make it trivial to find gadgets or
 * defeat KASLR.
 *
 * Only use global pages when it is really worth it.
 */
static inline bool pti_kernel_image_global_ok(void)
{ … }

/*
 * For some configurations, map all of kernel text into the user page
 * tables.  This reduces TLB misses, especially on non-PCID systems.
 */
static void pti_clone_kernel_text(void)
{ … }

static void pti_set_kernel_image_nonglobal(void)
{ … }

/*
 * Initialize kernel page table isolation
 */
void __init pti_init(void)
{ … }

/*
 * Finalize the kernel mappings in the userspace page-table. Some of the
 * mappings for the kernel image might have changed since pti_init()
 * cloned them. This is because parts of the kernel image have been
 * mapped RO and/or NX.  These changes need to be cloned again to the
 * userspace page-table.
 */
void pti_finalize(void)
{ … }