/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_PGTABLE_DEFS_H #define _ASM_X86_PGTABLE_DEFS_H #include <linux/const.h> #include <linux/mem_encrypt.h> #include <asm/page_types.h> #define _PAGE_BIT_PRESENT … #define _PAGE_BIT_RW … #define _PAGE_BIT_USER … #define _PAGE_BIT_PWT … #define _PAGE_BIT_PCD … #define _PAGE_BIT_ACCESSED … #define _PAGE_BIT_DIRTY … #define _PAGE_BIT_PSE … #define _PAGE_BIT_PAT … #define _PAGE_BIT_GLOBAL … #define _PAGE_BIT_SOFTW1 … #define _PAGE_BIT_SOFTW2 … #define _PAGE_BIT_SOFTW3 … #define _PAGE_BIT_PAT_LARGE … #define _PAGE_BIT_SOFTW4 … #define _PAGE_BIT_SOFTW5 … #define _PAGE_BIT_PKEY_BIT0 … #define _PAGE_BIT_PKEY_BIT1 … #define _PAGE_BIT_PKEY_BIT2 … #define _PAGE_BIT_PKEY_BIT3 … #define _PAGE_BIT_NX … #define _PAGE_BIT_SPECIAL … #define _PAGE_BIT_CPA_TEST … #define _PAGE_BIT_UFFD_WP … #define _PAGE_BIT_SOFT_DIRTY … #define _PAGE_BIT_DEVMAP … #ifdef CONFIG_X86_64 #define _PAGE_BIT_SAVED_DIRTY … #else /* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ #define _PAGE_BIT_SAVED_DIRTY … #endif /* If _PAGE_BIT_PRESENT is clear, we use these: */ /* - if the user mapped it with PROT_NONE; pte_present gives true */ #define _PAGE_BIT_PROTNONE … #define _PAGE_PRESENT … #define _PAGE_RW … #define _PAGE_USER … #define _PAGE_PWT … #define _PAGE_PCD … #define _PAGE_ACCESSED … #define _PAGE_DIRTY … #define _PAGE_PSE … #define _PAGE_GLOBAL … #define _PAGE_SOFTW1 … #define _PAGE_SOFTW2 … #define _PAGE_SOFTW3 … #define _PAGE_PAT … #define _PAGE_PAT_LARGE … #define _PAGE_SPECIAL … #define _PAGE_CPA_TEST … #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS #define _PAGE_PKEY_BIT0 … #define _PAGE_PKEY_BIT1 … #define _PAGE_PKEY_BIT2 … #define _PAGE_PKEY_BIT3 … #else #define _PAGE_PKEY_BIT0 … #define _PAGE_PKEY_BIT1 … #define _PAGE_PKEY_BIT2 … #define _PAGE_PKEY_BIT3 … #endif #define _PAGE_PKEY_MASK … #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) #define _PAGE_KNL_ERRATUM_MASK … #else #define _PAGE_KNL_ERRATUM_MASK … #endif #ifdef CONFIG_MEM_SOFT_DIRTY #define _PAGE_SOFT_DIRTY … #else #define _PAGE_SOFT_DIRTY … #endif /* * Tracking soft dirty bit when a page goes to a swap is tricky. * We need a bit which can be stored in pte _and_ not conflict * with swap entry format. On x86 bits 1-4 are *not* involved * into swap entry computation, but bit 7 is used for thp migration, * so we borrow bit 1 for soft dirty tracking. * * Please note that this bit must be treated as swap dirty page * mark if and only if the PTE/PMD has present bit clear! */ #ifdef CONFIG_MEM_SOFT_DIRTY #define _PAGE_SWP_SOFT_DIRTY … #else #define _PAGE_SWP_SOFT_DIRTY … #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define _PAGE_UFFD_WP … #define _PAGE_SWP_UFFD_WP … #else #define _PAGE_UFFD_WP … #define _PAGE_SWP_UFFD_WP … #endif #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) #define _PAGE_NX … #define _PAGE_DEVMAP … #define _PAGE_SOFTW4 … #else #define _PAGE_NX … #define _PAGE_DEVMAP … #define _PAGE_SOFTW4 … #endif /* * The hardware requires shadow stack to be Write=0,Dirty=1. However, * there are valid cases where the kernel might create read-only PTEs that * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have * (Write=0,SavedDirty=1,Dirty=0) set. */ #define _PAGE_SAVED_DIRTY … #define _PAGE_DIRTY_BITS … #define _PAGE_PROTNONE … /* * Set of bits not changed in pte_modify. The pte's * protection key is treated like _PAGE_RW, for * instance, and is *not* included in this mask since * pte_modify() does modify it. */ #define _COMMON_PAGE_CHG_MASK … #define _PAGE_CHG_MASK … #define _HPAGE_CHG_MASK … /* * The cache modes defined here are used to translate between pure SW usage * and the HW defined cache mode bits and/or PAT entries. * * The resulting bits for PWT, PCD and PAT should be chosen in a way * to have the WB mode at index 0 (all bits clear). This is the default * right now and likely would break too much if changed. */ #ifndef __ASSEMBLY__ enum page_cache_mode { _PAGE_CACHE_MODE_WB = 0, _PAGE_CACHE_MODE_WC = 1, _PAGE_CACHE_MODE_UC_MINUS = 2, _PAGE_CACHE_MODE_UC = 3, _PAGE_CACHE_MODE_WT = 4, _PAGE_CACHE_MODE_WP = 5, _PAGE_CACHE_MODE_NUM = 8 }; #endif #define _PAGE_CC … #define _PAGE_ENC … #define _PAGE_CACHE_MASK … #define _PAGE_LARGE_CACHE_MASK … #define _PAGE_NOCACHE … #define _PAGE_CACHE_WP … #define __PP … #define __RW … #define _USR … #define ___A … #define ___D … #define ___G … #define __NX … #define _ENC … #define __WP … #define __NC … #define _PSE … #define pgprot_val(x) … #define __pgprot(x) … #define __pg(x) … #define PAGE_NONE … #define PAGE_SHARED … #define PAGE_SHARED_EXEC … #define PAGE_COPY_NOEXEC … #define PAGE_COPY_EXEC … #define PAGE_COPY … #define PAGE_READONLY … #define PAGE_READONLY_EXEC … #define __PAGE_KERNEL … #define __PAGE_KERNEL_EXEC … /* * Page tables needs to have Write=1 in order for any lower PTEs to be * writable. This includes shadow stack memory (Write=0, Dirty=1) */ #define _KERNPG_TABLE_NOENC … #define _KERNPG_TABLE … #define _PAGE_TABLE_NOENC … #define _PAGE_TABLE … #define __PAGE_KERNEL_RO … #define __PAGE_KERNEL_ROX … #define __PAGE_KERNEL … #define __PAGE_KERNEL_EXEC … #define __PAGE_KERNEL_NOCACHE … #define __PAGE_KERNEL_VVAR … #define __PAGE_KERNEL_LARGE … #define __PAGE_KERNEL_LARGE_EXEC … #define __PAGE_KERNEL_WP … #define __PAGE_KERNEL_IO … #define __PAGE_KERNEL_IO_NOCACHE … #ifndef __ASSEMBLY__ #define __PAGE_KERNEL_ENC … #define __PAGE_KERNEL_ENC_WP … #define __PAGE_KERNEL_NOENC … #define __PAGE_KERNEL_NOENC_WP … #define __pgprot_mask … #define PAGE_KERNEL … #define PAGE_KERNEL_NOENC … #define PAGE_KERNEL_RO … #define PAGE_KERNEL_EXEC … #define PAGE_KERNEL_EXEC_NOENC … #define PAGE_KERNEL_ROX … #define PAGE_KERNEL_NOCACHE … #define PAGE_KERNEL_LARGE … #define PAGE_KERNEL_LARGE_EXEC … #define PAGE_KERNEL_VVAR … #define PAGE_KERNEL_IO … #define PAGE_KERNEL_IO_NOCACHE … #endif /* __ASSEMBLY__ */ /* * early identity mapping pte attrib macros. */ #ifdef CONFIG_X86_64 #define __PAGE_KERNEL_IDENT_LARGE_EXEC … #else #define PTE_IDENT_ATTR … #define PDE_IDENT_ATTR … #define PGD_IDENT_ATTR … #endif #ifdef CONFIG_X86_32 # include <asm/pgtable_32_types.h> #else # include <asm/pgtable_64_types.h> #endif #ifndef __ASSEMBLY__ #include <linux/types.h> /* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */ #define PTE_PFN_MASK … /* * Extracts the flags from a (pte|pmd|pud|pgd)val_t * This includes the protection key value. */ #define PTE_FLAGS_MASK … typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; typedef struct { pgdval_t pgd; } pgd_t; static inline pgprot_t pgprot_nx(pgprot_t prot) { return __pgprot(pgprot_val(prot) | _PAGE_NX); } #define pgprot_nx … #ifdef CONFIG_X86_PAE /* * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't * use it here. */ #define PGD_PAE_PAGE_MASK … #define PGD_PAE_PHYS_MASK … /* * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries. * All other bits are Reserved MBZ */ #define PGD_ALLOWED_BITS … #else /* No need to mask any bits for !PAE */ #define PGD_ALLOWED_BITS … #endif static inline pgd_t native_make_pgd(pgdval_t val) { return (pgd_t) { val & PGD_ALLOWED_BITS }; } static inline pgdval_t native_pgd_val(pgd_t pgd) { return pgd.pgd & PGD_ALLOWED_BITS; } static inline pgdval_t pgd_flags(pgd_t pgd) { return native_pgd_val(pgd) & PTE_FLAGS_MASK; } #if CONFIG_PGTABLE_LEVELS > 4 typedef struct { p4dval_t p4d; } p4d_t; static inline p4d_t native_make_p4d(pudval_t val) { return (p4d_t) { val }; } static inline p4dval_t native_p4d_val(p4d_t p4d) { return p4d.p4d; } #else #include <asm-generic/pgtable-nop4d.h> static inline p4d_t native_make_p4d(pudval_t val) { return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) }; } static inline p4dval_t native_p4d_val(p4d_t p4d) { return native_pgd_val(p4d.pgd); } #endif #if CONFIG_PGTABLE_LEVELS > 3 typedef struct { pudval_t pud; } pud_t; static inline pud_t native_make_pud(pmdval_t val) { return (pud_t) { val }; } static inline pudval_t native_pud_val(pud_t pud) { return pud.pud; } #else #include <asm-generic/pgtable-nopud.h> static inline pud_t native_make_pud(pudval_t val) { return (pud_t) { .p4d.pgd = native_make_pgd(val) }; } static inline pudval_t native_pud_val(pud_t pud) { return native_pgd_val(pud.p4d.pgd); } #endif #if CONFIG_PGTABLE_LEVELS > 2 static inline pmd_t native_make_pmd(pmdval_t val) { return (pmd_t) { .pmd = val }; } static inline pmdval_t native_pmd_val(pmd_t pmd) { return pmd.pmd; } #else #include <asm-generic/pgtable-nopmd.h> static inline pmd_t native_make_pmd(pmdval_t val) { return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; } static inline pmdval_t native_pmd_val(pmd_t pmd) { return native_pgd_val(pmd.pud.p4d.pgd); } #endif static inline p4dval_t p4d_pfn_mask(p4d_t p4d) { /* No 512 GiB huge pages yet */ return PTE_PFN_MASK; } static inline p4dval_t p4d_flags_mask(p4d_t p4d) { return ~p4d_pfn_mask(p4d); } static inline p4dval_t p4d_flags(p4d_t p4d) { return native_p4d_val(p4d) & p4d_flags_mask(p4d); } static inline pudval_t pud_pfn_mask(pud_t pud) { if (native_pud_val(pud) & _PAGE_PSE) return PHYSICAL_PUD_PAGE_MASK; else return PTE_PFN_MASK; } static inline pudval_t pud_flags_mask(pud_t pud) { return ~pud_pfn_mask(pud); } static inline pudval_t pud_flags(pud_t pud) { return native_pud_val(pud) & pud_flags_mask(pud); } static inline pmdval_t pmd_pfn_mask(pmd_t pmd) { if (native_pmd_val(pmd) & _PAGE_PSE) return PHYSICAL_PMD_PAGE_MASK; else return PTE_PFN_MASK; } static inline pmdval_t pmd_flags_mask(pmd_t pmd) { return ~pmd_pfn_mask(pmd); } static inline pmdval_t pmd_flags(pmd_t pmd) { return native_pmd_val(pmd) & pmd_flags_mask(pmd); } static inline pte_t native_make_pte(pteval_t val) { return (pte_t) { .pte = val }; } static inline pteval_t native_pte_val(pte_t pte) { return pte.pte; } static inline pteval_t pte_flags(pte_t pte) { return native_pte_val(pte) & PTE_FLAGS_MASK; } #define __pte2cm_idx … #define __cm_idx2pte … unsigned long cachemode2protval(enum page_cache_mode pcm); static inline pgprotval_t protval_4k_2_large(pgprotval_t val) { return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | ((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); } static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot) { return __pgprot(protval_4k_2_large(pgprot_val(pgprot))); } static inline pgprotval_t protval_large_2_4k(pgprotval_t val) { return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | ((val & _PAGE_PAT_LARGE) >> (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); } static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) { return __pgprot(protval_large_2_4k(pgprot_val(pgprot))); } typedef struct page *pgtable_t; extern pteval_t __supported_pte_mask; extern pteval_t __default_kernel_pte_mask; #define pgprot_writecombine … extern pgprot_t pgprot_writecombine(pgprot_t prot); #define pgprot_writethrough … extern pgprot_t pgprot_writethrough(pgprot_t prot); /* Indicate that x86 has its own track and untrack pfn vma functions */ #define __HAVE_PFNMAP_TRACKING #define __HAVE_PHYS_MEM_ACCESS_PROT struct file; pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot); /* Install a pte for a particular vaddr in kernel space. */ void set_pte_vaddr(unsigned long vaddr, pte_t pte); #ifdef CONFIG_X86_32 extern void native_pagetable_init(void); #else #define native_pagetable_init … #endif enum pg_level { PG_LEVEL_NONE, PG_LEVEL_4K, PG_LEVEL_2M, PG_LEVEL_1G, PG_LEVEL_512G, PG_LEVEL_256T, PG_LEVEL_NUM }; #ifdef CONFIG_PROC_FS extern void update_page_count(int level, unsigned long pages); #else static inline void update_page_count(int level, unsigned long pages) { } #endif /* * Helper function that returns the kernel pagetable entry controlling * the virtual address 'address'. NULL means no pagetable entry present. * NOTE: the return type is pte_t but if the pmd is PSE then we return it * as a pte too. */ extern pte_t *lookup_address(unsigned long address, unsigned int *level); extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address, unsigned int *level); pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address, unsigned int *level, bool *nx, bool *rw); extern pmd_t *lookup_pmd_address(unsigned long address); extern phys_addr_t slow_virt_to_phys(void *__address); extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address, unsigned numpages, unsigned long page_flags); extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address, unsigned long numpages); #endif /* !__ASSEMBLY__ */ #endif /* _ASM_X86_PGTABLE_DEFS_H */
Codebase Browser
linux