/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_PGTABLE_H #define _ASM_X86_PGTABLE_H #include <linux/mem_encrypt.h> #include <asm/page.h> #include <asm/pgtable_types.h> /* * Macro to mark a page protection value as UC- */ #define pgprot_noncached(prot) … #ifndef __ASSEMBLY__ #include <linux/spinlock.h> #include <asm/x86_init.h> #include <asm/pkru.h> #include <asm/fpu/api.h> #include <asm/coco.h> #include <asm-generic/pgtable_uffd.h> #include <linux/page_table_check.h> extern pgd_t early_top_pgt[PTRS_PER_PGD]; bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); struct seq_file; void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm, bool user); bool ptdump_walk_pgd_level_checkwx(void); #define ptdump_check_wx … void ptdump_walk_user_pgd_level_checkwx(void); /* * Macros to add or remove encryption attribute */ #define pgprot_encrypted(prot) … #define pgprot_decrypted(prot) … #ifdef CONFIG_DEBUG_WX #define debug_checkwx_user() … #else #define debug_checkwx_user … #endif /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __visible; #define ZERO_PAGE(vaddr) … extern spinlock_t pgd_lock; extern struct list_head pgd_list; extern struct mm_struct *pgd_page_get_mm(struct page *page); extern pmdval_t early_pmd_flags; #ifdef CONFIG_PARAVIRT_XXL #include <asm/paravirt.h> #else /* !CONFIG_PARAVIRT_XXL */ #define set_pte … #define set_pte_atomic … #define set_pmd … #ifndef __PAGETABLE_P4D_FOLDED #define set_pgd … #define pgd_clear … #endif #ifndef set_p4d #define set_p4d … #endif #ifndef __PAGETABLE_PUD_FOLDED #define p4d_clear … #endif #ifndef set_pud #define set_pud … #endif #ifndef __PAGETABLE_PUD_FOLDED #define pud_clear … #endif #define pte_clear … #define pmd_clear … #define pgd_val … #define __pgd … #ifndef __PAGETABLE_P4D_FOLDED #define p4d_val … #define __p4d … #endif #ifndef __PAGETABLE_PUD_FOLDED #define pud_val … #define __pud … #endif #ifndef __PAGETABLE_PMD_FOLDED #define pmd_val … #define __pmd … #endif #define pte_val … #define __pte … #define arch_end_context_switch … #endif /* CONFIG_PARAVIRT_XXL */ /* * The following only work if pte_present() is true. * Undefined behaviour if not.. */ static inline bool pte_dirty(pte_t pte) { … } static inline bool pte_shstk(pte_t pte) { … } static inline int pte_young(pte_t pte) { … } static inline bool pte_decrypted(pte_t pte) { … } #define pmd_dirty … static inline bool pmd_dirty(pmd_t pmd) { … } static inline bool pmd_shstk(pmd_t pmd) { … } #define pmd_young … static inline int pmd_young(pmd_t pmd) { … } static inline bool pud_dirty(pud_t pud) { … } static inline int pud_young(pud_t pud) { … } static inline int pte_write(pte_t pte) { … } #define pmd_write … static inline int pmd_write(pmd_t pmd) { … } #define pud_write … static inline int pud_write(pud_t pud) { … } static inline int pte_huge(pte_t pte) { … } static inline int pte_global(pte_t pte) { … } static inline int pte_exec(pte_t pte) { … } static inline int pte_special(pte_t pte) { … } /* Entries that were set to PROT_NONE are inverted */ static inline u64 protnone_mask(u64 val); #define PFN_PTE_SHIFT … static inline unsigned long pte_pfn(pte_t pte) { … } static inline unsigned long pmd_pfn(pmd_t pmd) { … } #define pud_pfn … static inline unsigned long pud_pfn(pud_t pud) { … } static inline unsigned long p4d_pfn(p4d_t p4d) { … } static inline unsigned long pgd_pfn(pgd_t pgd) { … } #define p4d_leaf … static inline bool p4d_leaf(p4d_t p4d) { … } #define pte_page(pte) … #define pmd_leaf … static inline bool pmd_leaf(pmd_t pte) { … } #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* NOTE: when predicate huge page, consider also pmd_devmap, or use pmd_leaf */ static inline int pmd_trans_huge(pmd_t pmd) { … } #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD static inline int pud_trans_huge(pud_t pud) { … } #endif #define has_transparent_hugepage … static inline int has_transparent_hugepage(void) { … } #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP static inline int pmd_devmap(pmd_t pmd) { … } #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD static inline int pud_devmap(pud_t pud) { … } #else static inline int pud_devmap(pud_t pud) { return 0; } #endif static inline int pgd_devmap(pgd_t pgd) { … } #endif #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline pte_t pte_set_flags(pte_t pte, pteval_t set) { … } static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) { … } /* * Write protection operations can result in Dirty=1,Write=0 PTEs. But in the * case of X86_FEATURE_USER_SHSTK, these PTEs denote shadow stack memory. So * when creating dirty, write-protected memory, a software bit is used: * _PAGE_BIT_SAVED_DIRTY. The following functions take a PTE and transition the * Dirty bit to SavedDirty, and vice-vesra. * * This shifting is only done if needed. In the case of shifting * Dirty->SavedDirty, the condition is if the PTE is Write=0. In the case of * shifting SavedDirty->Dirty, the condition is Write=1. */ static inline pgprotval_t mksaveddirty_shift(pgprotval_t v) { … } static inline pgprotval_t clear_saveddirty_shift(pgprotval_t v) { … } static inline pte_t pte_mksaveddirty(pte_t pte) { … } static inline pte_t pte_clear_saveddirty(pte_t pte) { … } static inline pte_t pte_wrprotect(pte_t pte) { … } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { … } static inline pte_t pte_mkuffd_wp(pte_t pte) { … } static inline pte_t pte_clear_uffd_wp(pte_t pte) { … } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ static inline pte_t pte_mkclean(pte_t pte) { … } static inline pte_t pte_mkold(pte_t pte) { … } static inline pte_t pte_mkexec(pte_t pte) { … } static inline pte_t pte_mkdirty(pte_t pte) { … } static inline pte_t pte_mkwrite_shstk(pte_t pte) { … } static inline pte_t pte_mkyoung(pte_t pte) { … } static inline pte_t pte_mkwrite_novma(pte_t pte) { … } struct vm_area_struct; pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma); #define pte_mkwrite … static inline pte_t pte_mkhuge(pte_t pte) { … } static inline pte_t pte_clrhuge(pte_t pte) { … } static inline pte_t pte_mkglobal(pte_t pte) { … } static inline pte_t pte_clrglobal(pte_t pte) { … } static inline pte_t pte_mkspecial(pte_t pte) { … } static inline pte_t pte_mkdevmap(pte_t pte) { … } static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set) { … } static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) { … } /* See comments above mksaveddirty_shift() */ static inline pmd_t pmd_mksaveddirty(pmd_t pmd) { … } /* See comments above mksaveddirty_shift() */ static inline pmd_t pmd_clear_saveddirty(pmd_t pmd) { … } static inline pmd_t pmd_wrprotect(pmd_t pmd) { … } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pmd_uffd_wp(pmd_t pmd) { … } static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) { … } static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) { … } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ static inline pmd_t pmd_mkold(pmd_t pmd) { … } static inline pmd_t pmd_mkclean(pmd_t pmd) { … } static inline pmd_t pmd_mkdirty(pmd_t pmd) { … } static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd) { … } static inline pmd_t pmd_mkdevmap(pmd_t pmd) { … } static inline pmd_t pmd_mkhuge(pmd_t pmd) { … } static inline pmd_t pmd_mkyoung(pmd_t pmd) { … } static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) { … } pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); #define pmd_mkwrite … static inline pud_t pud_set_flags(pud_t pud, pudval_t set) { … } static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) { … } /* See comments above mksaveddirty_shift() */ static inline pud_t pud_mksaveddirty(pud_t pud) { … } /* See comments above mksaveddirty_shift() */ static inline pud_t pud_clear_saveddirty(pud_t pud) { … } static inline pud_t pud_mkold(pud_t pud) { … } static inline pud_t pud_mkclean(pud_t pud) { … } static inline pud_t pud_wrprotect(pud_t pud) { … } static inline pud_t pud_mkdirty(pud_t pud) { … } static inline pud_t pud_mkdevmap(pud_t pud) { … } static inline pud_t pud_mkhuge(pud_t pud) { … } static inline pud_t pud_mkyoung(pud_t pud) { … } static inline pud_t pud_mkwrite(pud_t pud) { … } #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline int pte_soft_dirty(pte_t pte) { … } static inline int pmd_soft_dirty(pmd_t pmd) { … } static inline int pud_soft_dirty(pud_t pud) { … } static inline pte_t pte_mksoft_dirty(pte_t pte) { … } static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) { … } static inline pud_t pud_mksoft_dirty(pud_t pud) { … } static inline pte_t pte_clear_soft_dirty(pte_t pte) { … } static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) { … } static inline pud_t pud_clear_soft_dirty(pud_t pud) { … } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ /* * Mask out unsupported bits in a present pgprot. Non-present pgprots * can use those bits for other purposes, so leave them be. */ static inline pgprotval_t massage_pgprot(pgprot_t pgprot) { … } static inline pgprotval_t check_pgprot(pgprot_t pgprot) { … } static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) { … } static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) { … } static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot) { … } static inline pmd_t pmd_mkinvalid(pmd_t pmd) { … } static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { … } static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { … } /* * mprotect needs to preserve PAT and encryption bits when updating * vm_page_prot */ #define pgprot_modify … static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) { … } #define pte_pgprot(x) … #define pmd_pgprot(x) … #define pud_pgprot(x) … #define p4d_pgprot(x) … #define canon_pgprot(p) … static inline int is_new_memtype_allowed(u64 paddr, unsigned long size, enum page_cache_mode pcm, enum page_cache_mode new_pcm) { … } pmd_t *populate_extra_pmd(unsigned long vaddr); pte_t *populate_extra_pte(unsigned long vaddr); #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd); /* * Take a PGD location (pgdp) and a pgd value that needs to be set there. * Populates the user and returns the resulting PGD that must be set in * the kernel copy of the page tables. */ static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) { … } #else /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */ static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) { return pgd; } #endif /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */ #endif /* __ASSEMBLY__ */ #ifdef CONFIG_X86_32 # include <asm/pgtable_32.h> #else # include <asm/pgtable_64.h> #endif #ifndef __ASSEMBLY__ #include <linux/mm_types.h> #include <linux/mmdebug.h> #include <linux/log2.h> #include <asm/fixmap.h> static inline int pte_none(pte_t pte) { … } #define __HAVE_ARCH_PTE_SAME static inline int pte_same(pte_t a, pte_t b) { … } static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) { … } #define pte_advance_pfn … static inline int pte_present(pte_t a) { … } #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP static inline int pte_devmap(pte_t a) { … } #endif #define pte_accessible … static inline bool pte_accessible(struct mm_struct *mm, pte_t a) { … } static inline int pmd_present(pmd_t pmd) { … } #ifdef CONFIG_NUMA_BALANCING /* * These work without NUMA balancing but the kernel does not care. See the * comment in include/linux/pgtable.h */ static inline int pte_protnone(pte_t pte) { … } static inline int pmd_protnone(pmd_t pmd) { … } #endif /* CONFIG_NUMA_BALANCING */ static inline int pmd_none(pmd_t pmd) { … } static inline unsigned long pmd_page_vaddr(pmd_t pmd) { … } /* * Currently stuck as a macro due to indirect forward reference to * linux/mmzone.h's __section_mem_map_addr() definition: */ #define pmd_page(pmd) … /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. * * (Currently stuck as a macro because of indirect forward reference * to linux/mm.h:page_to_nid()) */ #define mk_pte(page, pgprot) … static inline int pmd_bad(pmd_t pmd) { … } static inline unsigned long pages_to_mb(unsigned long npg) { … } #if CONFIG_PGTABLE_LEVELS > 2 static inline int pud_none(pud_t pud) { … } static inline int pud_present(pud_t pud) { … } static inline pmd_t *pud_pgtable(pud_t pud) { … } /* * Currently stuck as a macro due to indirect forward reference to * linux/mmzone.h's __section_mem_map_addr() definition: */ #define pud_page(pud) … #define pud_leaf … static inline bool pud_leaf(pud_t pud) { … } static inline int pud_bad(pud_t pud) { … } #endif /* CONFIG_PGTABLE_LEVELS > 2 */ #if CONFIG_PGTABLE_LEVELS > 3 static inline int p4d_none(p4d_t p4d) { … } static inline int p4d_present(p4d_t p4d) { … } static inline pud_t *p4d_pgtable(p4d_t p4d) { … } /* * Currently stuck as a macro due to indirect forward reference to * linux/mmzone.h's __section_mem_map_addr() definition: */ #define p4d_page(p4d) … static inline int p4d_bad(p4d_t p4d) { … } #endif /* CONFIG_PGTABLE_LEVELS > 3 */ static inline unsigned long p4d_index(unsigned long address) { … } #if CONFIG_PGTABLE_LEVELS > 4 static inline int pgd_present(pgd_t pgd) { … } static inline unsigned long pgd_page_vaddr(pgd_t pgd) { … } /* * Currently stuck as a macro due to indirect forward reference to * linux/mmzone.h's __section_mem_map_addr() definition: */ #define pgd_page(pgd) … /* to find an entry in a page-table-directory. */ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { … } static inline int pgd_bad(pgd_t pgd) { … } static inline int pgd_none(pgd_t pgd) { … } #endif /* CONFIG_PGTABLE_LEVELS > 4 */ #endif /* __ASSEMBLY__ */ #define KERNEL_PGD_BOUNDARY … #define KERNEL_PGD_PTRS … #ifndef __ASSEMBLY__ extern int direct_gbpages; void init_mem_mapping(void); void early_alloc_pgt_buf(void); void __init poking_init(void); unsigned long init_memory_mapping(unsigned long start, unsigned long end, pgprot_t prot); #ifdef CONFIG_X86_64 extern pgd_t trampoline_pgd_entry; #endif /* local pte updates need not use xchg for locking */ static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) { … } static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp) { … } static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp) { … } static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { … } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { … } /* * We only update the dirty/accessed state if we set * the dirty bit by hand in the kernel, since the hardware * will do the accessed bit for us, and we don't want to * race with other CPU's that might be updating the dirty * bit at the same time. */ struct vm_area_struct; #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep); #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH extern int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { … } #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) { … } #define __HAVE_ARCH_PTEP_SET_WRPROTECT static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { … } #define flush_tlb_fix_spurious_fault(vma, address, ptep) … #define mk_pmd(page, pgprot) … #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS extern int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty); extern int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty); #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp); extern int pudp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pud_t *pudp); #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH extern int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { … } #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pud_t *pudp) { … } #define __HAVE_ARCH_PMDP_SET_WRPROTECT static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { … } #ifndef pmdp_establish #define pmdp_establish … static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { … } #endif #define __HAVE_ARCH_PMDP_INVALIDATE_AD extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); /* * Page table pages are page-aligned. The lower half of the top * level is used for userspace and the top half for the kernel. * * Returns true for parts of the PGD that map userspace and * false for the parts that map the kernel. */ static inline bool pgdp_maps_userspace(void *__ptr) { … } #define pgd_leaf … static inline bool pgd_leaf(pgd_t pgd) { … } #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION /* * All top-level MITIGATION_PAGE_TABLE_ISOLATION page tables are order-1 pages * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and * the user one is in the last 4k. To switch between them, you * just need to flip the 12th bit in their addresses. */ #define PTI_PGTABLE_SWITCH_BIT … /* * This generates better code than the inline assembly in * __set_bit(). */ static inline void *ptr_set_bit(void *ptr, int bit) { … } static inline void *ptr_clear_bit(void *ptr, int bit) { … } static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) { … } static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) { … } static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) { … } static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) { … } #endif /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */ /* * clone_pgd_range(pgd_t *dst, pgd_t *src, int count); * * dst - pointer to pgd range anywhere on a pgd page * src - "" * count - the number of pgds to copy. * * dst and src can be on the same page, but the range must not overlap, * and must not cross a page boundary. */ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) { … } #define PTE_SHIFT … static inline int page_level_shift(enum pg_level level) { … } static inline unsigned long page_level_size(enum pg_level level) { … } static inline unsigned long page_level_mask(enum pg_level level) { … } /* * The x86 doesn't have any external MMU info: the kernel page * tables contain all the necessary information. */ static inline void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { … } static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { … } static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd) { … } static inline void update_mmu_cache_pud(struct vm_area_struct *vma, unsigned long addr, pud_t *pud) { … } static inline pte_t pte_swp_mkexclusive(pte_t pte) { … } static inline int pte_swp_exclusive(pte_t pte) { … } static inline pte_t pte_swp_clear_exclusive(pte_t pte) { … } #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline pte_t pte_swp_mksoft_dirty(pte_t pte) { … } static inline int pte_swp_soft_dirty(pte_t pte) { … } static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) { … } #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { … } static inline int pmd_swp_soft_dirty(pmd_t pmd) { … } static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { … } #endif #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { … } static inline int pte_swp_uffd_wp(pte_t pte) { … } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { … } static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) { … } static inline int pmd_swp_uffd_wp(pmd_t pmd) { … } static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) { … } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ static inline u16 pte_flags_pkey(unsigned long pte_flags) { … } static inline bool __pkru_allows_pkey(u16 pkey, bool write) { … } /* * 'pteval' can come from a PTE, PMD or PUD. We only check * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the * same value on all 3 types. */ static inline bool __pte_access_permitted(unsigned long pteval, bool write) { … } #define pte_access_permitted … static inline bool pte_access_permitted(pte_t pte, bool write) { … } #define pmd_access_permitted … static inline bool pmd_access_permitted(pmd_t pmd, bool write) { … } #define pud_access_permitted … static inline bool pud_access_permitted(pud_t pud, bool write) { … } #define __HAVE_ARCH_PFN_MODIFY_ALLOWED … extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot); static inline bool arch_has_pfn_modify_check(void) { … } #define arch_check_zapped_pte … void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte); #define arch_check_zapped_pmd … void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd); #ifdef CONFIG_XEN_PV #define arch_has_hw_nonleaf_pmd_young … static inline bool arch_has_hw_nonleaf_pmd_young(void) { … } #endif #ifdef CONFIG_PAGE_TABLE_CHECK static inline bool pte_user_accessible_page(pte_t pte) { … } static inline bool pmd_user_accessible_page(pmd_t pmd) { … } static inline bool pud_user_accessible_page(pud_t pud) { … } #endif #ifdef CONFIG_X86_SGX int arch_memory_failure(unsigned long pfn, int flags); #define arch_memory_failure … bool arch_is_platform_page(u64 paddr); #define arch_is_platform_page … #endif #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_PGTABLE_H */
Codebase Browser
linux