// SPDX-License-Identifier: GPL-2.0-only /* * CPU-agnostic ARM page table allocator. * * ARMv7 Short-descriptor format, supporting * - Basic memory attributes * - Simplified access permissions (AP[2:1] model) * - Backwards-compatible TEX remap * - Large pages/supersections (if indicated by the caller) * * Not supporting: * - Legacy access permissions (AP[2:0] model) * * Almost certainly never supporting: * - PXN * - Domains * * Copyright (C) 2014-2015 ARM Limited * Copyright (c) 2014-2015 MediaTek Inc. */ #define pr_fmt(fmt) … #include <linux/atomic.h> #include <linux/dma-mapping.h> #include <linux/gfp.h> #include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/kmemleak.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/types.h> #include <asm/barrier.h> /* Struct accessors */ #define io_pgtable_to_data(x) … #define io_pgtable_ops_to_data(x) … /* * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, * and 12 bits in a page. * MediaTek extend 2 bits to reach 34bits, 14 bits at lvl1 and 8 bits at lvl2. */ #define ARM_V7S_ADDR_BITS … #define _ARM_V7S_LVL_BITS(lvl, cfg) … #define ARM_V7S_LVL_SHIFT(lvl) … #define ARM_V7S_TABLE_SHIFT … #define ARM_V7S_PTES_PER_LVL(lvl, cfg) … #define ARM_V7S_TABLE_SIZE(lvl, cfg) … #define ARM_V7S_BLOCK_SIZE(lvl) … #define ARM_V7S_LVL_MASK(lvl) … #define ARM_V7S_TABLE_MASK … #define _ARM_V7S_IDX_MASK(lvl, cfg) … #define ARM_V7S_LVL_IDX(addr, lvl, cfg) … /* * Large page/supersection entries are effectively a block of 16 page/section * entries, along the lines of the LPAE contiguous hint, but all with the * same output address. For want of a better common name we'll call them * "contiguous" versions of their respective page/section entries here, but * noting the distinction (WRT to TLB maintenance) that they represent *one* * entry repeated 16 times, not 16 separate entries (as in the LPAE case). */ #define ARM_V7S_CONT_PAGES … /* PTE type bits: these are all mixed up with XN/PXN bits in most cases */ #define ARM_V7S_PTE_TYPE_TABLE … #define ARM_V7S_PTE_TYPE_PAGE … #define ARM_V7S_PTE_TYPE_CONT_PAGE … #define ARM_V7S_PTE_IS_VALID(pte) … #define ARM_V7S_PTE_IS_TABLE(pte, lvl) … /* Page table bits */ #define ARM_V7S_ATTR_XN(lvl) … #define ARM_V7S_ATTR_B … #define ARM_V7S_ATTR_C … #define ARM_V7S_ATTR_NS_TABLE … #define ARM_V7S_ATTR_NS_SECTION … #define ARM_V7S_CONT_SECTION … #define ARM_V7S_CONT_PAGE_XN_SHIFT … /* * The attribute bits are consistently ordered*, but occupy bits [17:10] of * a level 1 PTE vs. bits [11:4] at level 2. Thus we define the individual * fields relative to that 8-bit block, plus a total shift relative to the PTE. */ #define ARM_V7S_ATTR_SHIFT(lvl) … #define ARM_V7S_ATTR_MASK … #define ARM_V7S_ATTR_AP0 … #define ARM_V7S_ATTR_AP1 … #define ARM_V7S_ATTR_AP2 … #define ARM_V7S_ATTR_S … #define ARM_V7S_ATTR_NG … #define ARM_V7S_TEX_SHIFT … #define ARM_V7S_TEX_MASK … #define ARM_V7S_ATTR_TEX(val) … /* MediaTek extend the bits below for PA 32bit/33bit/34bit */ #define ARM_V7S_ATTR_MTK_PA_BIT32 … #define ARM_V7S_ATTR_MTK_PA_BIT33 … #define ARM_V7S_ATTR_MTK_PA_BIT34 … /* *well, except for TEX on level 2 large pages, of course :( */ #define ARM_V7S_CONT_PAGE_TEX_SHIFT … #define ARM_V7S_CONT_PAGE_TEX_MASK … /* Simplified access permissions */ #define ARM_V7S_PTE_AF … #define ARM_V7S_PTE_AP_UNPRIV … #define ARM_V7S_PTE_AP_RDONLY … /* Register bits */ #define ARM_V7S_RGN_NC … #define ARM_V7S_RGN_WBWA … #define ARM_V7S_RGN_WT … #define ARM_V7S_RGN_WB … #define ARM_V7S_PRRR_TYPE_DEVICE … #define ARM_V7S_PRRR_TYPE_NORMAL … #define ARM_V7S_PRRR_TR(n, type) … #define ARM_V7S_PRRR_DS0 … #define ARM_V7S_PRRR_DS1 … #define ARM_V7S_PRRR_NS0 … #define ARM_V7S_PRRR_NS1 … #define ARM_V7S_PRRR_NOS(n) … #define ARM_V7S_NMRR_IR(n, attr) … #define ARM_V7S_NMRR_OR(n, attr) … #define ARM_V7S_TTBR_S … #define ARM_V7S_TTBR_NOS … #define ARM_V7S_TTBR_ORGN_ATTR(attr) … #define ARM_V7S_TTBR_IRGN_ATTR(attr) … #ifdef CONFIG_ZONE_DMA32 #define ARM_V7S_TABLE_GFP_DMA … #define ARM_V7S_TABLE_SLAB_FLAGS … #else #define ARM_V7S_TABLE_GFP_DMA … #define ARM_V7S_TABLE_SLAB_FLAGS … #endif arm_v7s_iopte; static bool selftest_running; struct arm_v7s_io_pgtable { … }; static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl); static dma_addr_t __arm_v7s_dma_addr(void *pages) { … } static bool arm_v7s_is_mtk_enabled(struct io_pgtable_cfg *cfg) { … } static arm_v7s_iopte to_mtk_iopte(phys_addr_t paddr, arm_v7s_iopte pte) { … } static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, struct io_pgtable_cfg *cfg) { … } static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl, struct io_pgtable_cfg *cfg) { … } static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl, struct arm_v7s_io_pgtable *data) { … } static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, struct arm_v7s_io_pgtable *data) { … } static void __arm_v7s_free_table(void *table, int lvl, struct arm_v7s_io_pgtable *data) { … } static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, struct io_pgtable_cfg *cfg) { … } static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte, int num_entries, struct io_pgtable_cfg *cfg) { … } static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, struct io_pgtable_cfg *cfg) { … } static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) { … } static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl) { … } static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl) { … } static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) { … } static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *, struct iommu_iotlb_gather *, unsigned long, size_t, int, arm_v7s_iopte *); static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, int prot, int lvl, int num_entries, arm_v7s_iopte *ptep) { … } static arm_v7s_iopte arm_v7s_install_table(arm_v7s_iopte *table, arm_v7s_iopte *ptep, arm_v7s_iopte curr, struct io_pgtable_cfg *cfg) { … } static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, int prot, int lvl, arm_v7s_iopte *ptep, gfp_t gfp) { … } static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { … } static void arm_v7s_free_pgtable(struct io_pgtable *iop) { … } static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, unsigned long iova, int idx, int lvl, arm_v7s_iopte *ptep) { … } static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, arm_v7s_iopte blk_pte, arm_v7s_iopte *ptep) { … } static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, int lvl, arm_v7s_iopte *ptep) { … } static size_t arm_v7s_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) { … } static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { … } static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { … } struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns = …; #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S_SELFTEST static struct io_pgtable_cfg *cfg_cookie __initdata; static void __init dummy_tlb_flush_all(void *cookie) { … } static void __init dummy_tlb_flush(unsigned long iova, size_t size, size_t granule, void *cookie) { … } static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) { … } static const struct iommu_flush_ops dummy_tlb_ops __initconst = …; #define __FAIL(ops) … static int __init arm_v7s_do_selftests(void) { … } subsys_initcall(arm_v7s_do_selftests); #endif
Codebase Browser
linux