// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. * http://www.samsung.com */ #ifdef CONFIG_EXYNOS_IOMMU_DEBUG #define DEBUG #endif #include <linux/clk.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/io.h> #include <linux/iommu.h> #include <linux/interrupt.h> #include <linux/kmemleak.h> #include <linux/list.h> #include <linux/of.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> #include "iommu-pages.h" sysmmu_iova_t; sysmmu_pte_t; static struct iommu_domain exynos_identity_domain; /* We do not consider super section mapping (16MB) */ #define SECT_ORDER … #define LPAGE_ORDER … #define SPAGE_ORDER … #define SECT_SIZE … #define LPAGE_SIZE … #define SPAGE_SIZE … #define SECT_MASK … #define LPAGE_MASK … #define SPAGE_MASK … #define lv1ent_fault(sent) … #define lv1ent_zero(sent) … #define lv1ent_page_zero(sent) … #define lv1ent_page(sent) … #define lv1ent_section(sent) … #define lv2ent_fault(pent) … #define lv2ent_small(pent) … #define lv2ent_large(pent) … /* * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces * v5.0 introduced support for 36bit physical address space by shifting * all page entry values by 4 bits. * All SYSMMU controllers in the system support the address spaces of the same * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper * value (0 or 4). */ static short PG_ENT_SHIFT = …; #define SYSMMU_PG_ENT_SHIFT … #define SYSMMU_V5_PG_ENT_SHIFT … static const sysmmu_pte_t *LV1_PROT; static const sysmmu_pte_t SYSMMU_LV1_PROT[] = …; static const sysmmu_pte_t SYSMMU_V5_LV1_PROT[] = …; static const sysmmu_pte_t *LV2_PROT; static const sysmmu_pte_t SYSMMU_LV2_PROT[] = …; static const sysmmu_pte_t SYSMMU_V5_LV2_PROT[] = …; #define SYSMMU_SUPPORTED_PROT_BITS … #define sect_to_phys(ent) … #define section_phys(sent) … #define section_offs(iova) … #define lpage_phys(pent) … #define lpage_offs(iova) … #define spage_phys(pent) … #define spage_offs(iova) … #define NUM_LV1ENTRIES … #define NUM_LV2ENTRIES … static u32 lv1ent_offset(sysmmu_iova_t iova) { … } static u32 lv2ent_offset(sysmmu_iova_t iova) { … } #define LV1TABLE_SIZE … #define LV2TABLE_SIZE … #define SPAGES_PER_LPAGE … #define lv2table_base(sent) … #define mk_lv1ent_sect(pa, prot) … #define mk_lv1ent_page(pa) … #define mk_lv2ent_lpage(pa, prot) … #define mk_lv2ent_spage(pa, prot) … #define CTRL_ENABLE … #define CTRL_BLOCK … #define CTRL_DISABLE … #define CFG_LRU … #define CFG_EAP … #define CFG_QOS(n) … #define CFG_ACGEN … #define CFG_SYSSEL … #define CFG_FLPDCACHE … #define CTRL_VM_ENABLE … #define CTRL_VM_FAULT_MODE_STALL … #define CAPA0_CAPA1_EXIST … #define CAPA1_VCR_ENABLED … /* common registers */ #define REG_MMU_CTRL … #define REG_MMU_CFG … #define REG_MMU_STATUS … #define REG_MMU_VERSION … #define MMU_MAJ_VER(val) … #define MMU_MIN_VER(val) … #define MMU_RAW_VER(reg) … #define MAKE_MMU_VER(maj, min) … /* v1.x - v3.x registers */ #define REG_PAGE_FAULT_ADDR … #define REG_AW_FAULT_ADDR … #define REG_AR_FAULT_ADDR … #define REG_DEFAULT_SLAVE_ADDR … /* v5.x registers */ #define REG_V5_FAULT_AR_VA … #define REG_V5_FAULT_AW_VA … /* v7.x registers */ #define REG_V7_CAPA0 … #define REG_V7_CAPA1 … #define REG_V7_CTRL_VM … #define has_sysmmu(dev) … static struct device *dma_dev; static struct kmem_cache *lv2table_kmem_cache; static sysmmu_pte_t *zero_lv2_table; #define ZERO_LV2LINK … static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) { … } static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) { … } struct sysmmu_fault { … }; struct sysmmu_v1_fault_info { … }; static const struct sysmmu_v1_fault_info sysmmu_v1_faults[] = …; /* SysMMU v5 has the same faults for AR (0..4 bits) and AW (16..20 bits) */ static const char * const sysmmu_v5_fault_names[] = …; static const char * const sysmmu_v7_fault_names[] = …; /* * This structure is attached to dev->iommu->priv of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, * which are bound to given master device. It is usually referenced by 'owner' * pointer. */ struct exynos_iommu_owner { … }; /* * This structure exynos specific generalization of struct iommu_domain. * It contains list of SYSMMU controllers from all master devices, which has * been attached to this domain and page tables of IO address space defined by * it. It is usually referenced by 'domain' pointer. */ struct exynos_iommu_domain { … }; struct sysmmu_drvdata; /* * SysMMU version specific data. Contains offsets for the registers which can * be found in different SysMMU variants, but have different offset values. * Also contains version specific callbacks to abstract the hardware. */ struct sysmmu_variant { … }; /* * This structure hold all data of a single SYSMMU controller, this includes * hw resources like registers and clocks, pointers and list nodes to connect * it to all other structures, internal state and parameters read from device * tree. It is usually referenced by 'data' pointer. */ struct sysmmu_drvdata { … }; #define SYSMMU_REG(data, reg) … static int exynos_sysmmu_v1_get_fault_info(struct sysmmu_drvdata *data, unsigned int itype, struct sysmmu_fault *fault) { … } static int exynos_sysmmu_v5_get_fault_info(struct sysmmu_drvdata *data, unsigned int itype, struct sysmmu_fault *fault) { … } static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data, unsigned int itype, struct sysmmu_fault *fault) { … } /* SysMMU v1..v3 */ static const struct sysmmu_variant sysmmu_v1_variant = …; /* SysMMU v5 */ static const struct sysmmu_variant sysmmu_v5_variant = …; /* SysMMU v7: non-VM capable register layout */ static const struct sysmmu_variant sysmmu_v7_variant = …; /* SysMMU v7: VM capable register layout */ static const struct sysmmu_variant sysmmu_v7_vm_variant = …; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) { … } static void sysmmu_unblock(struct sysmmu_drvdata *data) { … } static bool sysmmu_block(struct sysmmu_drvdata *data) { … } static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { … } static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_iova_t iova, unsigned int num_inv) { … } static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { … } static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) { … } static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) { … } static bool __sysmmu_has_capa1(struct sysmmu_drvdata *data) { … } static void __sysmmu_get_vcr(struct sysmmu_drvdata *data) { … } static void __sysmmu_get_version(struct sysmmu_drvdata *data) { … } static void show_fault_information(struct sysmmu_drvdata *data, const struct sysmmu_fault *fault) { … } static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { … } static void __sysmmu_disable(struct sysmmu_drvdata *data) { … } static void __sysmmu_init_config(struct sysmmu_drvdata *data) { … } static void __sysmmu_enable_vid(struct sysmmu_drvdata *data) { … } static void __sysmmu_enable(struct sysmmu_drvdata *data) { … } static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { … } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_iova_t iova, size_t size) { … } static const struct iommu_ops exynos_iommu_ops; static int exynos_sysmmu_probe(struct platform_device *pdev) { … } static int __maybe_unused exynos_sysmmu_suspend(struct device *dev) { … } static int __maybe_unused exynos_sysmmu_resume(struct device *dev) { … } static const struct dev_pm_ops sysmmu_pm_ops = …; static const struct of_device_id sysmmu_of_match[] = …; static struct platform_driver exynos_sysmmu_driver __refdata = …; static inline void exynos_iommu_set_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { … } static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev) { … } static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) { … } static int exynos_iommu_identity_attach(struct iommu_domain *identity_domain, struct device *dev) { … } static struct iommu_domain_ops exynos_identity_ops = …; static struct iommu_domain exynos_identity_domain = …; static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { … } static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { … } static int lv1set_section(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, phys_addr_t paddr, int prot, short *pgcnt) { … } static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, int prot, short *pgcnt) { … } /* * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: * * System MMU v3.x has advanced logic to improve address translation * performance with caching more page table entries by a page table walk. * However, the logic has a bug that while caching faulty page table entries, * System MMU reports page fault if the cached fault entry is hit even though * the fault entry is updated to a valid entry after the entry is cached. * To prevent caching faulty page table entries which may be updated to valid * entries later, the virtual memory manager should care about the workaround * for the problem. The following describes the workaround. * * Any two consecutive I/O virtual address regions must have a hole of 128KiB * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug). * * Precisely, any start address of I/O virtual region must be aligned with * the following sizes for System MMU v3.1 and v3.2. * System MMU v3.1: 128KiB * System MMU v3.2: 256KiB * * Because System MMU v3.3 caches page table entries more aggressively, it needs * more workarounds. * - Any two consecutive I/O virtual regions must have a hole of size larger * than or equal to 128KiB. * - Start address of an I/O virtual region must be aligned by 128KiB. */ static int exynos_iommu_map(struct iommu_domain *iommu_domain, unsigned long l_iova, phys_addr_t paddr, size_t size, size_t count, int prot, gfp_t gfp, size_t *mapped) { … } static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, sysmmu_iova_t iova, size_t size) { … } static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, unsigned long l_iova, size_t size, size_t count, struct iommu_iotlb_gather *gather) { … } static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, dma_addr_t iova) { … } static struct iommu_device *exynos_iommu_probe_device(struct device *dev) { … } static void exynos_iommu_release_device(struct device *dev) { … } static int exynos_iommu_of_xlate(struct device *dev, const struct of_phandle_args *spec) { … } static const struct iommu_ops exynos_iommu_ops = …; static int __init exynos_iommu_init(void) { … } core_initcall(exynos_iommu_init);
Codebase Browser
linux