// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. // Copyright (C) 2019-2020, Cerno #include <linux/bitfield.h> #include <linux/bug.h> #include <linux/clk.h> #include <linux/device.h> #include <linux/dma-direction.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/ioport.h> #include <linux/log2.h> #include <linux/module.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/types.h> #include "iommu-pages.h" #define IOMMU_RESET_REG … #define IOMMU_RESET_RELEASE_ALL … #define IOMMU_ENABLE_REG … #define IOMMU_ENABLE_ENABLE … #define IOMMU_BYPASS_REG … #define IOMMU_AUTO_GATING_REG … #define IOMMU_AUTO_GATING_ENABLE … #define IOMMU_WBUF_CTRL_REG … #define IOMMU_OOO_CTRL_REG … #define IOMMU_4KB_BDY_PRT_CTRL_REG … #define IOMMU_TTB_REG … #define IOMMU_TLB_ENABLE_REG … #define IOMMU_TLB_PREFETCH_REG … #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) … #define IOMMU_TLB_FLUSH_REG … #define IOMMU_TLB_FLUSH_PTW_CACHE … #define IOMMU_TLB_FLUSH_MACRO_TLB … #define IOMMU_TLB_FLUSH_MICRO_TLB(i) … #define IOMMU_TLB_IVLD_ADDR_REG … #define IOMMU_TLB_IVLD_ADDR_MASK_REG … #define IOMMU_TLB_IVLD_ENABLE_REG … #define IOMMU_TLB_IVLD_ENABLE_ENABLE … #define IOMMU_PC_IVLD_ADDR_REG … #define IOMMU_PC_IVLD_ENABLE_REG … #define IOMMU_PC_IVLD_ENABLE_ENABLE … #define IOMMU_DM_AUT_CTRL_REG(d) … #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) … #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) … #define IOMMU_DM_AUT_OVWT_REG … #define IOMMU_INT_ENABLE_REG … #define IOMMU_INT_CLR_REG … #define IOMMU_INT_STA_REG … #define IOMMU_INT_ERR_ADDR_REG(i) … #define IOMMU_INT_ERR_ADDR_L1_REG … #define IOMMU_INT_ERR_ADDR_L2_REG … #define IOMMU_INT_ERR_DATA_REG(i) … #define IOMMU_L1PG_INT_REG … #define IOMMU_L2PG_INT_REG … #define IOMMU_INT_INVALID_L2PG … #define IOMMU_INT_INVALID_L1PG … #define IOMMU_INT_MASTER_PERMISSION(m) … #define IOMMU_INT_MASTER_MASK … #define IOMMU_INT_MASK … #define PT_ENTRY_SIZE … #define NUM_DT_ENTRIES … #define DT_SIZE … #define NUM_PT_ENTRIES … #define PT_SIZE … #define SPAGE_SIZE … struct sun50i_iommu { … }; struct sun50i_iommu_domain { … }; static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) { … } static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) { … } static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) { … } static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) { … } /* * The Allwinner H6 IOMMU uses a 2-level page table. * * The first level is the usual Directory Table (DT), that consists of * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page * Table (PT). * * Each PT consits of 256 4-bytes Page Table Entries (PTE), each * pointing to a 4kB page of physical memory. * * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG * register that contains its physical address. */ #define SUN50I_IOVA_DTE_MASK … #define SUN50I_IOVA_PTE_MASK … #define SUN50I_IOVA_PAGE_MASK … static u32 sun50i_iova_get_dte_index(dma_addr_t iova) { … } static u32 sun50i_iova_get_pte_index(dma_addr_t iova) { … } static u32 sun50i_iova_get_page_offset(dma_addr_t iova) { … } /* * Each Directory Table Entry has a Page Table address and a valid * bit: * +---------------------+-----------+-+ * | PT address | Reserved |V| * +---------------------+-----------+-+ * 31:10 - Page Table address * 9:2 - Reserved * 1:0 - 1 if the entry is valid */ #define SUN50I_DTE_PT_ADDRESS_MASK … #define SUN50I_DTE_PT_ATTRS … #define SUN50I_DTE_PT_VALID … static phys_addr_t sun50i_dte_get_pt_address(u32 dte) { … } static bool sun50i_dte_is_pt_valid(u32 dte) { … } static u32 sun50i_mk_dte(dma_addr_t pt_dma) { … } /* * Each PTE has a Page address, an authority index and a valid bit: * * +----------------+-----+-----+-----+---+-----+ * | Page address | Rsv | ACI | Rsv | V | Rsv | * +----------------+-----+-----+-----+---+-----+ * 31:12 - Page address * 11:8 - Reserved * 7:4 - Authority Control Index * 3:2 - Reserved * 1 - 1 if the entry is valid * 0 - Reserved * * The way permissions work is that the IOMMU has 16 "domains" that * can be configured to give each masters either read or write * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain * 0 seems like the default domain, and its permissions in the * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really * useful to enforce any particular permission. * * Each page entry will then have a reference to the domain they are * affected to, so that we can actually enforce them on a per-page * basis. * * In order to make it work with the IOMMU framework, we will be using * 4 different domains, starting at 1: RD_WR, RD, WR and NONE * depending on the permission we want to enforce. Each domain will * have each master setup in the same way, since the IOMMU framework * doesn't seem to restrict page access on a per-device basis. And * then we will use the relevant domain index when generating the page * table entry depending on the permissions we want to be enforced. */ enum sun50i_iommu_aci { … }; #define SUN50I_PTE_PAGE_ADDRESS_MASK … #define SUN50I_PTE_ACI_MASK … #define SUN50I_PTE_PAGE_VALID … static phys_addr_t sun50i_pte_get_page_address(u32 pte) { … } static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) { … } static bool sun50i_pte_is_page_valid(u32 pte) { … } static u32 sun50i_mk_pte(phys_addr_t page, int prot) { … } static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, void *vaddr, unsigned int count) { … } static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, unsigned long iova) { … } static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, unsigned long iova) { … } static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, unsigned long iova, size_t size) { … } static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) { … } static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) { … } static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { … } static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { … } static int sun50i_iommu_enable(struct sun50i_iommu *iommu) { … } static void sun50i_iommu_disable(struct sun50i_iommu *iommu) { … } static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, gfp_t gfp) { … } static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, u32 *page_table) { … } static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, dma_addr_t iova, gfp_t gfp) { … } static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, size_t count, int prot, gfp_t gfp, size_t *mapped) { … } static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size, size_t count, struct iommu_iotlb_gather *gather) { … } static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { … } static struct iommu_domain * sun50i_iommu_domain_alloc_paging(struct device *dev) { … } static void sun50i_iommu_domain_free(struct iommu_domain *domain) { … } static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, struct sun50i_iommu_domain *sun50i_domain) { … } static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, struct sun50i_iommu_domain *sun50i_domain) { … } static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain, struct device *dev) { … } static struct iommu_domain_ops sun50i_iommu_identity_ops = …; static struct iommu_domain sun50i_iommu_identity_domain = …; static int sun50i_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { … } static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) { … } static int sun50i_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) { … } static const struct iommu_ops sun50i_iommu_ops = …; static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, unsigned master, phys_addr_t iova, unsigned prot) { … } static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, unsigned addr_reg, unsigned blame_reg) { … } static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) { … } static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) { … } static int sun50i_iommu_probe(struct platform_device *pdev) { … } static const struct of_device_id sun50i_iommu_dt[] = …; MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); static struct platform_driver sun50i_iommu_driver = …; builtin_platform_driver_probe(…); MODULE_DESCRIPTION(…) …; MODULE_AUTHOR(…) …; MODULE_AUTHOR(…) …;
Codebase Browser
linux