// SPDX-License-Identifier: GPL-2.0-only /* * IOMMU API for Rockchip * * Module Authors: Simon Xue <[email protected]> * Daniel Kurtz <[email protected]> */ #include <linux/clk.h> #include <linux/compiler.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/list.h> #include <linux/mm.h> #include <linux/init.h> #include <linux/of.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> #include <linux/spinlock.h> #include "iommu-pages.h" /** MMU register offsets */ #define RK_MMU_DTE_ADDR … #define RK_MMU_STATUS … #define RK_MMU_COMMAND … #define RK_MMU_PAGE_FAULT_ADDR … #define RK_MMU_ZAP_ONE_LINE … #define RK_MMU_INT_RAWSTAT … #define RK_MMU_INT_CLEAR … #define RK_MMU_INT_MASK … #define RK_MMU_INT_STATUS … #define RK_MMU_AUTO_GATING … #define DTE_ADDR_DUMMY … #define RK_MMU_POLL_PERIOD_US … #define RK_MMU_FORCE_RESET_TIMEOUT_US … #define RK_MMU_POLL_TIMEOUT_US … /* RK_MMU_STATUS fields */ #define RK_MMU_STATUS_PAGING_ENABLED … #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE … #define RK_MMU_STATUS_STALL_ACTIVE … #define RK_MMU_STATUS_IDLE … #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY … #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE … #define RK_MMU_STATUS_STALL_NOT_ACTIVE … /* RK_MMU_COMMAND command values */ #define RK_MMU_CMD_ENABLE_PAGING … #define RK_MMU_CMD_DISABLE_PAGING … #define RK_MMU_CMD_ENABLE_STALL … #define RK_MMU_CMD_DISABLE_STALL … #define RK_MMU_CMD_ZAP_CACHE … #define RK_MMU_CMD_PAGE_FAULT_DONE … #define RK_MMU_CMD_FORCE_RESET … /* RK_MMU_INT_* register fields */ #define RK_MMU_IRQ_PAGE_FAULT … #define RK_MMU_IRQ_BUS_ERROR … #define RK_MMU_IRQ_MASK … #define NUM_DT_ENTRIES … #define NUM_PT_ENTRIES … #define SPAGE_ORDER … #define SPAGE_SIZE … /* * Support mapping any size that fits in one page table: * 4 KiB to 4 MiB */ #define RK_IOMMU_PGSIZE_BITMAP … struct rk_iommu_domain { … }; /* list of clocks required by IOMMU */ static const char * const rk_iommu_clocks[] = …; struct rk_iommu_ops { … }; struct rk_iommu { … }; struct rk_iommudata { … }; static struct device *dma_dev; static const struct rk_iommu_ops *rk_ops; static struct iommu_domain rk_identity_domain; static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, unsigned int count) { … } static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) { … } /* * The Rockchip rk3288 iommu uses a 2-level page table. * The first level is the "Directory Table" (DT). * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing * to a "Page Table". * The second level is the 1024 Page Tables (PT). * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to * a 4 KB page of physical memory. * * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). * Each iommu device has a MMU_DTE_ADDR register that contains the physical * address of the start of the DT page. * * The structure of the page table is as follows: * * DT * MMU_DTE_ADDR -> +-----+ * | | * +-----+ PT * | DTE | -> +-----+ * +-----+ | | Memory * | | +-----+ Page * | | | PTE | -> +-----+ * +-----+ +-----+ | | * | | | | * | | | | * +-----+ | | * | | * | | * +-----+ */ /* * Each DTE has a PT address and a valid bit: * +---------------------+-----------+-+ * | PT address | Reserved |V| * +---------------------+-----------+-+ * 31:12 - PT address (PTs always starts on a 4 KB boundary) * 11: 1 - Reserved * 0 - 1 if PT @ PT address is valid */ #define RK_DTE_PT_ADDRESS_MASK … #define RK_DTE_PT_VALID … static inline phys_addr_t rk_dte_pt_address(u32 dte) { … } /* * In v2: * 31:12 - PT address bit 31:0 * 11: 8 - PT address bit 35:32 * 7: 4 - PT address bit 39:36 * 3: 1 - Reserved * 0 - 1 if PT @ PT address is valid */ #define RK_DTE_PT_ADDRESS_MASK_V2 … #define DTE_HI_MASK1 … #define DTE_HI_MASK2 … #define DTE_HI_SHIFT1 … #define DTE_HI_SHIFT2 … #define PAGE_DESC_HI_MASK1 … #define PAGE_DESC_HI_MASK2 … static inline phys_addr_t rk_dte_pt_address_v2(u32 dte) { … } static inline bool rk_dte_is_pt_valid(u32 dte) { … } static inline u32 rk_mk_dte(dma_addr_t pt_dma) { … } static inline u32 rk_mk_dte_v2(dma_addr_t pt_dma) { … } /* * Each PTE has a Page address, some flags and a valid bit: * +---------------------+---+-------+-+ * | Page address |Rsv| Flags |V| * +---------------------+---+-------+-+ * 31:12 - Page address (Pages always start on a 4 KB boundary) * 11: 9 - Reserved * 8: 1 - Flags * 8 - Read allocate - allocate cache space on read misses * 7 - Read cache - enable cache & prefetch of data * 6 - Write buffer - enable delaying writes on their way to memory * 5 - Write allocate - allocate cache space on write misses * 4 - Write cache - different writes can be merged together * 3 - Override cache attributes * if 1, bits 4-8 control cache attributes * if 0, the system bus defaults are used * 2 - Writable * 1 - Readable * 0 - 1 if Page @ Page address is valid */ #define RK_PTE_PAGE_ADDRESS_MASK … #define RK_PTE_PAGE_FLAGS_MASK … #define RK_PTE_PAGE_WRITABLE … #define RK_PTE_PAGE_READABLE … #define RK_PTE_PAGE_VALID … static inline bool rk_pte_is_page_valid(u32 pte) { … } /* TODO: set cache flags per prot IOMMU_CACHE */ static u32 rk_mk_pte(phys_addr_t page, int prot) { … } /* * In v2: * 31:12 - Page address bit 31:0 * 11: 8 - Page address bit 35:32 * 7: 4 - Page address bit 39:36 * 3 - Security * 2 - Writable * 1 - Readable * 0 - 1 if Page @ Page address is valid */ static u32 rk_mk_pte_v2(phys_addr_t page, int prot) { … } static u32 rk_mk_pte_invalid(u32 pte) { … } /* * rk3288 iova (IOMMU Virtual Address) format * 31 22.21 12.11 0 * +-----------+-----------+-------------+ * | DTE index | PTE index | Page offset | * +-----------+-----------+-------------+ * 31:22 - DTE index - index of DTE in DT * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address * 11: 0 - Page offset - offset into page @ PTE.page_address */ #define RK_IOVA_DTE_MASK … #define RK_IOVA_DTE_SHIFT … #define RK_IOVA_PTE_MASK … #define RK_IOVA_PTE_SHIFT … #define RK_IOVA_PAGE_MASK … #define RK_IOVA_PAGE_SHIFT … static u32 rk_iova_dte_index(dma_addr_t iova) { … } static u32 rk_iova_pte_index(dma_addr_t iova) { … } static u32 rk_iova_page_offset(dma_addr_t iova) { … } static u32 rk_iommu_read(void __iomem *base, u32 offset) { … } static void rk_iommu_write(void __iomem *base, u32 offset, u32 value) { … } static void rk_iommu_command(struct rk_iommu *iommu, u32 command) { … } static void rk_iommu_base_command(void __iomem *base, u32 command) { … } static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start, size_t size) { … } static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) { … } static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) { … } static bool rk_iommu_is_reset_done(struct rk_iommu *iommu) { … } static int rk_iommu_enable_stall(struct rk_iommu *iommu) { … } static int rk_iommu_disable_stall(struct rk_iommu *iommu) { … } static int rk_iommu_enable_paging(struct rk_iommu *iommu) { … } static int rk_iommu_disable_paging(struct rk_iommu *iommu) { … } static int rk_iommu_force_reset(struct rk_iommu *iommu) { … } static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova) { … } static irqreturn_t rk_iommu_irq(int irq, void *dev_id) { … } static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { … } static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, dma_addr_t iova, size_t size) { … } static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, dma_addr_t iova, size_t size) { … } static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { … } static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, dma_addr_t pte_dma, size_t size) { … } static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, dma_addr_t pte_dma, dma_addr_t iova, phys_addr_t paddr, size_t size, int prot) { … } static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, phys_addr_t paddr, size_t size, size_t count, int prot, gfp_t gfp, size_t *mapped) { … } static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, size_t size, size_t count, struct iommu_iotlb_gather *gather) { … } static struct rk_iommu *rk_iommu_from_dev(struct device *dev) { … } /* Must be called with iommu powered on and attached */ static void rk_iommu_disable(struct rk_iommu *iommu) { … } /* Must be called with iommu powered on and attached */ static int rk_iommu_enable(struct rk_iommu *iommu) { … } static int rk_iommu_identity_attach(struct iommu_domain *identity_domain, struct device *dev) { … } static struct iommu_domain_ops rk_identity_ops = …; static struct iommu_domain rk_identity_domain = …; static int rk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { … } static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev) { … } static void rk_iommu_domain_free(struct iommu_domain *domain) { … } static struct iommu_device *rk_iommu_probe_device(struct device *dev) { … } static void rk_iommu_release_device(struct device *dev) { … } static int rk_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) { … } static const struct iommu_ops rk_iommu_ops = …; static int rk_iommu_probe(struct platform_device *pdev) { … } static void rk_iommu_shutdown(struct platform_device *pdev) { … } static int __maybe_unused rk_iommu_suspend(struct device *dev) { … } static int __maybe_unused rk_iommu_resume(struct device *dev) { … } static const struct dev_pm_ops rk_iommu_pm_ops = …; static struct rk_iommu_ops iommu_data_ops_v1 = …; static struct rk_iommu_ops iommu_data_ops_v2 = …; static const struct of_device_id rk_iommu_dt_ids[] = …; static struct platform_driver rk_iommu_driver = …; builtin_platform_driver(…) …;
Codebase Browser
linux