// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu <[email protected]> */ #include <linux/arm-smccc.h> #include <linux/bitfield.h> #include <linux/bug.h> #include <linux/clk.h> #include <linux/component.h> #include <linux/device.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/io-pgtable.h> #include <linux/list.h> #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/regmap.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/soc/mediatek/infracfg.h> #include <linux/soc/mediatek/mtk_sip_svc.h> #include <asm/barrier.h> #include <soc/mediatek/smi.h> #include <dt-bindings/memory/mtk-memory-port.h> #define REG_MMU_PT_BASE_ADDR … #define REG_MMU_INVALIDATE … #define F_ALL_INVLD … #define F_MMU_INV_RANGE … #define REG_MMU_INVLD_START_A … #define REG_MMU_INVLD_END_A … #define REG_MMU_INV_SEL_GEN2 … #define REG_MMU_INV_SEL_GEN1 … #define F_INVLD_EN0 … #define F_INVLD_EN1 … #define REG_MMU_MISC_CTRL … #define F_MMU_IN_ORDER_WR_EN_MASK … #define F_MMU_STANDARD_AXI_MODE_MASK … #define REG_MMU_DCM_DIS … #define F_MMU_DCM … #define REG_MMU_WR_LEN_CTRL … #define F_MMU_WR_THROT_DIS_MASK … #define REG_MMU_CTRL_REG … #define F_MMU_TF_PROT_TO_PROGRAM_ADDR … #define F_MMU_PREFETCH_RT_REPLACE_MOD … #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 … #define REG_MMU_IVRP_PADDR … #define REG_MMU_VLD_PA_RNG … #define F_MMU_VLD_PA_RNG(EA, SA) … #define REG_MMU_INT_CONTROL0 … #define F_L2_MULIT_HIT_EN … #define F_TABLE_WALK_FAULT_INT_EN … #define F_PREETCH_FIFO_OVERFLOW_INT_EN … #define F_MISS_FIFO_OVERFLOW_INT_EN … #define F_PREFETCH_FIFO_ERR_INT_EN … #define F_MISS_FIFO_ERR_INT_EN … #define F_INT_CLR_BIT … #define REG_MMU_INT_MAIN_CONTROL … /* mmu0 | mmu1 */ #define F_INT_TRANSLATION_FAULT … #define F_INT_MAIN_MULTI_HIT_FAULT … #define F_INT_INVALID_PA_FAULT … #define F_INT_ENTRY_REPLACEMENT_FAULT … #define F_INT_TLB_MISS_FAULT … #define F_INT_MISS_TRANSACTION_FIFO_FAULT … #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT … #define REG_MMU_CPE_DONE … #define REG_MMU_FAULT_ST1 … #define F_REG_MMU0_FAULT_MASK … #define F_REG_MMU1_FAULT_MASK … #define REG_MMU0_FAULT_VA … #define F_MMU_INVAL_VA_31_12_MASK … #define F_MMU_INVAL_VA_34_32_MASK … #define F_MMU_INVAL_PA_34_32_MASK … #define F_MMU_FAULT_VA_WRITE_BIT … #define F_MMU_FAULT_VA_LAYER_BIT … #define REG_MMU0_INVLD_PA … #define REG_MMU1_FAULT_VA … #define REG_MMU1_INVLD_PA … #define REG_MMU0_INT_ID … #define REG_MMU1_INT_ID … #define F_MMU_INT_ID_COMM_ID(a) … #define F_MMU_INT_ID_SUB_COMM_ID(a) … #define F_MMU_INT_ID_COMM_ID_EXT(a) … #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) … /* Macro for 5 bits length port ID field (default) */ #define F_MMU_INT_ID_LARB_ID(a) … #define F_MMU_INT_ID_PORT_ID(a) … /* Macro for 6 bits length port ID field */ #define F_MMU_INT_ID_LARB_ID_WID_6(a) … #define F_MMU_INT_ID_PORT_ID_WID_6(a) … #define MTK_PROTECT_PA_ALIGN … #define MTK_IOMMU_BANK_SZ … #define PERICFG_IOMMU_1 … #define HAS_4GB_MODE … /* HW will use the EMI clock if there isn't the "bclk". */ #define HAS_BCLK … #define HAS_VLD_PA_RNG … #define RESET_AXI … #define OUT_ORDER_WR_EN … #define HAS_SUB_COMM_2BITS … #define HAS_SUB_COMM_3BITS … #define WR_THROT_EN … #define HAS_LEGACY_IVRP_PADDR … #define IOVA_34_EN … #define SHARE_PGTABLE … #define DCM_DISABLE … #define STD_AXI_MODE … /* 2 bits: iommu type */ #define MTK_IOMMU_TYPE_MM … #define MTK_IOMMU_TYPE_INFRA … #define MTK_IOMMU_TYPE_MASK … /* PM and clock always on. e.g. infra iommu */ #define PM_CLK_AO … #define IFA_IOMMU_PCIE_SUPPORT … #define PGTABLE_PA_35_EN … #define TF_PORT_TO_ADDR_MT8173 … #define INT_ID_PORT_WIDTH_6 … #define CFG_IFA_MASTER_IN_ATF … #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) … #define MTK_IOMMU_HAS_FLAG(pdata, _x) … #define MTK_IOMMU_IS_TYPE(pdata, _x) … #define MTK_INVALID_LARBID … #define MTK_LARB_COM_MAX … #define MTK_LARB_SUBCOM_MAX … #define MTK_IOMMU_GROUP_MAX … #define MTK_IOMMU_BANK_MAX … enum mtk_iommu_plat { … }; struct mtk_iommu_iova_region { … }; struct mtk_iommu_suspend_reg { … }; struct mtk_iommu_plat_data { … }; struct mtk_iommu_bank_data { … }; struct mtk_iommu_data { … }; struct mtk_iommu_domain { … }; static int mtk_iommu_bind(struct device *dev) { … } static void mtk_iommu_unbind(struct device *dev) { … } static const struct iommu_ops mtk_iommu_ops; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid); #define MTK_IOMMU_TLB_ADDR(iova) … /* * In M4U 4GB mode, the physical address is remapped as below: * * CPU Physical address: * ==================== * * 0 1G 2G 3G 4G 5G * |---A---|---B---|---C---|---D---|---E---| * +--I/O--+------------Memory-------------+ * * IOMMU output physical address: * ============================= * * 4G 5G 6G 7G 8G * |---E---|---B---|---C---|---D---| * +------------Memory-------------+ * * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the * bit32 of the CPU physical address always is needed to set, and for Region * 'E', the CPU physical address keep as is. * Additionally, The iommu consumers always use the CPU phyiscal address. */ #define MTK_IOMMU_4GB_MODE_REMAP_BASE … static LIST_HEAD(m4ulist); /* List all the M4U HWs */ #define for_each_m4u(data, head) … #define MTK_IOMMU_IOVA_SZ_4G … static const struct mtk_iommu_iova_region single_domain[] = …; #define MT8192_MULTI_REGION_NR_MAX … #define MT8192_MULTI_REGION_NR … static const struct mtk_iommu_iova_region mt8192_multi_dom[MT8192_MULTI_REGION_NR] = …; /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) { … } static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) { … } static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { … } static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, struct mtk_iommu_bank_data *bank) { … } static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { … } static unsigned int mtk_iommu_get_bank_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { … } static int mtk_iommu_get_iova_region_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { … } static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, bool enable, unsigned int regionid) { … } static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, struct mtk_iommu_data *data, unsigned int region_id) { … } static struct iommu_domain *mtk_iommu_domain_alloc_paging(struct device *dev) { … } static void mtk_iommu_domain_free(struct iommu_domain *domain) { … } static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { … } static int mtk_iommu_identity_attach(struct iommu_domain *identity_domain, struct device *dev) { … } static struct iommu_domain_ops mtk_iommu_identity_ops = …; static struct iommu_domain mtk_iommu_identity_domain = …; static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { … } static size_t mtk_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) { … } static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) { … } static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { … } static int mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { … } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { … } static struct iommu_device *mtk_iommu_probe_device(struct device *dev) { … } static void mtk_iommu_release_device(struct device *dev) { … } static int mtk_iommu_get_group_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { … } static struct iommu_group *mtk_iommu_device_group(struct device *dev) { … } static int mtk_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) { … } static void mtk_iommu_get_resv_regions(struct device *dev, struct list_head *head) { … } static const struct iommu_ops mtk_iommu_ops = …; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid) { … } static const struct component_master_ops mtk_iommu_com_ops = …; static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, struct mtk_iommu_data *data) { … } static int mtk_iommu_probe(struct platform_device *pdev) { … } static void mtk_iommu_remove(struct platform_device *pdev) { … } static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) { … } static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) { … } static const struct dev_pm_ops mtk_iommu_pm_ops = …; static const struct mtk_iommu_plat_data mt2712_data = …; static const struct mtk_iommu_plat_data mt6779_data = …; static const struct mtk_iommu_plat_data mt6795_data = …; static const struct mtk_iommu_plat_data mt8167_data = …; static const struct mtk_iommu_plat_data mt8173_data = …; static const struct mtk_iommu_plat_data mt8183_data = …; static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = …; static const struct mtk_iommu_plat_data mt8186_data_mm = …; static const struct mtk_iommu_plat_data mt8188_data_infra = …; static const u32 mt8188_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = …; static const struct mtk_iommu_plat_data mt8188_data_vdo = …; static const struct mtk_iommu_plat_data mt8188_data_vpp = …; static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = …; static const struct mtk_iommu_plat_data mt8192_data = …; static const struct mtk_iommu_plat_data mt8195_data_infra = …; static const unsigned int mt8195_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = …; static const struct mtk_iommu_plat_data mt8195_data_vdo = …; static const struct mtk_iommu_plat_data mt8195_data_vpp = …; static const struct mtk_iommu_plat_data mt8365_data = …; static const struct of_device_id mtk_iommu_of_ids[] = …; MODULE_DEVICE_TABLE(of, mtk_iommu_of_ids); static struct platform_driver mtk_iommu_driver = …; module_platform_driver(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux