// SPDX-License-Identifier: GPL-2.0-only /* * Apple DART (Device Address Resolution Table) IOMMU driver * * Copyright (C) 2021 The Asahi Linux Contributors * * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c * Copyright (C) 2013 ARM Limited * Copyright (C) 2015 ARM Limited * and on exynos-iommu.c * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. */ #include <linux/atomic.h> #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dev_printk.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_iommu.h> #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/swab.h> #include <linux/types.h> #include "dma-iommu.h" #define DART_MAX_STREAMS … #define DART_MAX_TTBR … #define MAX_DARTS_PER_DEVICE … /* Common registers */ #define DART_PARAMS1 … #define DART_PARAMS1_PAGE_SHIFT … #define DART_PARAMS2 … #define DART_PARAMS2_BYPASS_SUPPORT … /* T8020/T6000 registers */ #define DART_T8020_STREAM_COMMAND … #define DART_T8020_STREAM_COMMAND_BUSY … #define DART_T8020_STREAM_COMMAND_INVALIDATE … #define DART_T8020_STREAM_SELECT … #define DART_T8020_ERROR … #define DART_T8020_ERROR_STREAM … #define DART_T8020_ERROR_CODE … #define DART_T8020_ERROR_FLAG … #define DART_T8020_ERROR_READ_FAULT … #define DART_T8020_ERROR_WRITE_FAULT … #define DART_T8020_ERROR_NO_PTE … #define DART_T8020_ERROR_NO_PMD … #define DART_T8020_ERROR_NO_TTBR … #define DART_T8020_CONFIG … #define DART_T8020_CONFIG_LOCK … #define DART_STREAM_COMMAND_BUSY_TIMEOUT … #define DART_T8020_ERROR_ADDR_HI … #define DART_T8020_ERROR_ADDR_LO … #define DART_T8020_STREAMS_ENABLE … #define DART_T8020_TCR … #define DART_T8020_TCR_TRANSLATE_ENABLE … #define DART_T8020_TCR_BYPASS_DART … #define DART_T8020_TCR_BYPASS_DAPF … #define DART_T8020_TTBR … #define DART_T8020_USB4_TTBR … #define DART_T8020_TTBR_VALID … #define DART_T8020_TTBR_ADDR_FIELD_SHIFT … #define DART_T8020_TTBR_SHIFT … /* T8110 registers */ #define DART_T8110_PARAMS3 … #define DART_T8110_PARAMS3_PA_WIDTH … #define DART_T8110_PARAMS3_VA_WIDTH … #define DART_T8110_PARAMS3_VER_MAJ … #define DART_T8110_PARAMS3_VER_MIN … #define DART_T8110_PARAMS4 … #define DART_T8110_PARAMS4_NUM_CLIENTS … #define DART_T8110_PARAMS4_NUM_SIDS … #define DART_T8110_TLB_CMD … #define DART_T8110_TLB_CMD_BUSY … #define DART_T8110_TLB_CMD_OP … #define DART_T8110_TLB_CMD_OP_FLUSH_ALL … #define DART_T8110_TLB_CMD_OP_FLUSH_SID … #define DART_T8110_TLB_CMD_STREAM … #define DART_T8110_ERROR … #define DART_T8110_ERROR_STREAM … #define DART_T8110_ERROR_CODE … #define DART_T8110_ERROR_FLAG … #define DART_T8110_ERROR_MASK … #define DART_T8110_ERROR_READ_FAULT … #define DART_T8110_ERROR_WRITE_FAULT … #define DART_T8110_ERROR_NO_PTE … #define DART_T8110_ERROR_NO_PMD … #define DART_T8110_ERROR_NO_PGD … #define DART_T8110_ERROR_NO_TTBR … #define DART_T8110_ERROR_ADDR_LO … #define DART_T8110_ERROR_ADDR_HI … #define DART_T8110_PROTECT … #define DART_T8110_UNPROTECT … #define DART_T8110_PROTECT_LOCK … #define DART_T8110_PROTECT_TTBR_TCR … #define DART_T8110_ENABLE_STREAMS … #define DART_T8110_DISABLE_STREAMS … #define DART_T8110_TCR … #define DART_T8110_TCR_REMAP … #define DART_T8110_TCR_REMAP_EN … #define DART_T8110_TCR_BYPASS_DAPF … #define DART_T8110_TCR_BYPASS_DART … #define DART_T8110_TCR_TRANSLATE_ENABLE … #define DART_T8110_TTBR … #define DART_T8110_TTBR_VALID … #define DART_T8110_TTBR_ADDR_FIELD_SHIFT … #define DART_T8110_TTBR_SHIFT … #define DART_TCR(dart, sid) … #define DART_TTBR(dart, sid, idx) … struct apple_dart_stream_map; enum dart_type { … }; struct apple_dart_hw { … }; /* * Private structure associated with each DART device. * * @dev: device struct * @hw: SoC-specific hardware data * @regs: mapped MMIO region * @irq: interrupt number, can be shared with other DARTs * @clks: clocks associated with this DART * @num_clks: number of @clks * @lock: lock for hardware operations involving this dart * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ struct apple_dart { … }; /* * Convenience struct to identify streams. * * The normal variant is used inside apple_dart_master_cfg which isn't written * to concurrently. * The atomic variant is used inside apple_dart_domain where we have to guard * against races from potential parallel calls to attach/detach_device. * Note that even inside the atomic variant the apple_dart pointer is not * protected: This pointer is initialized once under the domain init mutex * and never changed again afterwards. Devices with different dart pointers * cannot be attached to the same domain. * * @dart dart pointer * @sid stream id bitmap */ struct apple_dart_stream_map { … }; struct apple_dart_atomic_stream_map { … }; /* * This structure is attached to each iommu domain handled by a DART. * * @pgtbl_ops: pagetable ops allocated by io-pgtable * @finalized: true if the domain has been completely initialized * @init_lock: protects domain initialization * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) * @domain: core iommu domain pointer */ struct apple_dart_domain { … }; /* * This structure is attached to devices with dev_iommu_priv_set() on of_xlate * and contains a list of streams bound to this device. * So far the worst case seen is a single device with two streams * from different darts, such that this simple static array is enough. * * @streams: streams for this device */ struct apple_dart_master_cfg { … }; /* * Helper macro to iterate over apple_dart_master_cfg.stream_maps and * apple_dart_domain.stream_maps * * @i int used as loop variable * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain) * @stream pointer to the apple_dart_streams struct for each loop iteration */ #define for_each_stream_map(i, base, stream_map) … static struct platform_driver apple_dart_driver; static const struct iommu_ops apple_dart_iommu_ops; static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) { … } static void apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) { … } static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) { … } static void apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) { … } static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, u8 idx, phys_addr_t paddr) { … } static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { … } static void apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) { … } static int apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) { … } static int apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, u32 command) { … } static int apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { … } static int apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { … } static int apple_dart_hw_reset(struct apple_dart *dart) { … } static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) { … } static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) { … } static void apple_dart_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { … } static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { … } static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { … } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { … } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) { … } static void apple_dart_setup_translation(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) { … } static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, struct apple_dart_master_cfg *cfg) { … } static int apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, struct apple_dart_stream_map *master_maps, bool add_streams) { … } static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, struct apple_dart_master_cfg *cfg) { … } static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct device *dev) { … } static int apple_dart_attach_dev_identity(struct iommu_domain *domain, struct device *dev) { … } static const struct iommu_domain_ops apple_dart_identity_ops = …; static struct iommu_domain apple_dart_identity_domain = …; static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct device *dev) { … } static const struct iommu_domain_ops apple_dart_blocked_ops = …; static struct iommu_domain apple_dart_blocked_domain = …; static struct iommu_device *apple_dart_probe_device(struct device *dev) { … } static void apple_dart_release_device(struct device *dev) { … } static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) { … } static void apple_dart_domain_free(struct iommu_domain *domain) { … } static int apple_dart_of_xlate(struct device *dev, const struct of_phandle_args *args) { … } static DEFINE_MUTEX(apple_dart_groups_lock); static void apple_dart_release_group(void *iommu_data) { … } static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, struct apple_dart_master_cfg *src) { … } static struct iommu_group *apple_dart_device_group(struct device *dev) { … } static int apple_dart_def_domain_type(struct device *dev) { … } #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR … #endif #define DOORBELL_ADDR … static void apple_dart_get_resv_regions(struct device *dev, struct list_head *head) { … } static const struct iommu_ops apple_dart_iommu_ops = …; static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) { … } static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) { … } static int apple_dart_probe(struct platform_device *pdev) { … } static void apple_dart_remove(struct platform_device *pdev) { … } static const struct apple_dart_hw apple_dart_hw_t8103 = …; static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = …; static const struct apple_dart_hw apple_dart_hw_t6000 = …; static const struct apple_dart_hw apple_dart_hw_t8110 = …; static __maybe_unused int apple_dart_suspend(struct device *dev) { … } static __maybe_unused int apple_dart_resume(struct device *dev) { … } static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); static const struct of_device_id apple_dart_of_match[] = …; MODULE_DEVICE_TABLE(of, apple_dart_of_match); static struct platform_driver apple_dart_driver = …; module_platform_driver(…) …; MODULE_DESCRIPTION(…) …; MODULE_AUTHOR(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux