// SPDX-License-Identifier: GPL-2.0-only /* * omap iommu: tlb and pagetable primitives * * Copyright (C) 2008-2010 Nokia Corporation * Copyright (C) 2013-2017 Texas Instruments Incorporated - https://www.ti.com/ * * Written by Hiroshi DOYU <[email protected]>, * Paul Mundt and Toshihiro Kobayashi */ #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/slab.h> #include <linux/interrupt.h> #include <linux/ioport.h> #include <linux/platform_device.h> #include <linux/iommu.h> #include <linux/omap-iommu.h> #include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/io.h> #include <linux/pm_runtime.h> #include <linux/of.h> #include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/regmap.h> #include <linux/mfd/syscon.h> #include <linux/platform_data/iommu-omap.h> #include "omap-iopgtable.h" #include "omap-iommu.h" static const struct iommu_ops omap_iommu_ops; #define to_iommu(dev) … /* bitmap of the page sizes currently supported */ #define OMAP_IOMMU_PGSIZES … #define MMU_LOCK_BASE_SHIFT … #define MMU_LOCK_BASE_MASK … #define MMU_LOCK_BASE(x) … #define MMU_LOCK_VICT_SHIFT … #define MMU_LOCK_VICT_MASK … #define MMU_LOCK_VICT(x) … static struct platform_driver omap_iommu_driver; static struct kmem_cache *iopte_cachep; /** * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain * @dom: generic iommu domain handle **/ static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) { … } /** * omap_iommu_save_ctx - Save registers for pm off-mode support * @dev: client device * * This should be treated as an deprecated API. It is preserved only * to maintain existing functionality for OMAP3 ISP driver. **/ void omap_iommu_save_ctx(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); /** * omap_iommu_restore_ctx - Restore registers for pm off-mode support * @dev: client device * * This should be treated as an deprecated API. It is preserved only * to maintain existing functionality for OMAP3 ISP driver. **/ void omap_iommu_restore_ctx(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable) { … } static void __iommu_set_twl(struct omap_iommu *obj, bool on) { … } static int omap2_iommu_enable(struct omap_iommu *obj) { … } static void omap2_iommu_disable(struct omap_iommu *obj) { … } static int iommu_enable(struct omap_iommu *obj) { … } static void iommu_disable(struct omap_iommu *obj) { … } /* * TLB operations */ static u32 iotlb_cr_to_virt(struct cr_regs *cr) { … } static u32 get_iopte_attr(struct iotlb_entry *e) { … } static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) { … } void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) { … } void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) { … } static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) { … } static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) { … } /* only used in iotlb iteration for-loop */ struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) { … } #ifdef PREFETCH_IOTLB static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, struct iotlb_entry *e) { struct cr_regs *cr; if (!e) return NULL; if (e->da & ~(get_cam_va_mask(e->pgsz))) { dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__, e->da); return ERR_PTR(-EINVAL); } cr = kmalloc(sizeof(*cr), GFP_KERNEL); if (!cr) return ERR_PTR(-ENOMEM); cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid; cr->ram = e->pa | e->endian | e->elsz | e->mixed; return cr; } /** * load_iotlb_entry - Set an iommu tlb entry * @obj: target iommu * @e: an iommu tlb entry info **/ static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) { int err = 0; struct iotlb_lock l; struct cr_regs *cr; if (!obj || !obj->nr_tlb_entries || !e) return -EINVAL; pm_runtime_get_sync(obj->dev); iotlb_lock_get(obj, &l); if (l.base == obj->nr_tlb_entries) { dev_warn(obj->dev, "%s: preserve entries full\n", __func__); err = -EBUSY; goto out; } if (!e->prsvd) { int i; struct cr_regs tmp; for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) if (!iotlb_cr_valid(&tmp)) break; if (i == obj->nr_tlb_entries) { dev_dbg(obj->dev, "%s: full: no entry\n", __func__); err = -EBUSY; goto out; } iotlb_lock_get(obj, &l); } else { l.vict = l.base; iotlb_lock_set(obj, &l); } cr = iotlb_alloc_cr(obj, e); if (IS_ERR(cr)) { pm_runtime_put_sync(obj->dev); return PTR_ERR(cr); } iotlb_load_cr(obj, cr); kfree(cr); if (e->prsvd) l.base++; /* increment victim for next tlb load */ if (++l.vict == obj->nr_tlb_entries) l.vict = l.base; iotlb_lock_set(obj, &l); out: pm_runtime_put_sync(obj->dev); return err; } #else /* !PREFETCH_IOTLB */ static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) { … } #endif /* !PREFETCH_IOTLB */ static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) { … } /** * flush_iotlb_page - Clear an iommu tlb entry * @obj: target iommu * @da: iommu device virtual address * * Clear an iommu tlb entry which includes 'da' address. **/ static void flush_iotlb_page(struct omap_iommu *obj, u32 da) { … } /** * flush_iotlb_all - Clear all iommu tlb entries * @obj: target iommu **/ static void flush_iotlb_all(struct omap_iommu *obj) { … } /* * H/W pagetable operations */ static void flush_iopte_range(struct device *dev, dma_addr_t dma, unsigned long offset, int num_entries) { … } static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid) { … } static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, dma_addr_t *pt_dma, u32 da) { … } static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { … } static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { … } static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { … } static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { … } static int iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) { … } /** * omap_iopgtable_store_entry - Make an iommu pte entry * @obj: target iommu * @e: an iommu tlb entry info **/ static int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) { … } /** * iopgtable_lookup_entry - Lookup an iommu pte entry * @obj: target iommu * @da: iommu device virtual address * @ppgd: iommu pgd entry pointer to be returned * @ppte: iommu pte entry pointer to be returned **/ static void iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) { … } static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) { … } /** * iopgtable_clear_entry - Remove an iommu pte entry * @obj: target iommu * @da: iommu device virtual address **/ static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) { … } static void iopgtable_clear_entry_all(struct omap_iommu *obj) { … } /* * Device IOMMU generic operations */ static irqreturn_t iommu_fault_handler(int irq, void *data) { … } /** * omap_iommu_attach() - attach iommu device to an iommu domain * @obj: target omap iommu device * @iopgd: page table **/ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd) { … } /** * omap_iommu_detach - release iommu device * @obj: target iommu **/ static void omap_iommu_detach(struct omap_iommu *obj) { … } static void omap_iommu_save_tlb_entries(struct omap_iommu *obj) { … } static void omap_iommu_restore_tlb_entries(struct omap_iommu *obj) { … } /** * omap_iommu_domain_deactivate - deactivate attached iommu devices * @domain: iommu domain attached to the target iommu device * * This API allows the client devices of IOMMU devices to suspend * the IOMMUs they control at runtime, after they are idled and * suspended all activity. System Suspend will leverage the PM * driver late callbacks. **/ int omap_iommu_domain_deactivate(struct iommu_domain *domain) { … } EXPORT_SYMBOL_GPL(…); /** * omap_iommu_domain_activate - activate attached iommu devices * @domain: iommu domain attached to the target iommu device * * This API allows the client devices of IOMMU devices to resume the * IOMMUs they control at runtime, before they can resume operations. * System Resume will leverage the PM driver late callbacks. **/ int omap_iommu_domain_activate(struct iommu_domain *domain) { … } EXPORT_SYMBOL_GPL(…); /** * omap_iommu_runtime_suspend - disable an iommu device * @dev: iommu device * * This function performs all that is necessary to disable an * IOMMU device, either during final detachment from a client * device, or during system/runtime suspend of the device. This * includes programming all the appropriate IOMMU registers, and * managing the associated omap_hwmod's state and the device's * reset line. This function also saves the context of any * locked TLBs if suspending. **/ static __maybe_unused int omap_iommu_runtime_suspend(struct device *dev) { … } /** * omap_iommu_runtime_resume - enable an iommu device * @dev: iommu device * * This function performs all that is necessary to enable an * IOMMU device, either during initial attachment to a client * device, or during system/runtime resume of the device. This * includes programming all the appropriate IOMMU registers, and * managing the associated omap_hwmod's state and the device's * reset line. The function also restores any locked TLBs if * resuming after a suspend. **/ static __maybe_unused int omap_iommu_runtime_resume(struct device *dev) { … } /** * omap_iommu_prepare - prepare() dev_pm_ops implementation * @dev: iommu device * * This function performs the necessary checks to determine if the IOMMU * device needs suspending or not. The function checks if the runtime_pm * status of the device is suspended, and returns 1 in that case. This * results in the PM core to skip invoking any of the Sleep PM callbacks * (suspend, suspend_late, resume, resume_early etc). */ static int omap_iommu_prepare(struct device *dev) { … } static bool omap_iommu_can_register(struct platform_device *pdev) { … } static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev, struct omap_iommu *obj) { … } /* * OMAP Device MMU(IOMMU) detection */ static int omap_iommu_probe(struct platform_device *pdev) { … } static void omap_iommu_remove(struct platform_device *pdev) { … } static const struct dev_pm_ops omap_iommu_pm_ops = …; static const struct of_device_id omap_iommu_of_match[] = …; static struct platform_driver omap_iommu_driver = …; static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) { … } static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, phys_addr_t pa, size_t bytes, size_t count, int prot, gfp_t gfp, size_t *mapped) { … } static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, size_t size, size_t count, struct iommu_iotlb_gather *gather) { … } static int omap_iommu_count(struct device *dev) { … } /* caller should call cleanup if this function fails */ static int omap_iommu_attach_init(struct device *dev, struct omap_iommu_domain *odomain) { … } static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain) { … } static int omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { … } static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, struct device *dev) { … } static int omap_iommu_identity_attach(struct iommu_domain *identity_domain, struct device *dev) { … } static struct iommu_domain_ops omap_iommu_identity_ops = …; static struct iommu_domain omap_iommu_identity_domain = …; static struct iommu_domain *omap_iommu_domain_alloc_paging(struct device *dev) { … } static void omap_iommu_domain_free(struct iommu_domain *domain) { … } static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t da) { … } static struct iommu_device *omap_iommu_probe_device(struct device *dev) { … } static void omap_iommu_release_device(struct device *dev) { … } static const struct iommu_ops omap_iommu_ops = …; static int __init omap_iommu_init(void) { … } subsys_initcall(omap_iommu_init); /* must be ready before omap3isp is probed */
Codebase Browser
linux