linux/drivers/media/pci/intel/ipu6/ipu6-mmu.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2013--2024 Intel Corporation
 */
#include <asm/barrier.h>

#include <linux/align.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/bits.h>
#include <linux/bug.h>
#include <linux/cacheflush.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/gfp.h>
#include <linux/io.h>
#include <linux/iova.h>
#include <linux/math.h>
#include <linux/minmax.h>
#include <linux/mm.h>
#include <linux/pfn.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/vmalloc.h>

#include "ipu6.h"
#include "ipu6-dma.h"
#include "ipu6-mmu.h"
#include "ipu6-platform-regs.h"

#define ISP_PAGE_SHIFT …
#define ISP_PAGE_SIZE …
#define ISP_PAGE_MASK …

#define ISP_L1PT_SHIFT …
#define ISP_L1PT_MASK …

#define ISP_L2PT_SHIFT …
#define ISP_L2PT_MASK …

#define ISP_L1PT_PTES …
#define ISP_L2PT_PTES …

#define ISP_PADDR_SHIFT …

#define REG_TLB_INVALIDATE …

#define REG_L1_PHYS …
#define REG_INFO …

#define TBL_PHYS_ADDR(a) …

static void tlb_invalidate(struct ipu6_mmu *mmu)
{ … }

#ifdef DEBUG
static void page_table_dump(struct ipu6_mmu_info *mmu_info)
{
	u32 l1_idx;

	dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");

	for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
		u32 l2_idx;
		u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;

		if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)
			continue;
		dev_dbg(mmu_info->dev,
			"l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pa\n",
			l1_idx, iova, iova + ISP_PAGE_SIZE,
			TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]));

		for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {
			u32 *l2_pt = mmu_info->l2_pts[l1_idx];
			u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);

			if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)
				continue;

			dev_dbg(mmu_info->dev,
				"\tl2 entry %u; iova 0x%8.8x, phys %pa\n",
				l2_idx, iova2,
				TBL_PHYS_ADDR(l2_pt[l2_idx]));
		}
	}

	dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");
}
#endif /* DEBUG */

static dma_addr_t map_single(struct ipu6_mmu_info *mmu_info, void *ptr)
{ … }

static int get_dummy_page(struct ipu6_mmu_info *mmu_info)
{ … }

static void free_dummy_page(struct ipu6_mmu_info *mmu_info)
{ … }

static int alloc_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
{ … }

static void free_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
{ … }

static u32 *alloc_l1_pt(struct ipu6_mmu_info *mmu_info)
{ … }

static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info)
{ … }

static int l2_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
		  phys_addr_t paddr, size_t size)
{ … }

static int __ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
			  phys_addr_t paddr, size_t size)
{ … }

static size_t l2_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
		       phys_addr_t dummy, size_t size)
{ … }

static size_t __ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info,
			       unsigned long iova, size_t size)
{ … }

static int allocate_trash_buffer(struct ipu6_mmu *mmu)
{ … }

int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp)
{ … }

void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)
{ … }

phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info,
				  dma_addr_t iova)
{ … }

static size_t ipu6_mmu_pgsize(unsigned long pgsize_bitmap,
			      unsigned long addr_merge, size_t size)
{ … }

size_t ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
		      size_t size)
{ … }

int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
		 phys_addr_t paddr, size_t size)
{ … }

static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)
{ … }

struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
			       void __iomem *base, int mmid,
			       const struct ipu6_hw_variants *hw)
{ … }

void ipu6_mmu_cleanup(struct ipu6_mmu *mmu)
{ … }