// SPDX-License-Identifier: GPL-2.0
/*
* iommu.c: IOMMU specific routines for memory management.
*
* Copyright (C) 1995 David S. Miller ([email protected])
* Copyright (C) 1995,2002 Pete Zaitcev ([email protected])
* Copyright (C) 1996 Eddie C. Dost ([email protected])
* Copyright (C) 1997,1998 Jakub Jelinek ([email protected])
*/
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/dma-map-ops.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <asm/io.h>
#include <asm/mxcc.h>
#include <asm/mbus.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/bitext.h>
#include <asm/iommu.h>
#include <asm/dma.h>
#include "mm_32.h"
/*
* This can be sized dynamically, but we will do this
* only when we have a guidance about actual I/O pressures.
*/
#define IOMMU_RNGE IOMMU_RNGE_256MB
#define IOMMU_START 0xF0000000
#define IOMMU_WINSIZE (256*1024*1024U)
#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
static int viking_flush;
/* viking.S */
extern void viking_flush_page(unsigned long page);
extern void viking_mxcc_flush_page(unsigned long page);
/*
* Values precomputed according to CPU type.
*/
static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
static pgprot_t dvma_prot; /* Consistent mapping pte flags */
#define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
static void __init sbus_iommu_init(struct platform_device *op)
{
struct iommu_struct *iommu;
unsigned int impl, vers;
unsigned long *bitmap;
unsigned long control;
unsigned long base;
unsigned long tmp;
iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
if (!iommu) {
prom_printf("Unable to allocate iommu structure\n");
prom_halt();
}
iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
"iommu_regs");
if (!iommu->regs) {
prom_printf("Cannot map IOMMU registers\n");
prom_halt();
}
control = sbus_readl(&iommu->regs->control);
impl = (control & IOMMU_CTRL_IMPL) >> 28;
vers = (control & IOMMU_CTRL_VERS) >> 24;
control &= ~(IOMMU_CTRL_RNGE);
control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
sbus_writel(control, &iommu->regs->control);
iommu_invalidate(iommu->regs);
iommu->start = IOMMU_START;
iommu->end = 0xffffffff;
/* Allocate IOMMU page table */
/* Stupid alignment constraints give me a headache.
We need 256K or 512K or 1M or 2M area aligned to
its size and current gfp will fortunately give
it to us. */
tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
if (!tmp) {
prom_printf("Unable to allocate iommu table [0x%lx]\n",
IOMMU_NPTES * sizeof(iopte_t));
prom_halt();
}
iommu->page_table = (iopte_t *)tmp;
/* Initialize new table. */
memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
flush_cache_all();
flush_tlb_all();
base = __pa((unsigned long)iommu->page_table) >> 4;
sbus_writel(base, &iommu->regs->base);
iommu_invalidate(iommu->regs);
bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
if (!bitmap) {
prom_printf("Unable to allocate iommu bitmap [%d]\n",
(int)(IOMMU_NPTES>>3));
prom_halt();
}
bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
/* To be coherent on HyperSparc, the page color of DVMA
* and physical addresses must match.
*/
if (srmmu_modtype == HyperSparc)
iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
else
iommu->usemap.num_colors = 1;
printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
impl, vers, iommu->page_table,
(int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
op->dev.archdata.iommu = iommu;
if (flush_page_for_dma_global)
op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
else
op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
}
static int __init iommu_init(void)
{
struct device_node *dp;
for_each_node_by_name(dp, "iommu") {
struct platform_device *op = of_find_device_by_node(dp);
sbus_iommu_init(op);
of_propagate_archdata(op);
}
return 0;
}
subsys_initcall(iommu_init);
/* Flush the iotlb entries to ram. */
/* This could be better if we didn't have to flush whole pages. */
static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
{
unsigned long start;
unsigned long end;
start = (unsigned long)iopte;
end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
start &= PAGE_MASK;
if (viking_mxcc_present) {
while(start < end) {
viking_mxcc_flush_page(start);
start += PAGE_SIZE;
}
} else if (viking_flush) {
while(start < end) {
viking_flush_page(start);
start += PAGE_SIZE;
}
} else {
while(start < end) {
__flush_page_to_ram(start);
start += PAGE_SIZE;
}
}
}
static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t len, bool per_page_flush)
{
struct iommu_struct *iommu = dev->archdata.iommu;
phys_addr_t paddr = page_to_phys(page) + offset;
unsigned long off = paddr & ~PAGE_MASK;
unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
unsigned long pfn = __phys_to_pfn(paddr);
unsigned int busa, busa0;
iopte_t *iopte, *iopte0;
int ioptex, i;
/* XXX So what is maxphys for us and how do drivers know it? */
if (!len || len > 256 * 1024)
return DMA_MAPPING_ERROR;
/*
* We expect unmapped highmem pages to be not in the cache.
* XXX Is this a good assumption?
* XXX What if someone else unmaps it here and races us?
*/
if (per_page_flush && !PageHighMem(page)) {
unsigned long vaddr, p;
vaddr = (unsigned long)page_address(page) + offset;
for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
flush_page_for_dma(p);
}
/* page color = pfn of page */
ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
if (ioptex < 0)
panic("iommu out");
busa0 = iommu->start + (ioptex << PAGE_SHIFT);
iopte0 = &iommu->page_table[ioptex];
busa = busa0;
iopte = iopte0;
for (i = 0; i < npages; i++) {
iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
iommu_invalidate_page(iommu->regs, busa);
busa += PAGE_SIZE;
iopte++;
pfn++;
}
iommu_flush_iotlb(iopte0, npages);
return busa0 + off;
}
static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
struct page *page, unsigned long offset, size_t len,
enum dma_data_direction dir, unsigned long attrs)
{
flush_page_for_dma(0);
return __sbus_iommu_map_page(dev, page, offset, len, false);
}
static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
struct page *page, unsigned long offset, size_t len,
enum dma_data_direction dir, unsigned long attrs)
{
return __sbus_iommu_map_page(dev, page, offset, len, true);
}
static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs,
bool per_page_flush)
{
struct scatterlist *sg;
int j;
for_each_sg(sgl, sg, nents, j) {
sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
sg->offset, sg->length, per_page_flush);
if (sg->dma_address == DMA_MAPPING_ERROR)
return -EIO;
sg->dma_length = sg->length;
}
return nents;
}
static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
flush_page_for_dma(0);
return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
}
static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
}
static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
size_t len, enum dma_data_direction dir, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
unsigned int busa = dma_addr & PAGE_MASK;
unsigned long off = dma_addr & ~PAGE_MASK;
unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
unsigned int i;
BUG_ON(busa < iommu->start);
for (i = 0; i < npages; i++) {
iopte_val(iommu->page_table[ioptex + i]) = 0;
iommu_invalidate_page(iommu->regs, busa);
busa += PAGE_SIZE;
}
bit_map_clear(&iommu->usemap, ioptex, npages);
}
static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i) {
sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
attrs);
sg->dma_address = 0x21212121;
}
}
#ifdef CONFIG_SBUS
static void *sbus_iommu_alloc(struct device *dev, size_t len,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
unsigned long va, addr, page, end, ret;
iopte_t *iopte = iommu->page_table;
iopte_t *first;
int ioptex;
/* XXX So what is maxphys for us and how do drivers know it? */
if (!len || len > 256 * 1024)
return NULL;
len = PAGE_ALIGN(len);
va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
if (va == 0)
return NULL;
addr = ret = sparc_dma_alloc_resource(dev, len);
if (!addr)
goto out_free_pages;
BUG_ON((va & ~PAGE_MASK) != 0);
BUG_ON((addr & ~PAGE_MASK) != 0);
BUG_ON((len & ~PAGE_MASK) != 0);
/* page color = physical address */
ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
addr >> PAGE_SHIFT);
if (ioptex < 0)
panic("iommu out");
iopte += ioptex;
first = iopte;
end = addr + len;
while(addr < end) {
page = va;
{
pmd_t *pmdp;
pte_t *ptep;
if (viking_mxcc_present)
viking_mxcc_flush_page(page);
else if (viking_flush)
viking_flush_page(page);
else
__flush_page_to_ram(page);
pmdp = pmd_off_k(addr);
ptep = pte_offset_kernel(pmdp, addr);
set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
}
iopte_val(*iopte++) =
MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
addr += PAGE_SIZE;
va += PAGE_SIZE;
}
/* P3: why do we need this?
*
* DAVEM: Because there are several aspects, none of which
* are handled by a single interface. Some cpus are
* completely not I/O DMA coherent, and some have
* virtually indexed caches. The driver DMA flushing
* methods handle the former case, but here during
* IOMMU page table modifications, and usage of non-cacheable
* cpu mappings of pages potentially in the cpu caches, we have
* to handle the latter case as well.
*/
flush_cache_all();
iommu_flush_iotlb(first, len >> PAGE_SHIFT);
flush_tlb_all();
iommu_invalidate(iommu->regs);
*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
return (void *)ret;
out_free_pages:
free_pages(va, get_order(len));
return NULL;
}
static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
dma_addr_t busa, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
iopte_t *iopte = iommu->page_table;
struct page *page = virt_to_page(cpu_addr);
int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
unsigned long end;
if (!sparc_dma_free_resource(cpu_addr, len))
return;
BUG_ON((busa & ~PAGE_MASK) != 0);
BUG_ON((len & ~PAGE_MASK) != 0);
iopte += ioptex;
end = busa + len;
while (busa < end) {
iopte_val(*iopte++) = 0;
busa += PAGE_SIZE;
}
flush_tlb_all();
iommu_invalidate(iommu->regs);
bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
__free_pages(page, get_order(len));
}
#endif
static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
#ifdef CONFIG_SBUS
.alloc = sbus_iommu_alloc,
.free = sbus_iommu_free,
#endif
.map_page = sbus_iommu_map_page_gflush,
.unmap_page = sbus_iommu_unmap_page,
.map_sg = sbus_iommu_map_sg_gflush,
.unmap_sg = sbus_iommu_unmap_sg,
};
static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
#ifdef CONFIG_SBUS
.alloc = sbus_iommu_alloc,
.free = sbus_iommu_free,
#endif
.map_page = sbus_iommu_map_page_pflush,
.unmap_page = sbus_iommu_unmap_page,
.map_sg = sbus_iommu_map_sg_pflush,
.unmap_sg = sbus_iommu_unmap_sg,
};
void __init ld_mmu_iommu(void)
{
if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
} else {
dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
}
}
Codebase Browser
linux