/* SPDX-License-Identifier: GPL-2.0-only */
/*
* IOMMU API for ARM architected SMMUv3 implementations.
*
* Copyright (C) 2015 ARM Limited
*/
#ifndef _ARM_SMMU_V3_H
#define _ARM_SMMU_V3_H
#include <linux/bitfield.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
#include <linux/mmzone.h>
#include <linux/sizes.h>
struct arm_smmu_device;
/* MMIO registers */
#define ARM_SMMU_IDR0 0x0
#define IDR0_ST_LVL GENMASK(28, 27)
#define IDR0_ST_LVL_2LVL 1
#define IDR0_STALL_MODEL GENMASK(25, 24)
#define IDR0_STALL_MODEL_STALL 0
#define IDR0_STALL_MODEL_FORCE 2
#define IDR0_TTENDIAN GENMASK(22, 21)
#define IDR0_TTENDIAN_MIXED 0
#define IDR0_TTENDIAN_LE 2
#define IDR0_TTENDIAN_BE 3
#define IDR0_CD2L (1 << 19)
#define IDR0_VMID16 (1 << 18)
#define IDR0_PRI (1 << 16)
#define IDR0_SEV (1 << 14)
#define IDR0_MSI (1 << 13)
#define IDR0_ASID16 (1 << 12)
#define IDR0_ATS (1 << 10)
#define IDR0_HYP (1 << 9)
#define IDR0_HTTU GENMASK(7, 6)
#define IDR0_HTTU_ACCESS 1
#define IDR0_HTTU_ACCESS_DIRTY 2
#define IDR0_COHACC (1 << 4)
#define IDR0_TTF GENMASK(3, 2)
#define IDR0_TTF_AARCH64 2
#define IDR0_TTF_AARCH32_64 3
#define IDR0_S1P (1 << 1)
#define IDR0_S2P (1 << 0)
#define ARM_SMMU_IDR1 0x4
#define IDR1_TABLES_PRESET (1 << 30)
#define IDR1_QUEUES_PRESET (1 << 29)
#define IDR1_REL (1 << 28)
#define IDR1_ATTR_TYPES_OVR (1 << 27)
#define IDR1_CMDQS GENMASK(25, 21)
#define IDR1_EVTQS GENMASK(20, 16)
#define IDR1_PRIQS GENMASK(15, 11)
#define IDR1_SSIDSIZE GENMASK(10, 6)
#define IDR1_SIDSIZE GENMASK(5, 0)
#define ARM_SMMU_IDR3 0xc
#define IDR3_RIL (1 << 10)
#define ARM_SMMU_IDR5 0x14
#define IDR5_STALL_MAX GENMASK(31, 16)
#define IDR5_GRAN64K (1 << 6)
#define IDR5_GRAN16K (1 << 5)
#define IDR5_GRAN4K (1 << 4)
#define IDR5_OAS GENMASK(2, 0)
#define IDR5_OAS_32_BIT 0
#define IDR5_OAS_36_BIT 1
#define IDR5_OAS_40_BIT 2
#define IDR5_OAS_42_BIT 3
#define IDR5_OAS_44_BIT 4
#define IDR5_OAS_48_BIT 5
#define IDR5_OAS_52_BIT 6
#define IDR5_VAX GENMASK(11, 10)
#define IDR5_VAX_52_BIT 1
#define ARM_SMMU_IIDR 0x18
#define IIDR_PRODUCTID GENMASK(31, 20)
#define IIDR_VARIANT GENMASK(19, 16)
#define IIDR_REVISION GENMASK(15, 12)
#define IIDR_IMPLEMENTER GENMASK(11, 0)
#define ARM_SMMU_CR0 0x20
#define CR0_ATSCHK (1 << 4)
#define CR0_CMDQEN (1 << 3)
#define CR0_EVTQEN (1 << 2)
#define CR0_PRIQEN (1 << 1)
#define CR0_SMMUEN (1 << 0)
#define ARM_SMMU_CR0ACK 0x24
#define ARM_SMMU_CR1 0x28
#define CR1_TABLE_SH GENMASK(11, 10)
#define CR1_TABLE_OC GENMASK(9, 8)
#define CR1_TABLE_IC GENMASK(7, 6)
#define CR1_QUEUE_SH GENMASK(5, 4)
#define CR1_QUEUE_OC GENMASK(3, 2)
#define CR1_QUEUE_IC GENMASK(1, 0)
/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
#define CR1_CACHE_NC 0
#define CR1_CACHE_WB 1
#define CR1_CACHE_WT 2
#define ARM_SMMU_CR2 0x2c
#define CR2_PTM (1 << 2)
#define CR2_RECINVSID (1 << 1)
#define CR2_E2H (1 << 0)
#define ARM_SMMU_GBPA 0x44
#define GBPA_UPDATE (1 << 31)
#define GBPA_ABORT (1 << 20)
#define ARM_SMMU_IRQ_CTRL 0x50
#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
#define ARM_SMMU_IRQ_CTRLACK 0x54
#define ARM_SMMU_GERROR 0x60
#define GERROR_SFM_ERR (1 << 8)
#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
#define GERROR_PRIQ_ABT_ERR (1 << 3)
#define GERROR_EVTQ_ABT_ERR (1 << 2)
#define GERROR_CMDQ_ERR (1 << 0)
#define GERROR_ERR_MASK 0x1fd
#define ARM_SMMU_GERRORN 0x64
#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
#define ARM_SMMU_STRTAB_BASE 0x80
#define STRTAB_BASE_RA (1UL << 62)
#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
#define ARM_SMMU_STRTAB_BASE_CFG 0x88
#define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
#define STRTAB_BASE_CFG_FMT_LINEAR 0
#define STRTAB_BASE_CFG_FMT_2LVL 1
#define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
#define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
#define ARM_SMMU_CMDQ_BASE 0x90
#define ARM_SMMU_CMDQ_PROD 0x98
#define ARM_SMMU_CMDQ_CONS 0x9c
#define ARM_SMMU_EVTQ_BASE 0xa0
#define ARM_SMMU_EVTQ_PROD 0xa8
#define ARM_SMMU_EVTQ_CONS 0xac
#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
#define ARM_SMMU_PRIQ_BASE 0xc0
#define ARM_SMMU_PRIQ_PROD 0xc8
#define ARM_SMMU_PRIQ_CONS 0xcc
#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
#define ARM_SMMU_REG_SZ 0xe00
/* Common MSI config fields */
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
#define MSI_CFG2_SH GENMASK(5, 4)
#define MSI_CFG2_MEMATTR GENMASK(3, 0)
/* Common memory attribute values */
#define ARM_SMMU_SH_NSH 0
#define ARM_SMMU_SH_OSH 2
#define ARM_SMMU_SH_ISH 3
#define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
#define ARM_SMMU_MEMATTR_OIWB 0xf
#define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
#define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
#define Q_OVERFLOW_FLAG (1U << 31)
#define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
#define Q_ENT(q, p) ((q)->base + \
Q_IDX(&((q)->llq), p) * \
(q)->ent_dwords)
#define Q_BASE_RWA (1UL << 62)
#define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
#define Q_BASE_LOG2SIZE GENMASK(4, 0)
/* Ensure DMA allocations are naturally aligned */
#ifdef CONFIG_CMA_ALIGNMENT
#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
#else
#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_PAGE_ORDER)
#endif
/*
* Stream table.
*
* Linear: Enough to cover 1 << IDR1.SIDSIZE entries
* 2lvl: 128k L1 entries,
* 256 lazy entries per table (each table covers a PCI bus)
*/
#define STRTAB_SPLIT 8
#define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
#define STRTAB_STE_DWORDS 8
struct arm_smmu_ste {
__le64 data[STRTAB_STE_DWORDS];
};
#define STRTAB_NUM_L2_STES (1 << STRTAB_SPLIT)
struct arm_smmu_strtab_l2 {
struct arm_smmu_ste stes[STRTAB_NUM_L2_STES];
};
struct arm_smmu_strtab_l1 {
__le64 l2ptr;
};
#define STRTAB_MAX_L1_ENTRIES (1 << 17)
static inline u32 arm_smmu_strtab_l1_idx(u32 sid)
{
return sid / STRTAB_NUM_L2_STES;
}
static inline u32 arm_smmu_strtab_l2_idx(u32 sid)
{
return sid % STRTAB_NUM_L2_STES;
}
#define STRTAB_STE_0_V (1UL << 0)
#define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
#define STRTAB_STE_0_CFG_ABORT 0
#define STRTAB_STE_0_CFG_BYPASS 4
#define STRTAB_STE_0_CFG_S1_TRANS 5
#define STRTAB_STE_0_CFG_S2_TRANS 6
#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
#define STRTAB_STE_0_S1FMT_LINEAR 0
#define STRTAB_STE_0_S1FMT_64K_L2 2
#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
#define STRTAB_STE_1_S1DSS_BYPASS 0x1
#define STRTAB_STE_1_S1DSS_SSID0 0x2
#define STRTAB_STE_1_S1C_CACHE_NC 0UL
#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
#define STRTAB_STE_1_S1C_CACHE_WT 2UL
#define STRTAB_STE_1_S1C_CACHE_WB 3UL
#define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
#define STRTAB_STE_1_S1STALLD (1UL << 27)
#define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
#define STRTAB_STE_1_EATS_ABT 0UL
#define STRTAB_STE_1_EATS_TRANS 1UL
#define STRTAB_STE_1_EATS_S1CHK 2UL
#define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
#define STRTAB_STE_1_STRW_NSEL1 0UL
#define STRTAB_STE_1_STRW_EL2 2UL
#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
#define STRTAB_STE_1_SHCFG_INCOMING 1UL
#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
#define STRTAB_STE_2_S2AA64 (1UL << 51)
#define STRTAB_STE_2_S2ENDI (1UL << 52)
#define STRTAB_STE_2_S2PTW (1UL << 54)
#define STRTAB_STE_2_S2S (1UL << 57)
#define STRTAB_STE_2_S2R (1UL << 58)
#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
/*
* Context descriptors.
*
* Linear: when less than 1024 SSIDs are supported
* 2lvl: at most 1024 L1 entries,
* 1024 lazy entries per table.
*/
#define CTXDESC_L2_ENTRIES 1024
#define CTXDESC_L1_DESC_V (1UL << 0)
#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
#define CTXDESC_CD_DWORDS 8
struct arm_smmu_cd {
__le64 data[CTXDESC_CD_DWORDS];
};
struct arm_smmu_cdtab_l2 {
struct arm_smmu_cd cds[CTXDESC_L2_ENTRIES];
};
struct arm_smmu_cdtab_l1 {
__le64 l2ptr;
};
static inline unsigned int arm_smmu_cdtab_l1_idx(unsigned int ssid)
{
return ssid / CTXDESC_L2_ENTRIES;
}
static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
{
return ssid % CTXDESC_L2_ENTRIES;
}
#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
#define CTXDESC_CD_0_ENDI (1UL << 15)
#define CTXDESC_CD_0_V (1UL << 31)
#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
#define CTXDESC_CD_0_TCR_HA (1UL << 43)
#define CTXDESC_CD_0_TCR_HD (1UL << 42)
#define CTXDESC_CD_0_AA64 (1UL << 41)
#define CTXDESC_CD_0_S (1UL << 44)
#define CTXDESC_CD_0_R (1UL << 45)
#define CTXDESC_CD_0_A (1UL << 46)
#define CTXDESC_CD_0_ASET (1UL << 47)
#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
/*
* When the SMMU only supports linear context descriptor tables, pick a
* reasonable size limit (64kB).
*/
#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / sizeof(struct arm_smmu_cd))
/* Command queue */
#define CMDQ_ENT_SZ_SHIFT 4
#define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
#define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
#define CMDQ_CONS_ERR GENMASK(30, 24)
#define CMDQ_ERR_CERROR_NONE_IDX 0
#define CMDQ_ERR_CERROR_ILL_IDX 1
#define CMDQ_ERR_CERROR_ABT_IDX 2
#define CMDQ_ERR_CERROR_ATC_INV_IDX 3
#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
/*
* This is used to size the command queue and therefore must be at least
* BITS_PER_LONG so that the valid_map works correctly (it relies on the
* total number of queue entries being a multiple of BITS_PER_LONG).
*/
#define CMDQ_BATCH_ENTRIES BITS_PER_LONG
#define CMDQ_0_OP GENMASK_ULL(7, 0)
#define CMDQ_0_SSV (1UL << 11)
#define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
#define CMDQ_CFGI_1_LEAF (1UL << 0)
#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
#define CMDQ_TLBI_RANGE_NUM_MAX 31
#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
#define CMDQ_TLBI_1_LEAF (1UL << 0)
#define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
#define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
#define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
#define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
#define CMDQ_ATC_0_GLOBAL (1UL << 9)
#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
#define CMDQ_RESUME_0_RESP_TERM 0UL
#define CMDQ_RESUME_0_RESP_RETRY 1UL
#define CMDQ_RESUME_0_RESP_ABORT 2UL
#define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12)
#define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32)
#define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0)
#define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
#define CMDQ_SYNC_0_CS_NONE 0
#define CMDQ_SYNC_0_CS_IRQ 1
#define CMDQ_SYNC_0_CS_SEV 2
#define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
#define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
/* Event queue */
#define EVTQ_ENT_SZ_SHIFT 5
#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
#define EVTQ_0_ID GENMASK_ULL(7, 0)
#define EVT_ID_TRANSLATION_FAULT 0x10
#define EVT_ID_ADDR_SIZE_FAULT 0x11
#define EVT_ID_ACCESS_FAULT 0x12
#define EVT_ID_PERMISSION_FAULT 0x13
#define EVTQ_0_SSV (1UL << 11)
#define EVTQ_0_SSID GENMASK_ULL(31, 12)
#define EVTQ_0_SID GENMASK_ULL(63, 32)
#define EVTQ_1_STAG GENMASK_ULL(15, 0)
#define EVTQ_1_STALL (1UL << 31)
#define EVTQ_1_PnU (1UL << 33)
#define EVTQ_1_InD (1UL << 34)
#define EVTQ_1_RnW (1UL << 35)
#define EVTQ_1_S2 (1UL << 39)
#define EVTQ_1_CLASS GENMASK_ULL(41, 40)
#define EVTQ_1_TT_READ (1UL << 44)
#define EVTQ_2_ADDR GENMASK_ULL(63, 0)
#define EVTQ_3_IPA GENMASK_ULL(51, 12)
/* PRI queue */
#define PRIQ_ENT_SZ_SHIFT 4
#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
#define PRIQ_0_SID GENMASK_ULL(31, 0)
#define PRIQ_0_SSID GENMASK_ULL(51, 32)
#define PRIQ_0_PERM_PRIV (1UL << 58)
#define PRIQ_0_PERM_EXEC (1UL << 59)
#define PRIQ_0_PERM_READ (1UL << 60)
#define PRIQ_0_PERM_WRITE (1UL << 61)
#define PRIQ_0_PRG_LAST (1UL << 62)
#define PRIQ_0_SSID_V (1UL << 63)
#define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
#define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
/* High-level queue structures */
#define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
#define ARM_SMMU_POLL_SPIN_COUNT 10
#define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000
enum pri_resp {
PRI_RESP_DENY = 0,
PRI_RESP_FAIL = 1,
PRI_RESP_SUCC = 2,
};
struct arm_smmu_cmdq_ent {
/* Common fields */
u8 opcode;
bool substream_valid;
/* Command-specific fields */
union {
#define CMDQ_OP_PREFETCH_CFG 0x1
struct {
u32 sid;
} prefetch;
#define CMDQ_OP_CFGI_STE 0x3
#define CMDQ_OP_CFGI_ALL 0x4
#define CMDQ_OP_CFGI_CD 0x5
#define CMDQ_OP_CFGI_CD_ALL 0x6
struct {
u32 sid;
u32 ssid;
union {
bool leaf;
u8 span;
};
} cfgi;
#define CMDQ_OP_TLBI_NH_ASID 0x11
#define CMDQ_OP_TLBI_NH_VA 0x12
#define CMDQ_OP_TLBI_EL2_ALL 0x20
#define CMDQ_OP_TLBI_EL2_ASID 0x21
#define CMDQ_OP_TLBI_EL2_VA 0x22
#define CMDQ_OP_TLBI_S12_VMALL 0x28
#define CMDQ_OP_TLBI_S2_IPA 0x2a
#define CMDQ_OP_TLBI_NSNH_ALL 0x30
struct {
u8 num;
u8 scale;
u16 asid;
u16 vmid;
bool leaf;
u8 ttl;
u8 tg;
u64 addr;
} tlbi;
#define CMDQ_OP_ATC_INV 0x40
#define ATC_INV_SIZE_ALL 52
struct {
u32 sid;
u32 ssid;
u64 addr;
u8 size;
bool global;
} atc;
#define CMDQ_OP_PRI_RESP 0x41
struct {
u32 sid;
u32 ssid;
u16 grpid;
enum pri_resp resp;
} pri;
#define CMDQ_OP_RESUME 0x44
struct {
u32 sid;
u16 stag;
u8 resp;
} resume;
#define CMDQ_OP_CMD_SYNC 0x46
struct {
u64 msiaddr;
} sync;
};
};
struct arm_smmu_ll_queue {
union {
u64 val;
struct {
u32 prod;
u32 cons;
};
struct {
atomic_t prod;
atomic_t cons;
} atomic;
u8 __pad[SMP_CACHE_BYTES];
} ____cacheline_aligned_in_smp;
u32 max_n_shift;
};
struct arm_smmu_queue {
struct arm_smmu_ll_queue llq;
int irq; /* Wired interrupt */
__le64 *base;
dma_addr_t base_dma;
u64 q_base;
size_t ent_dwords;
u32 __iomem *prod_reg;
u32 __iomem *cons_reg;
};
struct arm_smmu_queue_poll {
ktime_t timeout;
unsigned int delay;
unsigned int spin_cnt;
bool wfe;
};
struct arm_smmu_cmdq {
struct arm_smmu_queue q;
atomic_long_t *valid_map;
atomic_t owner_prod;
atomic_t lock;
bool (*supports_cmd)(struct arm_smmu_cmdq_ent *ent);
};
static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmdq_ent *ent)
{
return cmdq->supports_cmd ? cmdq->supports_cmd(ent) : true;
}
struct arm_smmu_cmdq_batch {
u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
struct arm_smmu_cmdq *cmdq;
int num;
};
struct arm_smmu_evtq {
struct arm_smmu_queue q;
struct iopf_queue *iopf;
u32 max_stalls;
};
struct arm_smmu_priq {
struct arm_smmu_queue q;
};
/* High-level stream table and context descriptor structures */
struct arm_smmu_ctx_desc {
u16 asid;
};
struct arm_smmu_ctx_desc_cfg {
union {
struct {
struct arm_smmu_cd *table;
unsigned int num_ents;
} linear;
struct {
struct arm_smmu_cdtab_l1 *l1tab;
struct arm_smmu_cdtab_l2 **l2ptrs;
unsigned int num_l1_ents;
} l2;
};
dma_addr_t cdtab_dma;
unsigned int used_ssids;
u8 in_ste;
u8 s1fmt;
/* log2 of the maximum number of CDs supported by this table */
u8 s1cdmax;
};
static inline bool
arm_smmu_cdtab_allocated(struct arm_smmu_ctx_desc_cfg *cfg)
{
return cfg->linear.table || cfg->l2.l1tab;
}
/* True if the cd table has SSIDS > 0 in use. */
static inline bool arm_smmu_ssids_in_use(struct arm_smmu_ctx_desc_cfg *cd_table)
{
return cd_table->used_ssids;
}
struct arm_smmu_s2_cfg {
u16 vmid;
};
struct arm_smmu_strtab_cfg {
union {
struct {
struct arm_smmu_ste *table;
dma_addr_t ste_dma;
unsigned int num_ents;
} linear;
struct {
struct arm_smmu_strtab_l1 *l1tab;
struct arm_smmu_strtab_l2 **l2ptrs;
dma_addr_t l1_dma;
unsigned int num_l1_ents;
} l2;
};
};
struct arm_smmu_impl_ops {
int (*device_reset)(struct arm_smmu_device *smmu);
void (*device_remove)(struct arm_smmu_device *smmu);
int (*init_structures)(struct arm_smmu_device *smmu);
struct arm_smmu_cmdq *(*get_secondary_cmdq)(
struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent);
};
/* An SMMUv3 instance */
struct arm_smmu_device {
struct device *dev;
struct device *impl_dev;
const struct arm_smmu_impl_ops *impl_ops;
void __iomem *base;
void __iomem *page1;
#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
#define ARM_SMMU_FEAT_TT_LE (1 << 2)
#define ARM_SMMU_FEAT_TT_BE (1 << 3)
#define ARM_SMMU_FEAT_PRI (1 << 4)
#define ARM_SMMU_FEAT_ATS (1 << 5)
#define ARM_SMMU_FEAT_SEV (1 << 6)
#define ARM_SMMU_FEAT_MSI (1 << 7)
#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
#define ARM_SMMU_FEAT_STALLS (1 << 11)
#define ARM_SMMU_FEAT_HYP (1 << 12)
#define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
#define ARM_SMMU_FEAT_VAX (1 << 14)
#define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
#define ARM_SMMU_FEAT_BTM (1 << 16)
#define ARM_SMMU_FEAT_SVA (1 << 17)
#define ARM_SMMU_FEAT_E2H (1 << 18)
#define ARM_SMMU_FEAT_NESTING (1 << 19)
#define ARM_SMMU_FEAT_ATTR_TYPES_OVR (1 << 20)
#define ARM_SMMU_FEAT_HA (1 << 21)
#define ARM_SMMU_FEAT_HD (1 << 22)
u32 features;
#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
#define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
#define ARM_SMMU_OPT_MSIPOLL (1 << 2)
#define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3)
#define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4)
u32 options;
struct arm_smmu_cmdq cmdq;
struct arm_smmu_evtq evtq;
struct arm_smmu_priq priq;
int gerr_irq;
int combined_irq;
unsigned long ias; /* IPA */
unsigned long oas; /* PA */
unsigned long pgsize_bitmap;
#define ARM_SMMU_MAX_ASIDS (1 << 16)
unsigned int asid_bits;
#define ARM_SMMU_MAX_VMIDS (1 << 16)
unsigned int vmid_bits;
struct ida vmid_map;
unsigned int ssid_bits;
unsigned int sid_bits;
struct arm_smmu_strtab_cfg strtab_cfg;
/* IOMMU core code handle */
struct iommu_device iommu;
struct rb_root streams;
struct mutex streams_mutex;
};
struct arm_smmu_stream {
u32 id;
struct arm_smmu_master *master;
struct rb_node node;
};
/* SMMU private data for each master */
struct arm_smmu_master {
struct arm_smmu_device *smmu;
struct device *dev;
struct arm_smmu_stream *streams;
/* Locked by the iommu core using the group mutex */
struct arm_smmu_ctx_desc_cfg cd_table;
unsigned int num_streams;
bool ats_enabled : 1;
bool ste_ats_enabled : 1;
bool stall_enabled;
bool sva_enabled;
bool iopf_enabled;
unsigned int ssid_bits;
};
/* SMMU private data for an IOMMU domain */
enum arm_smmu_domain_stage {
ARM_SMMU_DOMAIN_S1 = 0,
ARM_SMMU_DOMAIN_S2,
};
struct arm_smmu_domain {
struct arm_smmu_device *smmu;
struct mutex init_mutex; /* Protects smmu pointer */
struct io_pgtable_ops *pgtbl_ops;
atomic_t nr_ats_masters;
enum arm_smmu_domain_stage stage;
union {
struct arm_smmu_ctx_desc cd;
struct arm_smmu_s2_cfg s2_cfg;
};
struct iommu_domain domain;
/* List of struct arm_smmu_master_domain */
struct list_head devices;
spinlock_t devices_lock;
struct mmu_notifier mmu_notifier;
};
/* The following are exposed for testing purposes. */
struct arm_smmu_entry_writer_ops;
struct arm_smmu_entry_writer {
const struct arm_smmu_entry_writer_ops *ops;
struct arm_smmu_master *master;
};
struct arm_smmu_entry_writer_ops {
void (*get_used)(const __le64 *entry, __le64 *used);
void (*sync)(struct arm_smmu_entry_writer *writer);
};
#if IS_ENABLED(CONFIG_KUNIT)
void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
const __le64 *target);
void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);
void arm_smmu_make_abort_ste(struct arm_smmu_ste *target);
void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
struct arm_smmu_ste *target);
void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
struct arm_smmu_master *master, bool ats_enabled,
unsigned int s1dss);
void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain,
bool ats_enabled);
void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
struct arm_smmu_master *master, struct mm_struct *mm,
u16 asid);
#endif
struct arm_smmu_master_domain {
struct list_head devices_elm;
struct arm_smmu_master *master;
ioasid_t ssid;
};
static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
{
return container_of(dom, struct arm_smmu_domain, domain);
}
extern struct xarray arm_smmu_asid_xa;
extern struct mutex arm_smmu_asid_lock;
struct arm_smmu_domain *arm_smmu_domain_alloc(void);
void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid);
struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
u32 ssid);
void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain);
void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
struct arm_smmu_cd *cdptr,
const struct arm_smmu_cd *target);
int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
struct arm_smmu_cd *cd);
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,
struct arm_smmu_domain *smmu_domain);
int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
unsigned long iova, size_t size);
void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq);
int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
struct arm_smmu_queue *q, void __iomem *page,
unsigned long prod_off, unsigned long cons_off,
size_t dwords, const char *name);
int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq);
#ifdef CONFIG_ARM_SMMU_V3_SVA
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
void arm_smmu_sva_notifier_synchronize(void);
struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
struct mm_struct *mm);
#else /* CONFIG_ARM_SMMU_V3_SVA */
static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
{
return false;
}
static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
{
return false;
}
static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
{
return false;
}
static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
{
return -ENODEV;
}
static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
{
return -ENODEV;
}
static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
{
return false;
}
static inline void arm_smmu_sva_notifier_synchronize(void) {}
#define arm_smmu_sva_domain_alloc NULL
#endif /* CONFIG_ARM_SMMU_V3_SVA */
#ifdef CONFIG_TEGRA241_CMDQV
struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu);
#else /* CONFIG_TEGRA241_CMDQV */
static inline struct arm_smmu_device *
tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
{
return ERR_PTR(-ENODEV);
}
#endif /* CONFIG_TEGRA241_CMDQV */
#endif /* _ARM_SMMU_V3_H */
Codebase Browser
linux