linux/drivers/accel/ivpu/ivpu_mmu.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2020-2024 Intel Corporation
 */

#include <linux/circ_buf.h>
#include <linux/highmem.h>

#include "ivpu_drv.h"
#include "ivpu_hw.h"
#include "ivpu_hw_reg_io.h"
#include "ivpu_mmu.h"
#include "ivpu_mmu_context.h"
#include "ivpu_pm.h"

#define IVPU_MMU_REG_IDR0
#define IVPU_MMU_REG_IDR1
#define IVPU_MMU_REG_IDR3
#define IVPU_MMU_REG_IDR5
#define IVPU_MMU_REG_CR0
#define IVPU_MMU_REG_CR0ACK
#define IVPU_MMU_REG_CR0ACK_VAL_MASK
#define IVPU_MMU_REG_CR1
#define IVPU_MMU_REG_CR2
#define IVPU_MMU_REG_IRQ_CTRL
#define IVPU_MMU_REG_IRQ_CTRLACK
#define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK

#define IVPU_MMU_REG_GERROR
#define IVPU_MMU_REG_GERROR_CMDQ_MASK
#define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK
#define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK
#define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK
#define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK
#define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK
#define IVPU_MMU_REG_GERROR_MSI_ABT_MASK

#define IVPU_MMU_REG_GERRORN

#define IVPU_MMU_REG_STRTAB_BASE
#define IVPU_MMU_REG_STRTAB_BASE_CFG
#define IVPU_MMU_REG_CMDQ_BASE
#define IVPU_MMU_REG_CMDQ_PROD
#define IVPU_MMU_REG_CMDQ_CONS
#define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK
#define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK
#define IVPU_MMU_REG_EVTQ_BASE
#define IVPU_MMU_REG_EVTQ_PROD
#define IVPU_MMU_REG_EVTQ_CONS
#define IVPU_MMU_REG_EVTQ_PROD_SEC
#define IVPU_MMU_REG_EVTQ_CONS_SEC

#define IVPU_MMU_IDR0_REF
#define IVPU_MMU_IDR0_REF_SIMICS
#define IVPU_MMU_IDR1_REF
#define IVPU_MMU_IDR3_REF
#define IVPU_MMU_IDR5_REF
#define IVPU_MMU_IDR5_REF_SIMICS
#define IVPU_MMU_IDR5_REF_FPGA

#define IVPU_MMU_CDTAB_ENT_SIZE
#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2
#define IVPU_MMU_CDTAB_ENT_COUNT

#define IVPU_MMU_STREAM_ID0
#define IVPU_MMU_STREAM_ID3

#define IVPU_MMU_STRTAB_ENT_SIZE
#define IVPU_MMU_STRTAB_ENT_COUNT
#define IVPU_MMU_STRTAB_CFG_LOG2SIZE
#define IVPU_MMU_STRTAB_CFG

#define IVPU_MMU_Q_COUNT_LOG2
#define IVPU_MMU_Q_COUNT
#define IVPU_MMU_Q_WRAP_MASK
#define IVPU_MMU_Q_IDX_MASK
#define IVPU_MMU_Q_IDX(val)
#define IVPU_MMU_Q_WRP(val)

#define IVPU_MMU_CMDQ_CMD_SIZE
#define IVPU_MMU_CMDQ_SIZE

#define IVPU_MMU_EVTQ_CMD_SIZE
#define IVPU_MMU_EVTQ_SIZE

#define IVPU_MMU_CMD_OPCODE

#define IVPU_MMU_CMD_SYNC_0_CS
#define IVPU_MMU_CMD_SYNC_0_MSH
#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR
#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR
#define IVPU_MMU_CMD_SYNC_0_MSI_DATA

#define IVPU_MMU_CMD_CFGI_0_SSEC
#define IVPU_MMU_CMD_CFGI_0_SSV
#define IVPU_MMU_CMD_CFGI_0_SSID
#define IVPU_MMU_CMD_CFGI_0_SID
#define IVPU_MMU_CMD_CFGI_1_RANGE

#define IVPU_MMU_CMD_TLBI_0_ASID
#define IVPU_MMU_CMD_TLBI_0_VMID

#define CMD_PREFETCH_CFG
#define CMD_CFGI_STE
#define CMD_CFGI_ALL
#define CMD_CFGI_CD
#define CMD_CFGI_CD_ALL
#define CMD_TLBI_NH_ASID
#define CMD_TLBI_EL2_ALL
#define CMD_TLBI_NSNH_ALL
#define CMD_SYNC

#define IVPU_MMU_EVT_F_UUT
#define IVPU_MMU_EVT_C_BAD_STREAMID
#define IVPU_MMU_EVT_F_STE_FETCH
#define IVPU_MMU_EVT_C_BAD_STE
#define IVPU_MMU_EVT_F_BAD_ATS_TREQ
#define IVPU_MMU_EVT_F_STREAM_DISABLED
#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN
#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID
#define IVPU_MMU_EVT_F_CD_FETCH
#define IVPU_MMU_EVT_C_BAD_CD
#define IVPU_MMU_EVT_F_WALK_EABT
#define IVPU_MMU_EVT_F_TRANSLATION
#define IVPU_MMU_EVT_F_ADDR_SIZE
#define IVPU_MMU_EVT_F_ACCESS
#define IVPU_MMU_EVT_F_PERMISSION
#define IVPU_MMU_EVT_F_TLB_CONFLICT
#define IVPU_MMU_EVT_F_CFG_CONFLICT
#define IVPU_MMU_EVT_E_PAGE_REQUEST
#define IVPU_MMU_EVT_F_VMS_FETCH

#define IVPU_MMU_EVT_OP_MASK
#define IVPU_MMU_EVT_SSID_MASK

#define IVPU_MMU_Q_BASE_RWA
#define IVPU_MMU_Q_BASE_ADDR_MASK
#define IVPU_MMU_STRTAB_BASE_RA
#define IVPU_MMU_STRTAB_BASE_ADDR_MASK

#define IVPU_MMU_IRQ_EVTQ_EN
#define IVPU_MMU_IRQ_GERROR_EN

#define IVPU_MMU_CR0_ATSCHK
#define IVPU_MMU_CR0_CMDQEN
#define IVPU_MMU_CR0_EVTQEN
#define IVPU_MMU_CR0_PRIQEN
#define IVPU_MMU_CR0_SMMUEN

#define IVPU_MMU_CR1_TABLE_SH
#define IVPU_MMU_CR1_TABLE_OC
#define IVPU_MMU_CR1_TABLE_IC
#define IVPU_MMU_CR1_QUEUE_SH
#define IVPU_MMU_CR1_QUEUE_OC
#define IVPU_MMU_CR1_QUEUE_IC
#define IVPU_MMU_CACHE_NC
#define IVPU_MMU_CACHE_WB
#define IVPU_MMU_CACHE_WT
#define IVPU_MMU_SH_NSH
#define IVPU_MMU_SH_OSH
#define IVPU_MMU_SH_ISH

#define IVPU_MMU_CMDQ_OP

#define IVPU_MMU_CD_0_TCR_T0SZ
#define IVPU_MMU_CD_0_TCR_TG0
#define IVPU_MMU_CD_0_TCR_IRGN0
#define IVPU_MMU_CD_0_TCR_ORGN0
#define IVPU_MMU_CD_0_TCR_SH0
#define IVPU_MMU_CD_0_TCR_EPD0
#define IVPU_MMU_CD_0_TCR_EPD1
#define IVPU_MMU_CD_0_ENDI
#define IVPU_MMU_CD_0_V
#define IVPU_MMU_CD_0_TCR_IPS
#define IVPU_MMU_CD_0_TCR_TBI0
#define IVPU_MMU_CD_0_AA64
#define IVPU_MMU_CD_0_S
#define IVPU_MMU_CD_0_R
#define IVPU_MMU_CD_0_A
#define IVPU_MMU_CD_0_ASET
#define IVPU_MMU_CD_0_ASID

#define IVPU_MMU_T0SZ_48BIT
#define IVPU_MMU_T0SZ_38BIT

#define IVPU_MMU_IPS_48BIT
#define IVPU_MMU_IPS_44BIT
#define IVPU_MMU_IPS_42BIT
#define IVPU_MMU_IPS_40BIT
#define IVPU_MMU_IPS_36BIT
#define IVPU_MMU_IPS_32BIT

#define IVPU_MMU_CD_1_TTB0_MASK

#define IVPU_MMU_STE_0_S1CDMAX
#define IVPU_MMU_STE_0_S1FMT
#define IVPU_MMU_STE_0_S1FMT_LINEAR
#define IVPU_MMU_STE_DWORDS
#define IVPU_MMU_STE_0_CFG_S1_TRANS
#define IVPU_MMU_STE_0_CFG
#define IVPU_MMU_STE_0_S1CTXPTR_MASK
#define IVPU_MMU_STE_0_V

#define IVPU_MMU_STE_1_STRW_NSEL1
#define IVPU_MMU_STE_1_CONT
#define IVPU_MMU_STE_1_STRW
#define IVPU_MMU_STE_1_PRIVCFG
#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV
#define IVPU_MMU_STE_1_INSTCFG
#define IVPU_MMU_STE_1_INSTCFG_DATA
#define IVPU_MMU_STE_1_MEV
#define IVPU_MMU_STE_1_S1STALLD
#define IVPU_MMU_STE_1_S1C_CACHE_NC
#define IVPU_MMU_STE_1_S1C_CACHE_WBRA
#define IVPU_MMU_STE_1_S1C_CACHE_WT
#define IVPU_MMU_STE_1_S1C_CACHE_WB
#define IVPU_MMU_STE_1_S1CIR
#define IVPU_MMU_STE_1_S1COR
#define IVPU_MMU_STE_1_S1CSH
#define IVPU_MMU_STE_1_S1DSS
#define IVPU_MMU_STE_1_S1DSS_TERMINATE

#define IVPU_MMU_REG_TIMEOUT_US
#define IVPU_MMU_QUEUE_TIMEOUT_US

#define IVPU_MMU_GERROR_ERR_MASK

#define IVPU_MMU_CERROR_NONE
#define IVPU_MMU_CERROR_ILL
#define IVPU_MMU_CERROR_ABT
#define IVPU_MMU_CERROR_ATC_INV_SYNC

static const char *ivpu_mmu_event_to_str(u32 cmd)
{}

static const char *ivpu_mmu_cmdq_err_to_str(u32 err)
{}

static void ivpu_mmu_config_check(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
{}

static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
{}

static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
{}

static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
{}

static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)
{}

static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)
{}

static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
{}

static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q)
{}

static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q)
{}

static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
{}

static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
{}

static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
{}

static int ivpu_mmu_reset(struct ivpu_device *vdev)
{}

static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
{}

static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
{}

int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
{}

static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
{}

static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)
{}

static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)
{}

int ivpu_mmu_init(struct ivpu_device *vdev)
{}

int ivpu_mmu_enable(struct ivpu_device *vdev)
{}

void ivpu_mmu_disable(struct ivpu_device *vdev)
{}

static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)
{}

static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
{}

void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
{}

void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)
{}

void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
{}

int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)
{}

void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)
{}