// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2020-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #include "gaudi2P.h" #include "gaudi2_masks.h" #include "../include/gaudi2/gaudi2_special_blocks.h" #include "../include/hw_ip/mmu/mmu_general.h" #include "../include/hw_ip/mmu/mmu_v2_0.h" #include "../include/gaudi2/gaudi2_packets.h" #include "../include/gaudi2/gaudi2_reg_map.h" #include "../include/gaudi2/gaudi2_async_ids_map_extended.h" #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" #include <linux/module.h> #include <linux/pci.h> #include <linux/hwmon.h> #include <linux/iommu.h> #define GAUDI2_DMA_POOL_BLK_SIZE … #define GAUDI2_RESET_TIMEOUT_MSEC … #define GAUDI2_RESET_POLL_TIMEOUT_USEC … #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC … #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC … #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC … #define GAUDI2_RESET_POLL_CNT … #define GAUDI2_RESET_WAIT_MSEC … #define GAUDI2_CPU_RESET_WAIT_MSEC … #define GAUDI2_PLDM_RESET_WAIT_MSEC … #define GAUDI2_CB_POOL_CB_CNT … #define GAUDI2_CB_POOL_CB_SIZE … #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC … #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC … #define GAUDI2_TEST_QUEUE_WAIT_USEC … #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC … #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT … /* * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs * and the code relies on that value (for array size etc..) we define another value * for MAX faulty TPCs which reflects the cluster binning requirements */ #define MAX_CLUSTER_BINNING_FAULTY_TPCS … #define MAX_FAULTY_XBARS … #define MAX_FAULTY_EDMAS … #define MAX_FAULTY_DECODERS … #define GAUDI2_TPC_FULL_MASK … #define GAUDI2_HIF_HMMU_FULL_MASK … #define GAUDI2_DECODER_FULL_MASK … #define GAUDI2_NA_EVENT_CAUSE … #define GAUDI2_NUM_OF_QM_ERR_CAUSE … #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE … #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE … #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE … #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE … #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE … #define GAUDI2_NUM_OF_ROT_ERR_CAUSE … #define GAUDI2_NUM_OF_TPC_INTR_CAUSE … #define GAUDI2_NUM_OF_DEC_ERR_CAUSE … #define GAUDI2_NUM_OF_MME_ERR_CAUSE … #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE … #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE … #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE … #define GAUDI2_NUM_OF_HBM_SEI_CAUSE … #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE … #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE … #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE … #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE … #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE … #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE … #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC … #define GAUDI2_PLDM_MMU_TIMEOUT_USEC … #define GAUDI2_ARB_WDT_TIMEOUT … #define GAUDI2_VDEC_TIMEOUT_USEC … #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC … #define KDMA_TIMEOUT_USEC … #define IS_DMA_IDLE(dma_core_sts0) … #define IS_DMA_HALTED(dma_core_sts1) … #define IS_MME_IDLE(mme_arch_sts) … #define IS_TPC_IDLE(tpc_cfg_sts) … #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) … #define PCIE_DEC_EN_MASK … #define DEC_WORK_STATE_IDLE … #define DEC_WORK_STATE_PEND … #define IS_DEC_IDLE(dec_swreg15) … /* HBM MMU address scrambling parameters */ #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE … #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT … #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT … #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK … #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR … #define MMU_RANGE_INV_VA_LSB_SHIFT … #define MMU_RANGE_INV_VA_MSB_SHIFT … #define MMU_RANGE_INV_EN_SHIFT … #define MMU_RANGE_INV_ASID_EN_SHIFT … #define MMU_RANGE_INV_ASID_SHIFT … /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has * a 2 entries FIFO, and hence it is not enabled for it. */ #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK … #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK … #define GAUDI2_MAX_STRING_LEN … #define GAUDI2_VDEC_MSIX_ENTRIES … #define ENGINE_ID_DCORE_OFFSET … /* RAZWI initiator coordinates */ #define RAZWI_GET_AXUSER_XY(x) … #define RAZWI_GET_AXUSER_LOW_XY(x) … #define RAZWI_INITIATOR_AXUER_L_X_SHIFT … #define RAZWI_INITIATOR_AXUER_L_X_MASK … #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT … #define RAZWI_INITIATOR_AXUER_L_Y_MASK … #define RAZWI_INITIATOR_AXUER_H_X_SHIFT … #define RAZWI_INITIATOR_AXUER_H_X_MASK … #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) … #define RAZWI_INITIATOR_ID_X_HIGH(x) … #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) … #define PSOC_RAZWI_ENG_STR_SIZE … #define PSOC_RAZWI_MAX_ENG_PER_RTR … /* HW scrambles only bits 0-25 */ #define HW_UNSCRAMBLED_BITS_MASK … #define GAUDI2_GLBL_ERR_MAX_CAUSE_NUM … struct gaudi2_razwi_info { … }; static struct gaudi2_razwi_info common_razwi_info[] = …; static struct gaudi2_razwi_info mme_razwi_info[] = …; enum hl_pmmu_fatal_cause { … }; enum hl_pcie_drain_ind_cause { … }; static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = …; static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = …; static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = …; static const int gaudi2_qman_async_event_id[] = …; static const int gaudi2_dma_core_async_event_id[] = …; static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = …; static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = …; static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = …; static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = …; static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = …; static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = …; static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = …; static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = …; static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = …; static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = …; static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = …; static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = …; static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = …; struct gaudi2_sm_sei_cause_data { … }; static const struct gaudi2_sm_sei_cause_data gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = …; static const char * const gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = …; static const char * const gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = …; static const char * const gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = …; static const char * const gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = …; static const int gaudi2_queue_id_to_engine_id[] = …; const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = …; static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = …; static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = …; const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = …; static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = …; const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = …; const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = …; static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = …; static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = …; const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = …; static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = …; static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = …; static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = …; static const u32 gaudi2_mme_engine_id_to_mme_id[] = …; static const u32 gaudi2_edma_engine_id_to_edma_id[] = …; const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = …; static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = …; enum rtr_id { … }; static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = …; static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = …; static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = …; static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = …; static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = …; static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = …; static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = …; static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = …; static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = …; static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = …; static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = …; struct mme_initiators_rtr_id { … }; enum mme_initiators { … }; static const struct mme_initiators_rtr_id gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = …; enum razwi_event_sources { … }; struct hbm_mc_error_causes { … }; static struct hl_special_block_info gaudi2_special_blocks[] = …; /* Special blocks iterator is currently used to configure security protection bits, * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- * must be skipped. Following configurations are commonly used for both PB config * and global error reading, since currently they both share the same settings. * Once it changes, we must remember to use separate configurations for either one. */ static int gaudi2_iterator_skip_block_types[] = …; static struct range gaudi2_iterator_skip_block_ranges[] = …; static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = …; static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = …; struct mmu_spi_sei_cause { … }; static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = …; struct gaudi2_cache_invld_params { … }; struct gaudi2_tpc_idle_data { … }; struct gaudi2_tpc_mmu_data { … }; static s64 gaudi2_state_dump_specs_props[SP_MAX] = …; static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, bool is_memset); static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e); static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e); static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e); static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) { … } static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) { … } static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) { … } void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) { … } static bool gaudi2_host_phys_addr_valid(u64 addr) { … } static int set_number_of_functional_hbms(struct hl_device *hdev) { … } static bool gaudi2_is_edma_queue_id(u32 queue_id) { … } static int gaudi2_set_dram_properties(struct hl_device *hdev) { … } static int gaudi2_set_fixed_properties(struct hl_device *hdev) { … } static int gaudi2_pci_bars_map(struct hl_device *hdev) { … } static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) { … } static int gaudi2_init_iatu(struct hl_device *hdev) { … } static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) { … } static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) { … } static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) { … } static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) { … } static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) { … } static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) { … } static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) { … } static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) { … } static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) { … } static int gaudi2_set_binning_masks(struct hl_device *hdev) { … } static int gaudi2_cpucp_info_get(struct hl_device *hdev) { … } static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) { … } static int gaudi2_mmu_clear_pgt_range(struct hl_device *hdev) { … } static int gaudi2_early_init(struct hl_device *hdev) { … } static int gaudi2_early_fini(struct hl_device *hdev) { … } static bool gaudi2_is_arc_nic_owned(u64 arc_id) { … } static bool gaudi2_is_arc_tpc_owned(u64 arc_id) { … } static void gaudi2_init_arcs(struct hl_device *hdev) { … } static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) { … } static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) { … } static int gaudi2_late_init(struct hl_device *hdev) { … } static void gaudi2_late_fini(struct hl_device *hdev) { … } static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) { … } static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) { … } static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) { … } static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) { … } static void gaudi2_user_interrupt_setup(struct hl_device *hdev) { … } static inline int gaudi2_get_non_zero_random_int(void) { … } static void gaudi2_special_blocks_free(struct hl_device *hdev) { … } static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) { … } static bool gaudi2_special_block_skip(struct hl_device *hdev, struct hl_special_blocks_cfg *special_blocks_cfg, u32 blk_idx, u32 major, u32 minor, u32 sub_minor) { … } static int gaudi2_special_blocks_config(struct hl_device *hdev) { … } static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) { … } static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) { … } static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) { … } static int gaudi2_sw_init(struct hl_device *hdev) { … } static int gaudi2_sw_fini(struct hl_device *hdev) { … } static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) { … } /** * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters * * @hdev: pointer to the habanalabs device structure * @queue_id: queue to clear fence counters to * @skip_fence: if true set maximum fence value to all fence counters to avoid * getting stuck on any fence value. otherwise set all fence * counters to 0 (standard clear of fence counters) */ static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, bool skip_fence) { … } static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) { … } static void gaudi2_stop_dma_qmans(struct hl_device *hdev) { … } static void gaudi2_stop_mme_qmans(struct hl_device *hdev) { … } static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) { … } static void gaudi2_stop_rot_qmans(struct hl_device *hdev) { … } static void gaudi2_stop_nic_qmans(struct hl_device *hdev) { … } static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_dma_stall(struct hl_device *hdev) { … } static void gaudi2_mme_stall(struct hl_device *hdev) { … } static void gaudi2_tpc_stall(struct hl_device *hdev) { … } static void gaudi2_rotator_stall(struct hl_device *hdev) { … } static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_disable_dma_qmans(struct hl_device *hdev) { … } static void gaudi2_disable_mme_qmans(struct hl_device *hdev) { … } static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) { … } static void gaudi2_disable_rot_qmans(struct hl_device *hdev) { … } static void gaudi2_disable_nic_qmans(struct hl_device *hdev) { … } static void gaudi2_enable_timestamp(struct hl_device *hdev) { … } static void gaudi2_disable_timestamp(struct hl_device *hdev) { … } static const char *gaudi2_irq_name(u16 irq_number) { … } static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) { … } static int gaudi2_dec_enable_msix(struct hl_device *hdev) { … } static int gaudi2_enable_msix(struct hl_device *hdev) { … } static void gaudi2_sync_irqs(struct hl_device *hdev) { … } static void gaudi2_disable_msix(struct hl_device *hdev) { … } static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) { … } static void gaudi2_stop_pcie_dec(struct hl_device *hdev) { … } static void gaudi2_stop_dec(struct hl_device *hdev) { … } static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) { … } static void gaudi2_halt_arcs(struct hl_device *hdev) { … } static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) { … } static void gaudi2_reset_arcs(struct hl_device *hdev) { … } static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) { … } static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, u32 num_cores, u32 core_command) { … } static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) { … } static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) { … } static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) { … } static int gaudi2_set_engine_modes(struct hl_device *hdev, u32 *engine_ids, u32 num_engines, u32 engine_command) { … } static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, u32 num_engines, u32 engine_command) { … } static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) { … } static void gaudi2_init_firmware_loader(struct hl_device *hdev) { … } static int gaudi2_init_cpu(struct hl_device *hdev) { … } static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { … } static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, u32 queue_id_base) { … } static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, u32 queue_id_base) { … } static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) { … } static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, u32 queue_id_base) { … } static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, u32 queue_id_base) { … } static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, u32 dma_core_id, bool is_secure) { … } static void gaudi2_init_kdma(struct hl_device *hdev) { … } static void gaudi2_init_pdma(struct hl_device *hdev) { … } static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) { … } static void gaudi2_init_edma(struct hl_device *hdev) { … } /* * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. * @hdev: pointer to habanalabs device structure. * @sob_id: sync object ID. * @first_mon_id: ID of first monitor out of 3 consecutive monitors. * @interrupt_id: interrupt ID. * * Some initiators cannot have HBW address in their completion address registers, and thus cannot * write directly to the HBW host memory of the virtual MSI-X doorbell. * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. * * The mechanism in the sync manager block is composed of a master monitor with 3 messages. * In addition to the HBW write, the other 2 messages are for preparing the monitor to next * completion, by decrementing the sync object value and re-arming the monitor. */ static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, u32 first_mon_id, u32 interrupt_id) { … } static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) { … } static void gaudi2_init_sm(struct hl_device *hdev) { … } static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) { … } static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, bool config_qman_only) { … } static void gaudi2_init_mme(struct hl_device *hdev) { … } static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) { … } struct gaudi2_tpc_init_cfg_data { … }; static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, u32 offset, struct iterate_module_ctx *ctx) { … } static void gaudi2_init_tpc(struct hl_device *hdev) { … } static void gaudi2_init_rotator(struct hl_device *hdev) { … } static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) { … } static void gaudi2_init_dec(struct hl_device *hdev) { … } static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 stlb_base, u32 asid, u64 phys_addr) { … } static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, u32 start_offset, u32 inv_start_val, u32 flags) { … } static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, struct gaudi2_cache_invld_params *inv_params) { … } bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) { … } /* this function shall be called only for HMMUs for which capability bit is set */ static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) { … } static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, struct gaudi2_cache_invld_params *inv_params) { … } static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, int dcore_id, int hmmu_id, struct gaudi2_cache_invld_params *inv_params) { … } static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, int dcore_id, int hmmu_id, struct gaudi2_cache_invld_params *inv_params) { … } static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, struct gaudi2_cache_invld_params *inv_params) { … } static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) { … } static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, u32 flags, u32 asid, u64 va, u64 size) { … } static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base, bool host_resident_pgt) { … } static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base, bool host_resident_pgt) { … } static int gaudi2_pci_mmu_init(struct hl_device *hdev) { … } static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, int hmmu_id) { … } static int gaudi2_hbm_mmu_init(struct hl_device *hdev) { … } static int gaudi2_mmu_init(struct hl_device *hdev) { … } static int gaudi2_hw_init(struct hl_device *hdev) { … } /** * gaudi2_send_hard_reset_cmd - common function to handle reset * * @hdev: pointer to the habanalabs device structure * * This function handles the various possible scenarios for reset. * It considers if reset is handled by driver\FW and what FW components are loaded */ static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) { … } /** * gaudi2_execute_hard_reset - execute hard reset by driver/FW * * @hdev: pointer to the habanalabs device structure * * This function executes hard reset based on if driver/FW should do the reset */ static void gaudi2_execute_hard_reset(struct hl_device *hdev) { … } /** * gaudi2_execute_soft_reset - execute soft reset by driver/FW * * @hdev: pointer to the habanalabs device structure * @driver_performs_reset: true if driver should perform reset instead of f/w. * @poll_timeout_us: time to wait for response from f/w. * * This function executes soft reset based on if driver/FW should do the reset */ static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, u32 poll_timeout_us) { … } static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) { … } static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } static int gaudi2_suspend(struct hl_device *hdev) { … } static int gaudi2_resume(struct hl_device *hdev) { … } static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { … } static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) { … } static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) { … } static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) { … } static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) { … } static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { … } static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) { … } static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) { … } static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle) { … } static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, u32 timeout, u64 *result) { … } static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, dma_addr_t *dma_handle) { … } static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) { … } static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { … } static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { … } static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi2_send_heartbeat(struct hl_device *hdev) { … } /* This is an internal helper function, used to update the KDMA mmu props. * Should be called with a proper kdma lock. */ static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, bool mmu_bypass, u32 asid) { … } static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, u32 mon_payload, u32 sync_value) { … } /* This is an internal helper function used by gaudi2_send_job_to_kdma only */ static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, bool is_memset) { … } static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) { … } static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) { … } static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) { … } static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) { … } static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, struct gaudi2_queues_test_info *msg_info) { … } static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) { … } static int gaudi2_test_cpu_queue(struct hl_device *hdev) { … } static int gaudi2_test_queues(struct hl_device *hdev) { … } static int gaudi2_compute_reset_late_init(struct hl_device *hdev) { … } static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, struct iterate_module_ctx *ctx) { … } static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static void gaudi2_hw_queues_lock(struct hl_device *hdev) __acquires(&gaudi2->hw_queues_lock) { … } static void gaudi2_hw_queues_unlock(struct hl_device *hdev) __releases(&gaudi2->hw_queues_lock) { … } static u32 gaudi2_get_pci_id(struct hl_device *hdev) { … } static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) { … } static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) { … } static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) { … } static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) { … } static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) { … } static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) { … } static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, u32 rw_asid, u32 rw_mmu_bp) { … } static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) { … } static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) { … } static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) { … } static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, struct iterate_module_ctx *ctx) { … } /* zero the MMUBP and set the ASID */ static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) { … } static inline bool is_info_event(u32 event) { … } static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, bool ratelimited, const char *fmt, ...) { … } static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, struct hl_eq_ecc_data *ecc_data) { … } static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u32 engine_id) { … } static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, u64 qman_base, u32 qid_base, u64 *event_mask) { … } static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, u64 rtr_mstr_if_base_addr, bool is_write, char *name, enum gaudi2_engine_id id, u64 *event_mask) { … } static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, u64 rtr_mstr_if_base_addr, bool is_write, char *name, enum gaudi2_engine_id id, u64 *event_mask) { … } static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, enum razwi_event_sources module, u8 module_idx) { … } /* * This function handles RR(Range register) hit events. * raised be initiators not PSOC RAZWI. */ static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, enum razwi_event_sources module, u8 module_idx, u8 module_sub_idx, u64 *event_mask) { … } static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) { … } static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, u32 axuser_xy, u32 *base, u16 *eng_id, char *eng_name) { … } static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, u64 *event_mask) { … } /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) { … } static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) { … } static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, bool extended_err_check, u64 *event_mask) { … } static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) { … } static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, u64 *event_mask) { … } static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, u64 *event_mask) { … } static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, u64 *event_mask) { … } static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, u64 *event_mask) { … } static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) { … } static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, u64 *event_mask) { … } static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) { … } static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) { … } static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) { … } static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, u64 intr_cause_data, u64 *event_mask) { … } static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) { … } static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) { … } static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, u64 *event_mask) { … } static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) { … } static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, u64 mmu_base, bool is_pmmu, u64 *event_mask) { … } static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) { … } static u64 get_hmmu_base(u16 event_type) { … } static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) { … } static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) { … } static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) { … } /* Returns true if hard reset is needed or false otherwise */ static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, struct hl_eq_hbm_sei_data *sei_data) { … } static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) { … } static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) { … } static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, struct cpucp_pkt_sync_err *sync_err) { … } static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) { … } static int gaudi2_handle_pcie_drain(struct hl_device *hdev, struct hl_eq_pcie_drain_ind_data *drain_data) { … } static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) { … } static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, struct cpucp_pkt_sync_err *sync_err) { … } static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, struct hl_eq_engine_arc_intr_data *data) { … } static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) { … } static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) { … } static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, struct packet_lin_dma *lin_dma_pkt, u64 phys_addr, u32 hw_queue_id, u32 size, u64 addr, u32 val) { … } static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) { … } static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) { … } static int gaudi2_scrub_device_mem(struct hl_device *hdev) { … } static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) { … } static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) { … } static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) { … } static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) { … } static void gaudi2_restore_phase_topology(struct hl_device *hdev) { … } static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, struct dup_block_ctx *cfg_ctx) { … } static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, u64 mask) { … } void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) { … } static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) { … } static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) { … } static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) { … } static void gaudi2_restore_user_registers(struct hl_device *hdev) { … } static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) { … } static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) { … } static int gaudi2_ctx_init(struct hl_ctx *ctx) { … } static void gaudi2_ctx_fini(struct hl_ctx *ctx) { … } static int gaudi2_pre_schedule_cs(struct hl_cs *cs) { … } static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) { … } static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) { … } static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) { … } static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) { … } static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) { … } static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) { … } static void gaudi2_reset_sob(struct hl_device *hdev, void *data) { … } static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) { … } static u64 gaudi2_get_device_time(struct hl_device *hdev) { … } static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) { … } static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id, u32 encaps_signal_offset) { … } /* * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address * to DMMU page-size address (64MB) before mapping it in * the MMU. * The operation is performed on both the virtual and physical addresses. * for device with 6 HBMs the scramble is: * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] * * Example: * ============================================================================= * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA * Phys address in MMU last * HOP * ============================================================================= * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 * ============================================================================= */ static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) { … } static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) { … } static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) { … } static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, u32 *block_size, u32 *block_id) { … } static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size) { … } static void gaudi2_enable_events_from_fw(struct hl_device *hdev) { … } static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) { … } static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) { … } static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) { … } static void gaudi2_get_msi_info(__le32 *table) { … } static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) { … } static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) { … } static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) { … } static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, struct hl_device *hdev, struct hl_mon_state_dump *mon) { … } static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, u64 status_base_offset, enum hl_sync_engine_type engine_type, u32 engine_id, char **buf, size_t *size, size_t *offset) { … } static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = …; static void gaudi2_state_dump_init(struct hl_device *hdev) { … } static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) { … } static u32 *gaudi2_get_stream_master_qid_arr(void) { … } static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, struct attribute_group *dev_vrm_attr_grp) { … } static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, u32 page_size, u32 *real_page_size, bool is_dram_addr) { … } static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) { … } int gaudi2_send_device_activity(struct hl_device *hdev, bool open) { … } static u64 gaudi2_read_pte(struct hl_device *hdev, u64 addr) { … } static void gaudi2_write_pte(struct hl_device *hdev, u64 addr, u64 val) { … } static const struct hl_asic_funcs gaudi2_funcs = …; void gaudi2_set_asic_funcs(struct hl_device *hdev) { … }
Codebase Browser
linux