// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #include "gaudiP.h" #include "../include/hw_ip/mmu/mmu_general.h" #include "../include/hw_ip/mmu/mmu_v1_1.h" #include "../include/gaudi/gaudi_masks.h" #include "../include/gaudi/gaudi_fw_if.h" #include "../include/gaudi/gaudi_reg_map.h" #include "../include/gaudi/gaudi_async_ids_map_extended.h" #include <linux/module.h> #include <linux/pci.h> #include <linux/firmware.h> #include <linux/hwmon.h> #include <linux/iommu.h> #include <linux/seq_file.h> /* * Gaudi security scheme: * * 1. Host is protected by: * - Range registers * - MMU * * 2. DDR is protected by: * - Range registers (protect the first 512MB) * * 3. Configuration is protected by: * - Range registers * - Protection bits * * MMU is always enabled. * * QMAN DMA channels 0,1 (PCI DMAN): * - DMA is not secured. * - PQ and CQ are secured. * - CP is secured: The driver needs to parse CB but WREG should be allowed * because of TDMA (tensor DMA). Hence, WREG is always not * secured. * * When the driver needs to use DMA it will check that Gaudi is idle, set DMA * channel 0 to be secured, execute the DMA and change it back to not secured. * Currently, the driver doesn't use the DMA while there are compute jobs * running. * * The current use cases for the driver to use the DMA are: * - Clear SRAM on context switch (happens on context switch when device is * idle) * - MMU page tables area clear (happens on init) * * QMAN DMA 2-7, TPC, MME, NIC: * PQ is secured and is located on the Host (HBM CON TPC3 bug) * CQ, CP and the engine are not secured * */ #define GAUDI_BOOT_FIT_FILE … #define GAUDI_LINUX_FW_FILE … #define GAUDI_TPC_FW_FILE … MODULE_FIRMWARE(…); MODULE_FIRMWARE(…); MODULE_FIRMWARE(…); #define GAUDI_DMA_POOL_BLK_SIZE … #define GAUDI_RESET_TIMEOUT_MSEC … #define GAUDI_RESET_WAIT_MSEC … #define GAUDI_CPU_RESET_WAIT_MSEC … #define GAUDI_TEST_QUEUE_WAIT_USEC … #define GAUDI_PLDM_RESET_WAIT_MSEC … #define GAUDI_PLDM_HRESET_TIMEOUT_MSEC … #define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC … #define GAUDI_PLDM_MMU_TIMEOUT_USEC … #define GAUDI_PLDM_QMAN0_TIMEOUT_USEC … #define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC … #define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC … #define GAUDI_MSG_TO_CPU_TIMEOUT_USEC … #define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC … #define GAUDI_QMAN0_FENCE_VAL … #define GAUDI_MAX_STRING_LEN … #define GAUDI_CB_POOL_CB_CNT … #define GAUDI_CB_POOL_CB_SIZE … #define GAUDI_ALLOC_CPU_MEM_RETRY_CNT … #define GAUDI_NUM_OF_TPC_INTR_CAUSE … #define GAUDI_NUM_OF_QM_ERR_CAUSE … #define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE … #define GAUDI_ARB_WDT_TIMEOUT … #define HBM_SCRUBBING_TIMEOUT_US … #define BIN_REG_STRING_SIZE … #define MONITOR_SOB_STRING_SIZE … static u32 gaudi_stream_master[GAUDI_STREAM_MASTER_ARR_SIZE] = …; static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = …; static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = …; static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = …; static inline bool validate_packet_id(enum packet_id id) { … } static const char * const gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = …; static const char * const gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = …; static const char * const gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = …; static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = …; static struct hl_hw_obj_name_entry gaudi_so_id_to_str[] = …; static struct hl_hw_obj_name_entry gaudi_monitor_id_to_str[] = …; static s64 gaudi_state_dump_specs_props[] = …; static const int gaudi_queue_id_to_engine_id[] = …; /* The order here is opposite to the order of the indexing in the h/w. * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc. */ static const char * const gaudi_sync_manager_names[] = …; struct ecc_info_extract_params { … }; static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr); static int gaudi_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job); static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, u64 val); static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, u32 num_regs, u32 val); static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id); static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev); static int gaudi_cpucp_info_get(struct hl_device *hdev); static void gaudi_disable_clock_gating(struct hl_device *hdev); static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid); static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb); static u32 gaudi_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop); static inline enum hl_collective_mode get_collective_mode(struct hl_device *hdev, u32 queue_id) { … } static inline void set_default_power_values(struct hl_device *hdev) { … } static int gaudi_set_fixed_properties(struct hl_device *hdev) { … } static int gaudi_pci_bars_map(struct hl_device *hdev) { … } static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr) { … } static int gaudi_init_iatu(struct hl_device *hdev) { … } static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev) { … } static int gaudi_early_init(struct hl_device *hdev) { … } static int gaudi_early_fini(struct hl_device *hdev) { … } /** * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values * * @hdev: pointer to hl_device structure * */ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) { … } static int _gaudi_init_tpc_mem(struct hl_device *hdev, dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size) { … } /* * gaudi_init_tpc_mem() - Initialize TPC memories. * @hdev: Pointer to hl_device structure. * * Copy TPC kernel fw from firmware file and run it to initialize TPC memories. * * Return: 0 for success, negative value for error. */ static int gaudi_init_tpc_mem(struct hl_device *hdev) { … } static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream) { … } static void gaudi_sob_group_hw_reset(struct kref *ref) { … } static void gaudi_sob_group_reset_error(struct kref *ref) { … } static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi) { … } static int gaudi_collective_init(struct hl_device *hdev) { … } static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group) { … } static void gaudi_collective_master_init_job(struct hl_device *hdev, struct hl_cs_job *job, u32 stream, u32 sob_group_offset) { … } static void gaudi_collective_slave_init_job(struct hl_device *hdev, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { … } static int gaudi_collective_wait_init_cs(struct hl_cs *cs) { … } static u32 gaudi_get_patched_cb_extra_size(u32 user_cb_size) { … } static int gaudi_collective_wait_create_job(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id, u32 encaps_signal_offset) { … } static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id, u32 encaps_signal_offset) { … } static int gaudi_late_init(struct hl_device *hdev) { … } static void gaudi_late_fini(struct hl_device *hdev) { … } static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) { … } static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev) { … } static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev) { … } static void gaudi_set_pci_memory_regions(struct hl_device *hdev) { … } static int gaudi_sw_init(struct hl_device *hdev) { … } static int gaudi_sw_fini(struct hl_device *hdev) { … } static irqreturn_t gaudi_irq_handler_single(int irq, void *arg) { … } /* * For backward compatibility, new MSI interrupts should be set after the * existing CPU and NIC interrupts. */ static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr, bool cpu_eq) { … } static int gaudi_enable_msi_single(struct hl_device *hdev) { … } static int gaudi_enable_msi(struct hl_device *hdev) { … } static void gaudi_sync_irqs(struct hl_device *hdev) { … } static void gaudi_disable_msi(struct hl_device *hdev) { … } static void gaudi_init_scrambler_sram(struct hl_device *hdev) { … } static void gaudi_init_scrambler_hbm(struct hl_device *hdev) { … } static void gaudi_init_e2e(struct hl_device *hdev) { … } static void gaudi_init_hbm_cred(struct hl_device *hdev) { … } static void gaudi_init_golden_registers(struct hl_device *hdev) { … } static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, dma_addr_t qman_pq_addr) { … } static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) { … } static void gaudi_enable_qman(struct hl_device *hdev, int dma_id, u32 enable_mask) { … } static void gaudi_init_pci_dma_qmans(struct hl_device *hdev) { … } static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, u64 qman_base_addr) { … } static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev) { … } static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, int qman_id, u64 qman_base_addr) { … } static void gaudi_init_mme_qmans(struct hl_device *hdev) { … } static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, int qman_id, u64 qman_base_addr) { … } static void gaudi_init_tpc_qmans(struct hl_device *hdev) { … } static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, int qman_id, u64 qman_base_addr, int nic_id) { … } static void gaudi_init_nic_qmans(struct hl_device *hdev) { … } static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev) { … } static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev) { … } static void gaudi_disable_mme_qmans(struct hl_device *hdev) { … } static void gaudi_disable_tpc_qmans(struct hl_device *hdev) { … } static void gaudi_disable_nic_qmans(struct hl_device *hdev) { … } static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev) { … } static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev) { … } static void gaudi_stop_mme_qmans(struct hl_device *hdev) { … } static void gaudi_stop_tpc_qmans(struct hl_device *hdev) { … } static void gaudi_stop_nic_qmans(struct hl_device *hdev) { … } static void gaudi_pci_dma_stall(struct hl_device *hdev) { … } static void gaudi_hbm_dma_stall(struct hl_device *hdev) { … } static void gaudi_mme_stall(struct hl_device *hdev) { … } static void gaudi_tpc_stall(struct hl_device *hdev) { … } static void gaudi_disable_clock_gating(struct hl_device *hdev) { … } static void gaudi_enable_timestamp(struct hl_device *hdev) { … } static void gaudi_disable_timestamp(struct hl_device *hdev) { … } static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } static int gaudi_mmu_init(struct hl_device *hdev) { … } static int gaudi_load_firmware_to_device(struct hl_device *hdev) { … } static int gaudi_load_boot_fit_to_device(struct hl_device *hdev) { … } static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev) { … } static void gaudi_init_static_firmware_loader(struct hl_device *hdev) { … } static void gaudi_init_firmware_preload_params(struct hl_device *hdev) { … } static void gaudi_init_firmware_loader(struct hl_device *hdev) { … } static int gaudi_init_cpu(struct hl_device *hdev) { … } static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { … } static void gaudi_pre_hw_init(struct hl_device *hdev) { … } static int gaudi_hw_init(struct hl_device *hdev) { … } static int gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } static int gaudi_suspend(struct hl_device *hdev) { … } static int gaudi_resume(struct hl_device *hdev) { … } static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { … } static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { … } static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) { … } static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) { … } static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle) { … } static int gaudi_scrub_device_dram(struct hl_device *hdev, u64 val) { … } static int gaudi_scrub_device_mem(struct hl_device *hdev) { … } static void *gaudi_get_int_queue_base(struct hl_device *hdev, u32 queue_id, dma_addr_t *dma_handle, u16 *queue_len) { … } static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, u32 timeout, u64 *result) { … } static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id) { … } static int gaudi_test_cpu_queue(struct hl_device *hdev) { … } static int gaudi_test_queues(struct hl_device *hdev) { … } static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, dma_addr_t *dma_handle) { … } static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) { … } static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { … } static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { … } static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) { … } static int gaudi_pin_memory_before_cs(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt, u64 addr, enum dma_data_direction dir) { … } static int gaudi_validate_dma_pkt_host(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt, bool src_in_host) { … } static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt) { … } static int gaudi_validate_load_and_exe_pkt(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_load_and_exe *user_pkt) { … } static int gaudi_validate_cb(struct hl_device *hdev, struct hl_cs_parser *parser, bool is_mmu) { … } static int gaudi_patch_dma_packet(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt, struct packet_lin_dma *new_dma_pkt, u32 *new_dma_pkt_size) { … } static int gaudi_patch_cb(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi_parse_cb_mmu(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi_parse_cb_no_mmu(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, u32 len, u32 original_len, u64 cq_addr, u32 cq_val, u32 msi_vec, bool eb) { … } static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val) { … } static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, u64 val) { … } static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, u32 num_regs, u32 val) { … } static int gaudi_restore_sm_registers(struct hl_device *hdev) { … } static void gaudi_restore_dma_registers(struct hl_device *hdev) { … } static void gaudi_restore_qm_registers(struct hl_device *hdev) { … } static int gaudi_restore_user_registers(struct hl_device *hdev) { … } static int gaudi_context_switch(struct hl_device *hdev, u32 asid) { … } static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev) { … } static void gaudi_restore_phase_topology(struct hl_device *hdev) { … } static int gaudi_dma_core_transfer(struct hl_device *hdev, int dma_id, u64 addr, u32 size_to_dma, dma_addr_t dma_addr) { … } static int gaudi_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) { … } static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr) { … } static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val) { … } void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) { … } static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) { … } static int gaudi_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) { … } static void gaudi_get_event_desc(u16 event_type, char *desc, size_t size) { … } static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev, u32 x_y, bool is_write, u16 *engine_id_1, u16 *engine_id_2) { … } static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev, bool is_write, u16 *engine_id_1, u16 *engine_id_2) { … } static void gaudi_print_and_get_razwi_info(struct hl_device *hdev, u16 *engine_id_1, u16 *engine_id_2, bool *is_read, bool *is_write) { … } static void gaudi_print_and_get_mmu_error_info(struct hl_device *hdev, u64 *addr, u64 *event_mask) { … } /* * +-------------------+------------------------------------------------------+ * | Configuration Reg | Description | * | Address | | * +-------------------+------------------------------------------------------+ * | 0xF30 - 0xF3F |ECC single error indication (1 bit per memory wrapper)| * | |0xF30 memory wrappers 31:0 (MSB to LSB) | * | |0xF34 memory wrappers 63:32 | * | |0xF38 memory wrappers 95:64 | * | |0xF3C memory wrappers 127:96 | * +-------------------+------------------------------------------------------+ * | 0xF40 - 0xF4F |ECC double error indication (1 bit per memory wrapper)| * | |0xF40 memory wrappers 31:0 (MSB to LSB) | * | |0xF44 memory wrappers 63:32 | * | |0xF48 memory wrappers 95:64 | * | |0xF4C memory wrappers 127:96 | * +-------------------+------------------------------------------------------+ */ static int gaudi_extract_ecc_info(struct hl_device *hdev, struct ecc_info_extract_params *params, u64 *ecc_address, u64 *ecc_syndrom, u8 *memory_wrapper_idx) { … } /* * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap * * @idx: the current pi/ci value * @q_len: the queue length (power of 2) * * @return the cyclically decremented index */ static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len) { … } /** * gaudi_handle_sw_config_stream_data - print SW config stream data * * @hdev: pointer to the habanalabs device structure * @stream: the QMAN's stream * @qman_base: base address of QMAN registers block * @event_mask: mask of the last events occurred */ static void gaudi_handle_sw_config_stream_data(struct hl_device *hdev, u32 stream, u64 qman_base, u64 event_mask) { … } /** * gaudi_handle_last_pqes_on_err - print last PQEs on error * * @hdev: pointer to the habanalabs device structure * @qid_base: first QID of the QMAN (out of 4 streams) * @stream: the QMAN's stream * @qman_base: base address of QMAN registers block * @event_mask: mask of the last events occurred * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE) */ static void gaudi_handle_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, u32 stream, u64 qman_base, u64 event_mask, bool pr_sw_conf) { … } /** * handle_qman_data_on_err - extract QMAN data on error * * @hdev: pointer to the habanalabs device structure * @qid_base: first QID of the QMAN (out of 4 streams) * @stream: the QMAN's stream * @qman_base: base address of QMAN registers block * @event_mask: mask of the last events occurred * * This function attempt to exatract as much data as possible on QMAN error. * On upper CP print the SW config stream data and last 8 PQEs. * On lower CP print SW config data and last PQEs of ALL 4 upper CPs */ static void handle_qman_data_on_err(struct hl_device *hdev, u32 qid_base, u32 stream, u64 qman_base, u64 event_mask) { … } static void gaudi_handle_qman_err_generic(struct hl_device *hdev, const char *qm_name, u64 qman_base, u32 qid_base, u64 *event_mask) { … } static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, struct hl_eq_sm_sei_data *sei_data) { … } static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, struct hl_eq_ecc_data *ecc_data) { … } static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, bool check_razwi, u64 *event_mask) { … } static void gaudi_print_out_of_sync_info(struct hl_device *hdev, struct cpucp_pkt_sync_err *sync_err) { … } static void gaudi_print_fw_alive_info(struct hl_device *hdev, struct hl_eq_fw_alive *fw_alive) { … } static void gaudi_print_nic_axi_irq_info(struct hl_device *hdev, u16 event_type, void *data) { … } static int gaudi_compute_reset_late_init(struct hl_device *hdev) { … } static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, struct hl_eq_hbm_ecc_data *hbm_ecc_data) { … } static int gaudi_hbm_event_to_dev(u16 hbm_event_type) { … } static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id, char *interrupt_name) { … } static int tpc_dec_event_to_tpc_id(u16 tpc_dec_event_type) { … } static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type) { … } static void gaudi_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … } static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) { … } static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) { … } static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) { … } static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, u32 flags, u32 asid, u64 va, u64 size) { … } static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr) { … } static int gaudi_send_heartbeat(struct hl_device *hdev) { … } static int gaudi_cpucp_info_get(struct hl_device *hdev) { … } static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static void gaudi_hw_queues_lock(struct hl_device *hdev) __acquires(&gaudi->hw_queues_lock) { … } static void gaudi_hw_queues_unlock(struct hl_device *hdev) __releases(&gaudi->hw_queues_lock) { … } static u32 gaudi_get_pci_id(struct hl_device *hdev) { … } static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) { … } static int gaudi_get_monitor_dump(struct hl_device *hdev, void *data) { … } /* * this function should be used only during initialization and/or after reset, * when there are no active users. */ static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id) { … } static int gaudi_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) { … } static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) { … } static int gaudi_ctx_init(struct hl_ctx *ctx) { … } static void gaudi_ctx_fini(struct hl_ctx *ctx) { … } static int gaudi_pre_schedule_cs(struct hl_cs *cs) { … } static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) { … } static u32 gaudi_get_signal_cb_size(struct hl_device *hdev) { … } static u32 gaudi_get_wait_cb_size(struct hl_device *hdev) { … } static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id) { … } static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) { … } static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) { … } static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, u16 sob_base, u8 sob_mask, u16 sob_val, u16 mon_id) { … } static u32 gaudi_add_fence_pkt(struct packet_fence *pkt) { … } static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr) { … } static u32 gaudi_add_mon_pkts(void *buf, u16 mon_id, u64 fence_addr) { … } static u32 gaudi_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) { … } static void gaudi_reset_sob(struct hl_device *hdev, void *data) { … } static u64 gaudi_get_device_time(struct hl_device *hdev) { … } static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr, u32 *block_size, u32 *block_id) { … } static int gaudi_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size) { … } static void gaudi_enable_events_from_fw(struct hl_device *hdev) { … } static int gaudi_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) { … } static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) { … } static int gaudi_add_sync_to_engine_map_entry( struct hl_sync_to_engine_map *map, u32 reg_value, enum hl_sync_engine_type engine_type, u32 engine_id) { … } static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) { … } static int gaudi_monitor_valid(struct hl_mon_state_dump *mon) { … } static void gaudi_fill_sobs_from_mon(char *sobs, struct hl_mon_state_dump *mon) { … } static int gaudi_print_single_monitor(char **buf, size_t *size, size_t *offset, struct hl_device *hdev, struct hl_mon_state_dump *mon) { … } static int gaudi_print_fences_single_engine( struct hl_device *hdev, u64 base_offset, u64 status_base_offset, enum hl_sync_engine_type engine_type, u32 engine_id, char **buf, size_t *size, size_t *offset) { … } static struct hl_state_dump_specs_funcs gaudi_state_dump_funcs = …; static void gaudi_state_dump_init(struct hl_device *hdev) { … } static u32 *gaudi_get_stream_master_qid_arr(void) { … } static int gaudi_set_dram_properties(struct hl_device *hdev) { … } static int gaudi_set_binning_masks(struct hl_device *hdev) { … } static void gaudi_check_if_razwi_happened(struct hl_device *hdev) { … } static ssize_t infineon_ver_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(infineon_ver); static struct attribute *gaudi_vrm_dev_attrs[] = …; static void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, struct attribute_group *dev_vrm_attr_grp) { … } static int gaudi_send_device_activity(struct hl_device *hdev, bool open) { … } static const struct hl_asic_funcs gaudi_funcs = …; /** * gaudi_set_asic_funcs - set GAUDI function pointers * * @hdev: pointer to hl_device structure * */ void gaudi_set_asic_funcs(struct hl_device *hdev) { … }
Codebase Browser
linux