// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #include "goyaP.h" #include "../include/hw_ip/mmu/mmu_general.h" #include "../include/hw_ip/mmu/mmu_v1_0.h" #include "../include/goya/asic_reg/goya_masks.h" #include "../include/goya/goya_reg_map.h" #include <linux/pci.h> #include <linux/hwmon.h> #include <linux/iommu.h> #include <linux/seq_file.h> /* * GOYA security scheme: * * 1. Host is protected by: * - Range registers (When MMU is enabled, DMA RR does NOT protect host) * - MMU * * 2. DRAM is protected by: * - Range registers (protect the first 512MB) * - MMU (isolation between users) * * 3. Configuration is protected by: * - Range registers * - Protection bits * * When MMU is disabled: * * QMAN DMA: PQ, CQ, CP, DMA are secured. * PQ, CB and the data are on the host. * * QMAN TPC/MME: * PQ, CQ and CP are not secured. * PQ, CB and the data are on the SRAM/DRAM. * * Since QMAN DMA is secured, the driver is parsing the DMA CB: * - checks DMA pointer * - WREG, MSG_PROT are not allowed. * - MSG_LONG/SHORT are allowed. * * A read/write transaction by the QMAN to a protected area will succeed if * and only if the QMAN's CP is secured and MSG_PROT is used * * * When MMU is enabled: * * QMAN DMA: PQ, CQ and CP are secured. * MMU is set to bypass on the Secure props register of the QMAN. * The reasons we don't enable MMU for PQ, CQ and CP are: * - PQ entry is in kernel address space and the driver doesn't map it. * - CP writes to MSIX register and to kernel address space (completion * queue). * * DMA is not secured but because CP is secured, the driver still needs to parse * the CB, but doesn't need to check the DMA addresses. * * For QMAN DMA 0, DMA is also secured because only the driver uses this DMA and * the driver doesn't map memory in MMU. * * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode) * * DMA RR does NOT protect host because DMA is not secured * */ #define GOYA_BOOT_FIT_FILE … #define GOYA_LINUX_FW_FILE … #define GOYA_MMU_REGS_NUM … #define GOYA_DMA_POOL_BLK_SIZE … #define GOYA_RESET_TIMEOUT_MSEC … #define GOYA_PLDM_RESET_TIMEOUT_MSEC … #define GOYA_RESET_WAIT_MSEC … #define GOYA_CPU_RESET_WAIT_MSEC … #define GOYA_PLDM_RESET_WAIT_MSEC … #define GOYA_TEST_QUEUE_WAIT_USEC … #define GOYA_PLDM_MMU_TIMEOUT_USEC … #define GOYA_PLDM_QMAN0_TIMEOUT_USEC … #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC … #define GOYA_MSG_TO_CPU_TIMEOUT_USEC … #define GOYA_WAIT_FOR_BL_TIMEOUT_USEC … #define GOYA_QMAN0_FENCE_VAL … #define GOYA_MAX_STRING_LEN … #define GOYA_CB_POOL_CB_CNT … #define GOYA_CB_POOL_CB_SIZE … #define IS_QM_IDLE(engine, qm_glbl_sts0) … #define IS_DMA_QM_IDLE(qm_glbl_sts0) … #define IS_TPC_QM_IDLE(qm_glbl_sts0) … #define IS_MME_QM_IDLE(qm_glbl_sts0) … #define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) … #define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) … #define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) … #define IS_DMA_IDLE(dma_core_sts0) … #define IS_TPC_IDLE(tpc_cfg_sts) … #define IS_MME_IDLE(mme_arch_sts) … static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = …; static u16 goya_packet_sizes[MAX_PACKET_ID] = …; static inline bool validate_packet_id(enum packet_id id) { … } static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = …; static u32 goya_all_events[] = …; static s64 goya_state_dump_specs_props[SP_MAX] = …; static int goya_mmu_clear_pgt_range(struct hl_device *hdev); static int goya_mmu_set_dram_default_page(struct hl_device *hdev); static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev); static void goya_mmu_prepare(struct hl_device *hdev, u32 asid); int goya_set_fixed_properties(struct hl_device *hdev) { … } /* * goya_pci_bars_map - Map PCI BARS of Goya device * * @hdev: pointer to hl_device structure * * Request PCI regions and map them to kernel virtual addresses. * Returns 0 on success * */ static int goya_pci_bars_map(struct hl_device *hdev) { … } static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr) { … } /* * goya_init_iatu - Initialize the iATU unit inside the PCI controller * * @hdev: pointer to hl_device structure * * This is needed in case the firmware doesn't initialize the iATU * */ static int goya_init_iatu(struct hl_device *hdev) { … } static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev) { … } /* * goya_early_init - GOYA early initialization code * * @hdev: pointer to hl_device structure * * Verify PCI bars * Set DMA masks * PCI controller initialization * Map PCI bars * */ static int goya_early_init(struct hl_device *hdev) { … } /* * goya_early_fini - GOYA early finalization code * * @hdev: pointer to hl_device structure * * Unmap PCI bars * */ static int goya_early_fini(struct hl_device *hdev) { … } static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) { … } static void goya_qman0_set_security(struct hl_device *hdev, bool secure) { … } /* * goya_fetch_psoc_frequency - Fetch PSOC frequency values * * @hdev: pointer to hl_device structure * */ static void goya_fetch_psoc_frequency(struct hl_device *hdev) { … } /* * goya_set_frequency - set the frequency of the device * * @hdev: pointer to habanalabs device structure * @freq: the new frequency value * * Change the frequency if needed. This function has no protection against * concurrency, therefore it is assumed that the calling function has protected * itself against the case of calling this function from multiple threads with * different values * * Returns 0 if no change was done, otherwise returns 1 */ int goya_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq) { … } static void goya_set_freq_to_low_job(struct work_struct *work) { … } int goya_late_init(struct hl_device *hdev) { … } /* * goya_late_fini - GOYA late tear-down code * * @hdev: pointer to hl_device structure * * Free sensors allocated structures */ void goya_late_fini(struct hl_device *hdev) { … } static void goya_set_pci_memory_regions(struct hl_device *hdev) { … } /* * goya_sw_init - Goya software initialization code * * @hdev: pointer to hl_device structure * */ static int goya_sw_init(struct hl_device *hdev) { … } /* * goya_sw_fini - Goya software tear-down code * * @hdev: pointer to hl_device structure * */ static int goya_sw_fini(struct hl_device *hdev) { … } static void goya_init_dma_qman(struct hl_device *hdev, int dma_id, dma_addr_t bus_address) { … } static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) { … } /* * goya_init_dma_qmans - Initialize QMAN DMA registers * * @hdev: pointer to hl_device structure * * Initialize the H/W registers of the QMAN DMA channels * */ void goya_init_dma_qmans(struct hl_device *hdev) { … } /* * goya_disable_external_queues - Disable external queues * * @hdev: pointer to hl_device structure * */ static void goya_disable_external_queues(struct hl_device *hdev) { … } static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg, u32 cp_sts_reg, u32 glbl_sts0_reg) { … } /* * goya_stop_external_queues - Stop external queues * * @hdev: pointer to hl_device structure * * Returns 0 on success * */ static int goya_stop_external_queues(struct hl_device *hdev) { … } /* * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU * * @hdev: pointer to hl_device structure * * Returns 0 on success * */ int goya_init_cpu_queues(struct hl_device *hdev) { … } static void goya_set_pll_refclk(struct hl_device *hdev) { … } static void goya_disable_clk_rlx(struct hl_device *hdev) { … } static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id) { … } static void goya_tpc_mbist_workaround(struct hl_device *hdev) { … } /* * goya_init_golden_registers - Initialize golden registers * * @hdev: pointer to hl_device structure * * Initialize the H/W registers of the device * */ static void goya_init_golden_registers(struct hl_device *hdev) { … } static void goya_init_mme_qman(struct hl_device *hdev) { … } static void goya_init_mme_cmdq(struct hl_device *hdev) { … } void goya_init_mme_qmans(struct hl_device *hdev) { … } static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id) { … } static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id) { … } void goya_init_tpc_qmans(struct hl_device *hdev) { … } /* * goya_disable_internal_queues - Disable internal queues * * @hdev: pointer to hl_device structure * */ static void goya_disable_internal_queues(struct hl_device *hdev) { … } /* * goya_stop_internal_queues - Stop internal queues * * @hdev: pointer to hl_device structure * * Returns 0 on success * */ static int goya_stop_internal_queues(struct hl_device *hdev) { … } static void goya_dma_stall(struct hl_device *hdev) { … } static void goya_tpc_stall(struct hl_device *hdev) { … } static void goya_mme_stall(struct hl_device *hdev) { … } static int goya_enable_msix(struct hl_device *hdev) { … } static void goya_sync_irqs(struct hl_device *hdev) { … } static void goya_disable_msix(struct hl_device *hdev) { … } static void goya_enable_timestamp(struct hl_device *hdev) { … } static void goya_disable_timestamp(struct hl_device *hdev) { … } static void goya_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } /* * goya_load_firmware_to_device() - Load LINUX FW code to device. * @hdev: Pointer to hl_device structure. * * Copy LINUX fw code from firmware file to HBM BAR. * * Return: 0 on success, non-zero for failure. */ static int goya_load_firmware_to_device(struct hl_device *hdev) { … } /* * goya_load_boot_fit_to_device() - Load boot fit to device. * @hdev: Pointer to hl_device structure. * * Copy boot fit file to SRAM BAR. * * Return: 0 on success, non-zero for failure. */ static int goya_load_boot_fit_to_device(struct hl_device *hdev) { … } static void goya_init_dynamic_firmware_loader(struct hl_device *hdev) { … } static void goya_init_static_firmware_loader(struct hl_device *hdev) { … } static void goya_init_firmware_preload_params(struct hl_device *hdev) { … } static void goya_init_firmware_loader(struct hl_device *hdev) { … } static int goya_init_cpu(struct hl_device *hdev) { … } static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr) { … } int goya_mmu_init(struct hl_device *hdev) { … } /* * goya_hw_init - Goya hardware initialization code * * @hdev: pointer to hl_device structure * * Returns 0 on success * */ static int goya_hw_init(struct hl_device *hdev) { … } static int goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) { … } int goya_suspend(struct hl_device *hdev) { … } int goya_resume(struct hl_device *hdev) { … } static int goya_mmap(struct hl_device *hdev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { … } void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { … } void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) { … } static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) { … } static void goya_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle) { … } int goya_scrub_device_mem(struct hl_device *hdev) { … } void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, dma_addr_t *dma_handle, u16 *queue_len) { … } static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) { … } int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, u32 timeout, u64 *result) { … } int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) { … } int goya_test_cpu_queue(struct hl_device *hdev) { … } int goya_test_queues(struct hl_device *hdev) { … } static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, dma_addr_t *dma_handle) { … } static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) { … } void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { … } void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { … } u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) { … } static int goya_pin_memory_before_cs(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt, u64 addr, enum dma_data_direction dir) { … } static int goya_validate_dma_pkt_host(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt) { … } static int goya_validate_dma_pkt_no_host(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt) { … } static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt) { … } static int goya_validate_dma_pkt_mmu(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt) { … } static int goya_validate_wreg32(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_wreg32 *wreg_pkt) { … } static int goya_validate_cb(struct hl_device *hdev, struct hl_cs_parser *parser, bool is_mmu) { … } static int goya_patch_dma_packet(struct hl_device *hdev, struct hl_cs_parser *parser, struct packet_lin_dma *user_dma_pkt, struct packet_lin_dma *new_dma_pkt, u32 *new_dma_pkt_size) { … } static int goya_patch_cb(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int goya_parse_cb_mmu(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int goya_parse_cb_no_mmu(struct hl_device *hdev, struct hl_cs_parser *parser) { … } static int goya_parse_cb_no_ext_queue(struct hl_device *hdev, struct hl_cs_parser *parser) { … } int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) { … } void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, u32 len, u32 original_len, u64 cq_addr, u32 cq_val, u32 msix_vec, bool eb) { … } void goya_update_eq_ci(struct hl_device *hdev, u32 val) { … } void goya_restore_phase_topology(struct hl_device *hdev) { … } static void goya_clear_sm_regs(struct hl_device *hdev) { … } static int goya_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) { … } static u64 goya_read_pte(struct hl_device *hdev, u64 addr) { … } static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) { … } static const char *_goya_get_event_desc(u16 event_type) { … } static void goya_get_event_desc(u16 event_type, char *desc, size_t size) { … } static void goya_print_razwi_info(struct hl_device *hdev) { … } static void goya_print_mmu_error_info(struct hl_device *hdev) { … } static void goya_print_out_of_sync_info(struct hl_device *hdev, struct cpucp_pkt_sync_err *sync_err) { … } static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, bool razwi) { … } static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, size_t irq_arr_size) { … } static int goya_compute_reset_late_init(struct hl_device *hdev) { … } static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) { … } static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type) { … } void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) { … } void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) { … } static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val, bool is_dram) { … } int goya_context_switch(struct hl_device *hdev, u32 asid) { … } static int goya_mmu_clear_pgt_range(struct hl_device *hdev) { … } static int goya_mmu_set_dram_default_page(struct hl_device *hdev) { … } static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) { … } void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) { … } static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) { … } static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) { … } static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, u32 flags, u32 asid, u64 va, u64 size) { … } int goya_send_heartbeat(struct hl_device *hdev) { … } int goya_cpucp_info_get(struct hl_device *hdev) { … } static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, struct engines_data *e) { … } static void goya_hw_queues_lock(struct hl_device *hdev) __acquires(&goya->hw_queues_lock) { … } static void goya_hw_queues_unlock(struct hl_device *hdev) __releases(&goya->hw_queues_lock) { … } static u32 goya_get_pci_id(struct hl_device *hdev) { … } static int goya_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) { … } static void goya_cpu_init_scrambler_dram(struct hl_device *hdev) { … } static int goya_ctx_init(struct hl_ctx *ctx) { … } static int goya_pre_schedule_cs(struct hl_cs *cs) { … } u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) { … } static u32 goya_get_signal_cb_size(struct hl_device *hdev) { … } static u32 goya_get_wait_cb_size(struct hl_device *hdev) { … } static u32 goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) { … } static u32 goya_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) { … } static void goya_reset_sob(struct hl_device *hdev, void *data) { … } static void goya_reset_sob_group(struct hl_device *hdev, u16 sob_group) { … } u64 goya_get_device_time(struct hl_device *hdev) { … } static int goya_collective_wait_init_cs(struct hl_cs *cs) { … } static int goya_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id, u32 encaps_signal_offset) { … } static void goya_ctx_fini(struct hl_ctx *ctx) { … } static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, u32 *block_size, u32 *block_id) { … } static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size) { … } static void goya_enable_events_from_fw(struct hl_device *hdev) { … } static int goya_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) { … } static int goya_map_pll_idx_to_fw_idx(u32 pll_idx) { … } static int goya_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) { … } static int goya_monitor_valid(struct hl_mon_state_dump *mon) { … } static int goya_print_single_monitor(char **buf, size_t *size, size_t *offset, struct hl_device *hdev, struct hl_mon_state_dump *mon) { … } static int goya_print_fences_single_engine( struct hl_device *hdev, u64 base_offset, u64 status_base_offset, enum hl_sync_engine_type engine_type, u32 engine_id, char **buf, size_t *size, size_t *offset) { … } static struct hl_state_dump_specs_funcs goya_state_dump_funcs = …; static void goya_state_dump_init(struct hl_device *hdev) { … } static u32 goya_get_sob_addr(struct hl_device *hdev, u32 sob_id) { … } static u32 *goya_get_stream_master_qid_arr(void) { … } static int goya_get_monitor_dump(struct hl_device *hdev, void *data) { … } static void goya_check_if_razwi_happened(struct hl_device *hdev) { … } static int goya_scrub_device_dram(struct hl_device *hdev, u64 val) { … } static int goya_set_dram_properties(struct hl_device *hdev) { … } static int goya_set_binning_masks(struct hl_device *hdev) { … } static int goya_send_device_activity(struct hl_device *hdev, bool open) { … } static const struct hl_asic_funcs goya_funcs = …; /* * goya_set_asic_funcs - set Goya function pointers * * @*hdev: pointer to hl_device structure * */ void goya_set_asic_funcs(struct hl_device *hdev) { … }
Codebase Browser
linux