// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #include "habanalabs.h" #include <linux/habanalabs/hl_boot_if.h> #include <linux/pci.h> #include <linux/firmware.h> #include <linux/crc32.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/vmalloc.h> #include <trace/events/habanalabs.h> #define FW_FILE_MAX_SIZE … static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = …; static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = …; /** * hl_fw_version_cmp() - compares the FW version to a specific version * * @hdev: pointer to hl_device structure * @major: major number of a reference version * @minor: minor number of a reference version * @subminor: sub-minor number of a reference version * * Return 1 if FW version greater than the reference version, -1 if it's * smaller and 0 if versions are identical. */ int hl_fw_version_cmp(struct hl_device *hdev, u32 major, u32 minor, u32 subminor) { … } static char *extract_fw_ver_from_str(const char *fw_str) { … } /** * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32. * @str: the given string * @ver_num: the pointer to the extracted u32 to be returned to the caller. * @given_char: the given char at the end of the u32 in the string * * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL */ static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char) { … } /** * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor * from the version string * @hdev: pointer to the hl_device * @fw_str: the FW's version string * * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver. * * fw_str is expected to have one of two possible formats, examples: * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3' * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3' * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0. * * Return: 0 for success or a negative error code for failure. */ static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str) { … } /** * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string. * @hdev: pointer to the hl_device * @preboot_ver: the FW's version string * * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3 * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver. * * Return: 0 on success, negative error code for failure. */ static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver) { … } static int hl_request_fw(struct hl_device *hdev, const struct firmware **firmware_p, const char *fw_name) { … } /** * hl_release_firmware() - release FW * * @fw: fw descriptor * * note: this inline function added to serve as a comprehensive mirror for the * hl_request_fw function. */ static inline void hl_release_firmware(const struct firmware *fw) { … } /** * hl_fw_copy_fw_to_device() - copy FW to device * * @hdev: pointer to hl_device structure. * @fw: fw descriptor * @dst: IO memory mapped address space to copy firmware to * @src_offset: offset in src FW to copy from * @size: amount of bytes to copy (0 to copy the whole binary) * * actual copy of FW binary data to device, shared by static and dynamic loaders */ static int hl_fw_copy_fw_to_device(struct hl_device *hdev, const struct firmware *fw, void __iomem *dst, u32 src_offset, u32 size) { … } /** * hl_fw_copy_msg_to_device() - copy message to device * * @hdev: pointer to hl_device structure. * @msg: message * @dst: IO memory mapped address space to copy firmware to * @src_offset: offset in src message to copy from * @size: amount of bytes to copy (0 to copy the whole binary) * * actual copy of message data to device. */ static int hl_fw_copy_msg_to_device(struct hl_device *hdev, struct lkd_msg_comms *msg, void __iomem *dst, u32 src_offset, u32 size) { … } /** * hl_fw_load_fw_to_device() - Load F/W code to device's memory. * * @hdev: pointer to hl_device structure. * @fw_name: the firmware image name * @dst: IO memory mapped address space to copy firmware to * @src_offset: offset in src FW to copy from * @size: amount of bytes to copy (0 to copy the whole binary) * * Copy fw code from firmware file to device memory. * * Return: 0 on success, non-zero for failure. */ int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, void __iomem *dst, u32 src_offset, u32 size) { … } int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value) { … } /** * hl_fw_send_cpu_message() - send CPU message to the device. * * @hdev: pointer to hl_device structure. * @hw_queue_id: HW queue ID * @msg: raw data of the message/packet * @size: size of @msg in bytes * @timeout_us: timeout in usec to wait for CPU reply on the message * @result: return code reported by FW * * send message to the device CPU. * * Return: 0 on success, non-zero for failure. * -ENOMEM: memory allocation failure * -EAGAIN: CPU is disabled (try again when enabled) * -ETIMEDOUT: timeout waiting for FW response * -EIO: protocol error */ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 size, u32 timeout_us, u64 *result) { … } int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type) { … } int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr, size_t irq_arr_size) { … } int hl_fw_test_cpu_queue(struct hl_device *hdev) { … } void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { … } void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { … } int hl_fw_send_soft_reset(struct hl_device *hdev) { … } int hl_fw_send_device_activity(struct hl_device *hdev, bool open) { … } int hl_fw_send_heartbeat(struct hl_device *hdev) { … } static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, u32 sts_val) { … } /* placeholder for ERR1 as no errors defined there yet */ static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val, u32 sts_val) { … } static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg, u32 cpu_boot_dev_status1_reg) { … } int hl_fw_cpucp_info_get(struct hl_device *hdev, u32 sts_boot_dev_sts0_reg, u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, u32 boot_err1_reg) { … } static int hl_fw_send_msi_info_msg(struct hl_device *hdev) { … } int hl_fw_cpucp_handshake(struct hl_device *hdev, u32 sts_boot_dev_sts0_reg, u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, u32 boot_err1_reg) { … } int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) { … } int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data) { … } int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters) { … } int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy) { … } int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, enum pll_index *pll_index) { … } int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index, u16 *pll_freq_arr) { … } int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power) { … } int hl_fw_dram_replaced_row_get(struct hl_device *hdev, struct cpucp_hbm_row_info *info) { … } int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num) { … } int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid) { … } void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev) { … } void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev) { … } static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) { … } int hl_fw_wait_preboot_ready(struct hl_device *hdev) { … } static int hl_fw_read_preboot_caps(struct hl_device *hdev) { … } static int hl_fw_static_read_device_fw_version(struct hl_device *hdev, enum hl_fw_component fwc) { … } /** * hl_fw_preboot_update_state - update internal data structures during * handshake with preboot * * * @hdev: pointer to the habanalabs device structure * * @return 0 on success, otherwise non-zero error code */ static void hl_fw_preboot_update_state(struct hl_device *hdev) { … } static int hl_fw_static_read_preboot_status(struct hl_device *hdev) { … } int hl_fw_read_preboot_status(struct hl_device *hdev) { … } /* associate string with COMM status */ static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = …; /** * hl_fw_dynamic_report_error_status - report error status * * @hdev: pointer to the habanalabs device structure * @status: value of FW status register * @expected_status: the expected status */ static void hl_fw_dynamic_report_error_status(struct hl_device *hdev, u32 status, enum comms_sts expected_status) { … } /** * hl_fw_dynamic_send_cmd - send LKD to FW cmd * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @cmd: LKD to FW cmd code * @size: size of next FW component to be loaded (0 if not necessary) * * LDK to FW exact command layout is defined at struct comms_command. * note: the size argument is used only when the next FW component should be * loaded, otherwise it shall be 0. the size is used by the FW in later * protocol stages and when sending only indicating the amount of memory * to be allocated by the FW to receive the next boot component. */ static void hl_fw_dynamic_send_cmd(struct hl_device *hdev, struct fw_load_mgr *fw_loader, enum comms_cmd cmd, unsigned int size) { … } /** * hl_fw_dynamic_extract_fw_response - update the FW response * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @response: FW response * @status: the status read from CPU status register * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev, struct fw_load_mgr *fw_loader, struct fw_response *response, u32 status) { … } /** * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @expected_status: expected status to wait for * @timeout: timeout for status wait * * @return 0 on success, otherwise non-zero error code * * waiting for status from FW include polling the FW status register until * expected status is received or timeout occurs (whatever occurs first). */ static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev, struct fw_load_mgr *fw_loader, enum comms_sts expected_status, u32 timeout) { … } /** * hl_fw_dynamic_send_clear_cmd - send clear command to FW * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * * @return 0 on success, otherwise non-zero error code * * after command cycle between LKD to FW CPU (i.e. LKD got an expected status * from FW) we need to clear the CPU status register in order to avoid garbage * between command cycles. * This is done by sending clear command and polling the CPU to LKD status * register to hold the status NOOP */ static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @cmd: LKD to FW cmd code * @size: size of next FW component to be loaded (0 if not necessary) * @wait_ok: if true also wait for OK response from FW * @timeout: timeout for status wait * * @return 0 on success, otherwise non-zero error code * * brief: * when sending protocol command we have the following steps: * - send clear (clear command and verify clear status register) * - send the actual protocol command * - wait for ACK on the protocol command * - send clear * - send NOOP * if, in addition, the specific protocol command should wait for OK then: * - wait for OK * - send clear * - send NOOP * * NOTES: * send clear: this is necessary in order to clear the status register to avoid * leftovers between command * NOOP command: necessary to avoid loop on the clear command by the FW */ int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, struct fw_load_mgr *fw_loader, enum comms_cmd cmd, unsigned int size, bool wait_ok, u32 timeout) { … } /** * hl_fw_compat_crc32 - CRC compatible with FW * * @data: pointer to the data * @size: size of the data * * @return the CRC32 result * * NOTE: kernel's CRC32 differs from standard CRC32 calculation. * in order to be aligned we need to flip the bits of both the input * initial CRC and kernel's CRC32 result. * in addition both sides use initial CRC of 0, */ static u32 hl_fw_compat_crc32(u8 *data, size_t size) { … } /** * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory * transfer (image or descriptor) between * host and FW * * @hdev: pointer to the habanalabs device structure * @addr: device address of memory transfer * @size: memory transfer size * @region: PCI memory region * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev, u64 addr, size_t size, struct pci_mem_region *region) { … } /** * hl_fw_dynamic_validate_descriptor - validate FW descriptor * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @fw_desc: the descriptor from FW * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev, struct fw_load_mgr *fw_loader, struct lkd_fw_comms_desc *fw_desc) { … } static int hl_fw_dynamic_validate_response(struct hl_device *hdev, struct fw_response *response, struct pci_mem_region *region) { … } /* * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw * * @hdev: pointer to the habanalabs device structure * @fw_desc: the descriptor from FW */ static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev, struct lkd_fw_comms_desc *fw_desc) { … } /** * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @next_image_size: size to allocate for next FW component * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev, struct fw_load_mgr *fw_loader, size_t next_image_size) { … } /** * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties * * @hdev: pointer to the habanalabs device structure * @fwc: the firmware component * @fw_version: fw component's version string */ static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev, enum hl_fw_component fwc, const char *fw_version) { … } /** * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW * * @hdev: pointer to the habanalabs device structure * @fw: fw descriptor * @fw_loader: managing structure for loading device's FW */ static int hl_fw_dynamic_copy_image(struct hl_device *hdev, const struct firmware *fw, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW * * @hdev: pointer to the habanalabs device structure * @msg: message * @fw_loader: managing structure for loading device's FW */ static int hl_fw_dynamic_copy_msg(struct hl_device *hdev, struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_boot_fit_update_state - update internal data structures after boot-fit * is loaded * * @hdev: pointer to the habanalabs device structure * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 * * @return 0 on success, otherwise non-zero error code */ static void hl_fw_boot_fit_update_state(struct hl_device *hdev, u32 cpu_boot_dev_sts0_reg, u32 cpu_boot_dev_sts1_reg) { … } static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev) { … } /** * hl_fw_dynamic_load_image - load FW image using dynamic protocol * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @load_fwc: the FW component to be loaded * @img_ld_timeout: image load timeout * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_load_image(struct hl_device *hdev, struct fw_load_mgr *fw_loader, enum hl_fw_component load_fwc, u32 img_ld_timeout) { … } static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_linux_update_state - update internal data structures after Linux * is loaded. * Note: Linux initialization is comprised mainly * of two stages - loading kernel (SRAM_AVAIL) * & loading ARMCP. * Therefore reading boot device status in any of * these stages might result in different values. * * @hdev: pointer to the habanalabs device structure * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 * * @return 0 on success, otherwise non-zero error code */ static void hl_fw_linux_update_state(struct hl_device *hdev, u32 cpu_boot_dev_sts0_reg, u32 cpu_boot_dev_sts1_reg) { … } /** * hl_fw_dynamic_send_msg - send a COMMS message with attached data * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * @msg_type: message type * @data: data to be sent * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_dynamic_send_msg(struct hl_device *hdev, struct fw_load_mgr *fw_loader, u8 msg_type, void *data) { … } /** * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * * @return 0 on success, otherwise non-zero error code * * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol. * the communication is done using registers: * - LKD command register * - FW status register * the protocol is race free. this goal is achieved by splitting the requests * and response to known synchronization points between the LKD and the FW. * each response to LKD request is known and bound to a predefined timeout. * in case of timeout expiration without the desired status from FW- the * protocol (and hence the boot) will fail. */ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_static_init_cpu - initialize the device CPU using static protocol * * @hdev: pointer to the habanalabs device structure * @fw_loader: managing structure for loading device's FW * * @return 0 on success, otherwise non-zero error code */ static int hl_fw_static_init_cpu(struct hl_device *hdev, struct fw_load_mgr *fw_loader) { … } /** * hl_fw_init_cpu - initialize the device CPU * * @hdev: pointer to the habanalabs device structure * * @return 0 on success, otherwise non-zero error code * * perform necessary initializations for device's CPU. takes into account if * init protocol is static or dynamic. */ int hl_fw_init_cpu(struct hl_device *hdev) { … } void hl_fw_set_pll_profile(struct hl_device *hdev) { … } int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk) { … } long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr) { … } void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq) { … } long hl_fw_get_max_power(struct hl_device *hdev) { … } void hl_fw_set_max_power(struct hl_device *hdev) { … } static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size, u32 nonce, u32 timeout) { … } int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, u32 nonce) { … } int hl_fw_get_dev_info_signed(struct hl_device *hdev, struct cpucp_dev_info_signed *dev_info_signed, u32 nonce) { … } int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode, dma_addr_t buff, u32 *size) { … }
Codebase Browser
linux