// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #define pr_fmt(fmt) … #include <uapi/drm/habanalabs_accel.h> #include "habanalabs.h" #include <linux/pci.h> #include <linux/hwmon.h> #include <linux/vmalloc.h> #include <drm/drm_accel.h> #include <drm/drm_drv.h> #include <trace/events/habanalabs.h> #define HL_RESET_DELAY_USEC … #define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC … enum dma_alloc_type { … }; #define MEM_SCRUB_DEFAULT_VAL … static void hl_device_heartbeat(struct work_struct *work); /* * hl_set_dram_bar- sets the bar to allow later access to address * * @hdev: pointer to habanalabs device structure. * @addr: the address the caller wants to access. * @region: the PCI region. * @new_bar_region_base: the new BAR region base address. * * @return: the old BAR base address on success, U64_MAX for failure. * The caller should set it back to the old address after use. * * In case the bar space does not cover the whole address space, * the bar base address should be set to allow access to a given address. * This function can be called also if the bar doesn't need to be set, * in that case it just won't change the base. */ static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region, u64 *new_bar_region_base) { … } int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar) { … } static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flag, enum dma_alloc_type alloc_type, const char *caller) { … } static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle, enum dma_alloc_type alloc_type, const char *caller) { … } void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flag, const char *caller) { … } void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle, const char *caller) { … } void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, dma_addr_t *dma_handle, const char *caller) { … } void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, const char *caller) { … } void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { … } void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { … } int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir, const char *caller) { … } int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) { … } void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir, const char *caller) { … } void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) { … } /* * hl_access_cfg_region - access the config region * * @hdev: pointer to habanalabs device structure * @addr: the address to access * @val: the value to write from or read to * @acc_type: the type of access (read/write 64/32) */ int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type) { … } /* * hl_access_dev_mem - access device memory * * @hdev: pointer to habanalabs device structure * @region_type: the type of the region the address belongs to * @addr: the address to access * @val: the value to write from or read to * @acc_type: the type of access (r/w, 32/64) */ int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type) { … } void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...) { … } enum hl_device_status hl_device_status(struct hl_device *hdev) { … } bool hl_device_operational(struct hl_device *hdev, enum hl_device_status *status) { … } bool hl_ctrl_device_operational(struct hl_device *hdev, enum hl_device_status *status) { … } static void print_idle_status_mask(struct hl_device *hdev, const char *message, u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE]) { … } static void hpriv_release(struct kref *ref) { … } void hl_hpriv_get(struct hl_fpriv *hpriv) { … } int hl_hpriv_put(struct hl_fpriv *hpriv) { … } static void print_device_in_use_info(struct hl_device *hdev, struct hl_mem_mgr_fini_stats *mm_fini_stats, const char *message) { … } /* * hl_device_release() - release function for habanalabs device. * @ddev: pointer to DRM device structure. * @file: pointer to DRM file private data structure. * * Called when process closes an habanalabs device */ void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv) { … } static int hl_device_release_ctrl(struct inode *inode, struct file *filp) { … } static int __hl_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) { … } /* * hl_mmap - mmap function for habanalabs device * * @*filp: pointer to file structure * @*vma: pointer to vm_area_struct of the process * * Called when process does an mmap on habanalabs device. Call the relevant mmap * function at the end of the common code. */ int hl_mmap(struct file *filp, struct vm_area_struct *vma) { … } static const struct file_operations hl_ctrl_ops = …; static void device_release_func(struct device *dev) { … } /* * device_init_cdev - Initialize cdev and device for habanalabs device * * @hdev: pointer to habanalabs device structure * @class: pointer to the class object of the device * @minor: minor number of the specific device * @fops: file operations to install for this device * @name: name of the device as it will appear in the filesystem * @cdev: pointer to the char device object that will be initialized * @dev: pointer to the device object that will be initialized * * Initialize a cdev and a Linux device for habanalabs's device. */ static int device_init_cdev(struct hl_device *hdev, const struct class *class, int minor, const struct file_operations *fops, char *name, struct cdev *cdev, struct device **dev) { … } static int cdev_sysfs_debugfs_add(struct hl_device *hdev) { … } static void cdev_sysfs_debugfs_remove(struct hl_device *hdev) { … } static void device_hard_reset_pending(struct work_struct *work) { … } static void device_release_watchdog_func(struct work_struct *work) { … } /* * device_early_init - do some early initialization for the habanalabs device * * @hdev: pointer to habanalabs device structure * * Install the relevant function pointers and call the early_init function, * if such a function exists */ static int device_early_init(struct hl_device *hdev) { … } /* * device_early_fini - finalize all that was done in device_early_init * * @hdev: pointer to habanalabs device structure * */ static void device_early_fini(struct hl_device *hdev) { … } static bool is_pci_link_healthy(struct hl_device *hdev) { … } static void stringify_time_of_last_heartbeat(struct hl_device *hdev, char *time_str, size_t size, bool is_pq_hb) { … } static bool hl_device_eq_heartbeat_received(struct hl_device *hdev) { … } static void hl_device_heartbeat(struct work_struct *work) { … } /* * device_late_init - do late stuff initialization for the habanalabs device * * @hdev: pointer to habanalabs device structure * * Do stuff that either needs the device H/W queues to be active or needs * to happen after all the rest of the initialization is finished */ static int device_late_init(struct hl_device *hdev) { … } /* * device_late_fini - finalize all that was done in device_late_init * * @hdev: pointer to habanalabs device structure * */ static void device_late_fini(struct hl_device *hdev) { … } int hl_device_utilization(struct hl_device *hdev, u32 *utilization) { … } int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable) { … } static void take_release_locks(struct hl_device *hdev) { … } static void hl_abort_waiting_for_completions(struct hl_device *hdev) { … } static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset, bool skip_wq_flush) { … } /* * hl_device_suspend - initiate device suspend * * @hdev: pointer to habanalabs device structure * * Puts the hw in the suspend state (all asics). * Returns 0 for success or an error on failure. * Called at driver suspend. */ int hl_device_suspend(struct hl_device *hdev) { … } /* * hl_device_resume - initiate device resume * * @hdev: pointer to habanalabs device structure * * Bring the hw back to operating state (all asics). * Returns 0 for success or an error on failure. * Called at driver resume. */ int hl_device_resume(struct hl_device *hdev) { … } static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev) { … } static void device_disable_open_processes(struct hl_device *hdev, bool control_dev) { … } static void send_disable_pci_access(struct hl_device *hdev, u32 flags) { … } static void handle_reset_trigger(struct hl_device *hdev, u32 flags) { … } static void reset_heartbeat_debug_info(struct hl_device *hdev) { … } static inline void device_heartbeat_schedule(struct hl_device *hdev) { … } /* * hl_device_reset - reset the device * * @hdev: pointer to habanalabs device structure * @flags: reset flags. * * Block future CS and wait for pending CS to be enqueued * Call ASIC H/W fini * Flush all completions * Re-initialize all internal data structures * Call ASIC H/W init, late_init * Test queues * Enable device * * Returns 0 for success or an error on failure. */ int hl_device_reset(struct hl_device *hdev, u32 flags) { … } /* * hl_device_cond_reset() - conditionally reset the device. * @hdev: pointer to habanalabs device structure. * @reset_flags: reset flags. * @event_mask: events to notify user about. * * Conditionally reset the device, or alternatively schedule a watchdog work to reset the device * unless another reset precedes it. */ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) { … } static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask) { … } /* * hl_notifier_event_send_all - notify all user processes via eventfd * * @hdev: pointer to habanalabs device structure * @event_mask: the occurred event/s * Returns 0 for success or an error on failure. */ void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask) { … } /* * hl_device_init - main initialization function for habanalabs device * * @hdev: pointer to habanalabs device structure * * Allocate an id for the device, do early initialization and then call the * ASIC specific initialization functions. Finally, create the cdev and the * Linux device to expose it to the user */ int hl_device_init(struct hl_device *hdev) { … } /* * hl_device_fini - main tear-down function for habanalabs device * * @hdev: pointer to habanalabs device structure * * Destroy the device, call ASIC fini functions and release the id */ void hl_device_fini(struct hl_device *hdev) { … } /* * MMIO register access helper functions. */ /* * hl_rreg - Read an MMIO register * * @hdev: pointer to habanalabs device structure * @reg: MMIO register offset (in bytes) * * Returns the value of the MMIO register we are asked to read * */ inline u32 hl_rreg(struct hl_device *hdev, u32 reg) { … } /* * hl_wreg - Write to an MMIO register * * @hdev: pointer to habanalabs device structure * @reg: MMIO register offset (in bytes) * @val: 32-bit value * * Writes the 32-bit value into the MMIO register * */ inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) { … } void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, u8 flags) { … } void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, u8 flags, u64 *event_mask) { … } static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu) { … } void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu) { … } void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, u64 *event_mask) { … } static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id) { … } void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask) { … } static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info) { … } void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info) { … } void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count) { … } void hl_enable_err_info_capture(struct hl_error_info *captured_err_info) { … } void hl_init_cpu_for_irq(struct hl_device *hdev) { … } void hl_set_irq_affinity(struct hl_device *hdev, int irq) { … } void hl_eq_heartbeat_event_handle(struct hl_device *hdev) { … } void hl_handle_clk_change_event(struct hl_device *hdev, u16 event_type, u64 *event_mask) { … }
Codebase Browser
linux