// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019 HiSilicon Limited. */ #include <asm/page.h> #include <linux/acpi.h> #include <linux/bitmap.h> #include <linux/dma-mapping.h> #include <linux/idr.h> #include <linux/io.h> #include <linux/irqreturn.h> #include <linux/log2.h> #include <linux/pm_runtime.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/uacce.h> #include <linux/uaccess.h> #include <uapi/misc/uacce/hisi_qm.h> #include <linux/hisi_acc_qm.h> #include "qm_common.h" /* eq/aeq irq enable */ #define QM_VF_AEQ_INT_SOURCE … #define QM_VF_AEQ_INT_MASK … #define QM_VF_EQ_INT_SOURCE … #define QM_VF_EQ_INT_MASK … #define QM_IRQ_VECTOR_MASK … #define QM_IRQ_TYPE_MASK … #define QM_IRQ_TYPE_SHIFT … #define QM_ABN_IRQ_TYPE_MASK … /* mailbox */ #define QM_MB_PING_ALL_VFS … #define QM_MB_CMD_DATA_SHIFT … #define QM_MB_CMD_DATA_MASK … #define QM_MB_STATUS_MASK … /* sqc shift */ #define QM_SQ_HOP_NUM_SHIFT … #define QM_SQ_PAGE_SIZE_SHIFT … #define QM_SQ_BUF_SIZE_SHIFT … #define QM_SQ_SQE_SIZE_SHIFT … #define QM_SQ_PRIORITY_SHIFT … #define QM_SQ_ORDERS_SHIFT … #define QM_SQ_TYPE_SHIFT … #define QM_QC_PASID_ENABLE … #define QM_QC_PASID_ENABLE_SHIFT … #define QM_SQ_TYPE_MASK … #define QM_SQ_TAIL_IDX(sqc) … /* cqc shift */ #define QM_CQ_HOP_NUM_SHIFT … #define QM_CQ_PAGE_SIZE_SHIFT … #define QM_CQ_BUF_SIZE_SHIFT … #define QM_CQ_CQE_SIZE_SHIFT … #define QM_CQ_PHASE_SHIFT … #define QM_CQ_FLAG_SHIFT … #define QM_CQE_PHASE(cqe) … #define QM_QC_CQE_SIZE … #define QM_CQ_TAIL_IDX(cqc) … /* eqc shift */ #define QM_EQE_AEQE_SIZE … #define QM_EQC_PHASE_SHIFT … #define QM_EQE_PHASE(eqe) … #define QM_EQE_CQN_MASK … #define QM_AEQE_PHASE(aeqe) … #define QM_AEQE_TYPE_SHIFT … #define QM_AEQE_TYPE_MASK … #define QM_AEQE_CQN_MASK … #define QM_CQ_OVERFLOW … #define QM_EQ_OVERFLOW … #define QM_CQE_ERROR … #define QM_XQ_DEPTH_SHIFT … #define QM_XQ_DEPTH_MASK … #define QM_DOORBELL_CMD_SQ … #define QM_DOORBELL_CMD_CQ … #define QM_DOORBELL_CMD_EQ … #define QM_DOORBELL_CMD_AEQ … #define QM_DOORBELL_BASE_V1 … #define QM_DB_CMD_SHIFT_V1 … #define QM_DB_INDEX_SHIFT_V1 … #define QM_DB_PRIORITY_SHIFT_V1 … #define QM_PAGE_SIZE … #define QM_QP_DB_INTERVAL … #define QM_DB_TIMEOUT_CFG … #define QM_DB_TIMEOUT_SET … #define QM_MEM_START_INIT … #define QM_MEM_INIT_DONE … #define QM_VFT_CFG_RDY … #define QM_VFT_CFG_OP_WR … #define QM_VFT_CFG_TYPE … #define QM_VFT_CFG … #define QM_VFT_CFG_OP_ENABLE … #define QM_PM_CTRL … #define QM_IDLE_DISABLE … #define QM_VFT_CFG_DATA_L … #define QM_VFT_CFG_DATA_H … #define QM_SQC_VFT_BUF_SIZE … #define QM_SQC_VFT_SQC_SIZE … #define QM_SQC_VFT_INDEX_NUMBER … #define QM_SQC_VFT_START_SQN_SHIFT … #define QM_SQC_VFT_VALID … #define QM_SQC_VFT_SQN_SHIFT … #define QM_CQC_VFT_BUF_SIZE … #define QM_CQC_VFT_SQC_SIZE … #define QM_CQC_VFT_INDEX_NUMBER … #define QM_CQC_VFT_VALID … #define QM_SQC_VFT_BASE_SHIFT_V2 … #define QM_SQC_VFT_BASE_MASK_V2 … #define QM_SQC_VFT_NUM_SHIFT_V2 … #define QM_SQC_VFT_NUM_MASK_V2 … #define QM_ABNORMAL_INT_SOURCE … #define QM_ABNORMAL_INT_MASK … #define QM_ABNORMAL_INT_MASK_VALUE … #define QM_ABNORMAL_INT_STATUS … #define QM_ABNORMAL_INT_SET … #define QM_ABNORMAL_INF00 … #define QM_FIFO_OVERFLOW_TYPE … #define QM_FIFO_OVERFLOW_TYPE_SHIFT … #define QM_FIFO_OVERFLOW_VF … #define QM_FIFO_OVERFLOW_QP_SHIFT … #define QM_ABNORMAL_INF01 … #define QM_DB_TIMEOUT_TYPE … #define QM_DB_TIMEOUT_TYPE_SHIFT … #define QM_DB_TIMEOUT_VF … #define QM_DB_TIMEOUT_QP_SHIFT … #define QM_ABNORMAL_INF02 … #define QM_AXI_POISON_ERR … #define QM_RAS_CE_ENABLE … #define QM_RAS_FE_ENABLE … #define QM_RAS_NFE_ENABLE … #define QM_RAS_CE_THRESHOLD … #define QM_RAS_CE_TIMES_PER_IRQ … #define QM_OOO_SHUTDOWN_SEL … #define QM_AXI_RRESP_ERR … #define QM_ECC_MBIT … #define QM_DB_TIMEOUT … #define QM_OF_FIFO_OF … #define QM_RESET_WAIT_TIMEOUT … #define QM_PEH_VENDOR_ID … #define ACC_VENDOR_ID_VALUE … #define QM_PEH_DFX_INFO0 … #define QM_PEH_DFX_INFO1 … #define QM_PEH_DFX_MASK … #define QM_PEH_MSI_FINISH_MASK … #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT … #define ACC_PEH_MSI_DISABLE … #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN … #define ACC_MASTER_TRANS_RETURN_RW … #define ACC_MASTER_TRANS_RETURN … #define ACC_MASTER_GLOBAL_CTRL … #define ACC_AM_CFG_PORT_WR_EN … #define QM_RAS_NFE_MBIT_DISABLE … #define ACC_AM_ROB_ECC_INT_STS … #define ACC_ROB_ECC_ERR_MULTPL … #define QM_MSI_CAP_ENABLE … /* interfunction communication */ #define QM_IFC_READY_STATUS … #define QM_IFC_INT_SET_P … #define QM_IFC_INT_CFG … #define QM_IFC_INT_SOURCE_P … #define QM_IFC_INT_SOURCE_V … #define QM_IFC_INT_MASK … #define QM_IFC_INT_STATUS … #define QM_IFC_INT_SET_V … #define QM_IFC_SEND_ALL_VFS … #define QM_IFC_INT_SOURCE_CLR … #define QM_IFC_INT_SOURCE_MASK … #define QM_IFC_INT_DISABLE … #define QM_IFC_INT_STATUS_MASK … #define QM_IFC_INT_SET_MASK … #define QM_WAIT_DST_ACK … #define QM_MAX_PF_WAIT_COUNT … #define QM_MAX_VF_WAIT_COUNT … #define QM_VF_RESET_WAIT_US … #define QM_VF_RESET_WAIT_CNT … #define QM_VF_RESET_WAIT_TIMEOUT_US … #define POLL_PERIOD … #define POLL_TIMEOUT … #define WAIT_PERIOD_US_MAX … #define WAIT_PERIOD_US_MIN … #define MAX_WAIT_COUNTS … #define QM_CACHE_WB_START … #define QM_CACHE_WB_DONE … #define QM_FUNC_CAPS_REG … #define QM_CAPBILITY_VERSION … #define PCI_BAR_2 … #define PCI_BAR_4 … #define QMC_ALIGN(sz) … #define QM_DBG_READ_LEN … #define QM_PCI_COMMAND_INVALID … #define QM_RESET_STOP_TX_OFFSET … #define QM_RESET_STOP_RX_OFFSET … #define WAIT_PERIOD … #define REMOVE_WAIT_DELAY … #define QM_QOS_PARAM_NUM … #define QM_QOS_MAX_VAL … #define QM_QOS_RATE … #define QM_QOS_EXPAND_RATE … #define QM_SHAPER_CIR_B_MASK … #define QM_SHAPER_CIR_U_MASK … #define QM_SHAPER_CIR_S_MASK … #define QM_SHAPER_FACTOR_CIR_U_SHIFT … #define QM_SHAPER_FACTOR_CIR_S_SHIFT … #define QM_SHAPER_FACTOR_CBS_B_SHIFT … #define QM_SHAPER_FACTOR_CBS_S_SHIFT … #define QM_SHAPER_CBS_B … #define QM_SHAPER_VFT_OFFSET … #define QM_QOS_MIN_ERROR_RATE … #define QM_SHAPER_MIN_CBS_S … #define QM_QOS_TICK … #define QM_QOS_DIVISOR_CLK … #define QM_QOS_MAX_CIR_B … #define QM_QOS_MIN_CIR_B … #define QM_QOS_MAX_CIR_U … #define QM_AUTOSUSPEND_DELAY … #define QM_DEV_ALG_MAX_LEN … /* abnormal status value for stopping queue */ #define QM_STOP_QUEUE_FAIL … #define QM_DUMP_SQC_FAIL … #define QM_DUMP_CQC_FAIL … #define QM_FINISH_WAIT … #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) … #define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) … #define QM_MK_SQC_W13(priority, orders, alg_type) … #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) … #define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) … enum vft_type { … }; enum acc_err_result { … }; enum qm_alg_type { … }; enum qm_mb_cmd { … }; enum qm_basic_type { … }; enum qm_pre_store_cap_idx { … }; static const struct hisi_qm_cap_info qm_cap_info_comm[] = …; static const struct hisi_qm_cap_info qm_cap_info_pf[] = …; static const struct hisi_qm_cap_info qm_cap_info_vf[] = …; static const struct hisi_qm_cap_info qm_basic_info[] = …; static const u32 qm_pre_store_caps[] = …; struct qm_mailbox { … }; struct qm_doorbell { … }; struct hisi_qm_resource { … }; /** * struct qm_hw_err - Structure describing the device errors * @list: hardware error list * @timestamp: timestamp when the error occurred */ struct qm_hw_err { … }; struct hisi_qm_hw_ops { … }; struct hisi_qm_hw_error { … }; static const struct hisi_qm_hw_error qm_hw_error[] = …; static const char * const qm_db_timeout[] = …; static const char * const qm_fifo_overflow[] = …; struct qm_typical_qos_table { … }; /* the qos step is 100 */ static struct qm_typical_qos_table shaper_cir_s[] = …; static struct qm_typical_qos_table shaper_cbs_s[] = …; static void qm_irqs_unregister(struct hisi_qm *qm); static int qm_reset_device(struct hisi_qm *qm); static u32 qm_get_hw_error_status(struct hisi_qm *qm) { … } static u32 qm_get_dev_err_status(struct hisi_qm *qm) { … } /* Check if the error causes the master ooo block */ static bool qm_check_dev_error(struct hisi_qm *qm) { … } static int qm_wait_reset_finish(struct hisi_qm *qm) { … } static int qm_reset_prepare_ready(struct hisi_qm *qm) { … } static void qm_reset_bit_clear(struct hisi_qm *qm) { … } static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, u64 base, u16 queue, bool op) { … } /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ int hisi_qm_wait_mb_ready(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /* 128 bit should be written to hardware at one time to trigger a mailbox */ static void qm_mb_write(struct hisi_qm *qm, const void *src) { … } static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) { … } int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, bool op) { … } EXPORT_SYMBOL_GPL(…); /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) { … } static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) { … } static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) { … } static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) { … } static void qm_disable_clock_gate(struct hisi_qm *qm) { … } static int qm_dev_mem_reset(struct hisi_qm *qm) { … } /** * hisi_qm_get_hw_info() - Get device information. * @qm: The qm which want to get information. * @info_table: Array for storing device information. * @index: Index in info_table. * @is_read: Whether read from reg, 0: not support read from reg. * * This function returns device information the caller needs. */ u32 hisi_qm_get_hw_info(struct hisi_qm *qm, const struct hisi_qm_cap_info *info_table, u32 index, bool is_read) { … } EXPORT_SYMBOL_GPL(…); static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, u16 *high_bits, enum qm_basic_type type) { … } int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, u32 dev_algs_size) { … } EXPORT_SYMBOL_GPL(…); static u32 qm_get_irq_num(struct hisi_qm *qm) { … } static int qm_pm_get_sync(struct hisi_qm *qm) { … } static void qm_pm_put_sync(struct hisi_qm *qm) { … } static void qm_cq_head_update(struct hisi_qp *qp) { … } static void qm_poll_req_cb(struct hisi_qp *qp) { … } static void qm_work_process(struct work_struct *work) { … } static void qm_get_complete_eqe_num(struct hisi_qm *qm) { … } static irqreturn_t qm_eq_irq(int irq, void *data) { … } static irqreturn_t qm_mb_cmd_irq(int irq, void *data) { … } static void qm_set_qp_disable(struct hisi_qp *qp, int offset) { … } static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) { … } static void qm_reset_function(struct hisi_qm *qm) { … } static irqreturn_t qm_aeq_thread(int irq, void *data) { … } static void qm_init_qp_status(struct hisi_qp *qp) { … } static void qm_init_prefetch(struct hisi_qm *qm) { … } /* * acc_shaper_para_calc() Get the IR value by the qos formula, the return value * is the expected qos calculated. * the formula: * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps * * IR_b * (2 ^ IR_u) * 8000 * IR(Mbps) = ------------------------- * Tick * (2 ^ IR_s) */ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) { … } static u32 acc_shaper_calc_cbs_s(u32 ir) { … } static u32 acc_shaper_calc_cir_s(u32 ir) { … } static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) { … } static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, u32 number, struct qm_shaper_factor *factor) { … } static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, u32 fun_num, u32 base, u32 number) { … } static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) { … } /* The config should be conducted after qm_dev_mem_reset() */ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number) { … } static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) { … } static void qm_hw_error_init_v1(struct hisi_qm *qm) { … } static void qm_hw_error_cfg(struct hisi_qm *qm) { … } static void qm_hw_error_init_v2(struct hisi_qm *qm) { … } static void qm_hw_error_uninit_v2(struct hisi_qm *qm) { … } static void qm_hw_error_init_v3(struct hisi_qm *qm) { … } static void qm_hw_error_uninit_v3(struct hisi_qm *qm) { … } static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) { … } static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { … } static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) { … } static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) { … } static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) { … } static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) { … } static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) { … } static void qm_trigger_pf_interrupt(struct hisi_qm *qm) { … } static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) { … } static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) { … } static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) { … } static int qm_drain_qm(struct hisi_qm *qm) { … } static int qm_stop_qp(struct hisi_qp *qp) { … } static int qm_set_msi(struct hisi_qm *qm, bool set) { … } static void qm_wait_msi_finish(struct hisi_qm *qm) { … } static int qm_set_msi_v3(struct hisi_qm *qm, bool set) { … } static const struct hisi_qm_hw_ops qm_hw_ops_v1 = …; static const struct hisi_qm_hw_ops qm_hw_ops_v2 = …; static const struct hisi_qm_hw_ops qm_hw_ops_v3 = …; static void *qm_get_avail_sqe(struct hisi_qp *qp) { … } static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) { … } static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) { … } /** * hisi_qm_create_qp() - Create a queue pair from qm. * @qm: The qm we create a qp from. * @alg_type: Accelerator specific algorithm type in sqc. * * Return created qp, negative error code if failed. */ static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) { … } /** * hisi_qm_release_qp() - Release a qp back to its qm. * @qp: The qp we want to release. * * This function releases the resource of a qp. */ static void hisi_qm_release_qp(struct hisi_qp *qp) { … } static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { … } static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { … } static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { … } static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) { … } /** * hisi_qm_start_qp() - Start a qp into running. * @qp: The qp we want to start to run. * @arg: Accelerator specific argument. * * After this function, qp can receive request from user. Return 0 if * successful, negative error code if failed. */ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) { … } EXPORT_SYMBOL_GPL(…); /** * qp_stop_fail_cb() - call request cb. * @qp: stopped failed qp. * * Callback function should be called whether task completed or not. */ static void qp_stop_fail_cb(struct hisi_qp *qp) { … } static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) { … } /** * qm_drain_qp() - Drain a qp. * @qp: The qp we want to drain. * * If the device does not support stopping queue by sending mailbox, * determine whether the queue is cleared by judging the tail pointers of * sq and cq. */ static int qm_drain_qp(struct hisi_qp *qp) { … } static void qm_stop_qp_nolock(struct hisi_qp *qp) { … } /** * hisi_qm_stop_qp() - Stop a qp in qm. * @qp: The qp we want to stop. * * This function is reverse of hisi_qm_start_qp. */ void hisi_qm_stop_qp(struct hisi_qp *qp) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qp_send() - Queue up a task in the hardware queue. * @qp: The qp in which to put the message. * @msg: The message. * * This function will return -EBUSY if qp is currently full, and -EAGAIN * if qp related qm is resetting. * * Note: This function may run with qm_irq_thread and ACC reset at same time. * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC * reset may happen, we have no lock here considering performance. This * causes current qm_db sending fail or can not receive sended sqe. QM * sync/async receive function should handle the error sqe. ACC reset * done function should clear used sqe to 0. */ int hisi_qp_send(struct hisi_qp *qp, const void *msg) { … } EXPORT_SYMBOL_GPL(…); static void hisi_qm_cache_wb(struct hisi_qm *qm) { … } static void qm_qp_event_notifier(struct hisi_qp *qp) { … } /* This function returns free number of qp in qm. */ static int hisi_qm_get_available_instances(struct uacce_device *uacce) { … } static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) { … } static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, unsigned long arg, struct uacce_queue *q) { … } static void hisi_qm_uacce_put_queue(struct uacce_queue *q) { … } /* map sq/cq/doorbell to user space */ static int hisi_qm_uacce_mmap(struct uacce_queue *q, struct vm_area_struct *vma, struct uacce_qfile_region *qfr) { … } static int hisi_qm_uacce_start_queue(struct uacce_queue *q) { … } static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) { … } static int hisi_qm_is_q_updated(struct uacce_queue *q) { … } static void qm_set_sqctype(struct uacce_queue *q, u16 type) { … } static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, unsigned long arg) { … } /** * qm_hw_err_isolate() - Try to set the isolation status of the uacce device * according to user's configuration of error threshold. * @qm: the uacce device */ static int qm_hw_err_isolate(struct hisi_qm *qm) { … } static void qm_hw_err_destroy(struct hisi_qm *qm) { … } static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) { … } static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) { … } static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce) { … } static const struct uacce_ops uacce_qm_ops = …; static void qm_remove_uacce(struct hisi_qm *qm) { … } static int qm_alloc_uacce(struct hisi_qm *qm) { … } /** * qm_frozen() - Try to froze QM to cut continuous queue request. If * there is user on the QM, return failure without doing anything. * @qm: The qm needed to be fronzen. * * This function frozes QM, then we can do SRIOV disabling. */ static int qm_frozen(struct hisi_qm *qm) { … } static int qm_try_frozen_vfs(struct pci_dev *pdev, struct hisi_qm_list *qm_list) { … } /** * hisi_qm_wait_task_finish() - Wait until the task is finished * when removing the driver. * @qm: The qm needed to wait for the task to finish. * @qm_list: The list of all available devices. */ void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { … } EXPORT_SYMBOL_GPL(…); static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { … } static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, u16 sq_depth, u16 cq_depth) { … } static void hisi_qm_pre_init(struct hisi_qm *qm) { … } static void qm_cmd_uninit(struct hisi_qm *qm) { … } static void qm_cmd_init(struct hisi_qm *qm) { … } static void qm_put_pci_res(struct hisi_qm *qm) { … } static void hisi_qm_pci_uninit(struct hisi_qm *qm) { … } static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) { … } static void hisi_qm_unint_work(struct hisi_qm *qm) { … } static void hisi_qm_free_rsv_buf(struct hisi_qm *qm) { … } static void hisi_qm_memory_uninit(struct hisi_qm *qm) { … } /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. * * This function uninits qm related device resources. */ void hisi_qm_uninit(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_get_vft() - Get vft from a qm. * @qm: The qm we want to get its vft. * @base: The base number of queue in vft. * @number: The number of queues in vft. * * We can allocate multiple queues to a qm by configuring virtual function * table. We get related configures by this function. Normally, we call this * function in VF driver to get the queue information. * * qm hw v1 does not support this interface. */ static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) { … } /** * hisi_qm_set_vft() - Set vft to a qm. * @qm: The qm we want to set its vft. * @fun_num: The function number. * @base: The base number of queue in vft. * @number: The number of queues in vft. * * This function is alway called in PF driver, it is used to assign queues * among PF and VFs. * * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1) * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1) * (VF function number 0x2) */ static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number) { … } static void qm_init_eq_aeq_status(struct hisi_qm *qm) { … } static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) { … } static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) { … } static int qm_eq_ctx_cfg(struct hisi_qm *qm) { … } static int qm_aeq_ctx_cfg(struct hisi_qm *qm) { … } static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) { … } static int __hisi_qm_start(struct hisi_qm *qm) { … } /** * hisi_qm_start() - start qm * @qm: The qm to be started. * * This function starts a qm, then we can allocate qp from this qm. */ int hisi_qm_start(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); static int qm_restart(struct hisi_qm *qm) { … } /* Stop started qps in reset flow */ static void qm_stop_started_qp(struct hisi_qm *qm) { … } /** * qm_clear_queues() - Clear all queues memory in a qm. * @qm: The qm in which the queues will be cleared. * * This function clears all queues memory in a qm. Reset of accelerator can * use this to clear queues. */ static void qm_clear_queues(struct hisi_qm *qm) { … } /** * hisi_qm_stop() - Stop a qm. * @qm: The qm which will be stopped. * @r: The reason to stop qm. * * This function stops qm and its qps, then qm can not accept request. * Related resources are not released at this state, we can use hisi_qm_start * to let qm start again. */ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) { … } EXPORT_SYMBOL_GPL(…); static void qm_hw_error_init(struct hisi_qm *qm) { … } static void qm_hw_error_uninit(struct hisi_qm *qm) { … } static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm) { … } /** * hisi_qm_dev_err_init() - Initialize device error configuration. * @qm: The qm for which we want to do error initialization. * * Initialize QM and device error related configuration. */ void hisi_qm_dev_err_init(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. * @qm: The qm for which we want to do error uninitialization. * * Uninitialize QM and device error related configuration. */ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_free_qps() - free multiple queue pairs. * @qps: The queue pairs need to be freed. * @qp_num: The num of queue pairs. */ void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) { … } EXPORT_SYMBOL_GPL(…); static void free_list(struct list_head *head) { … } static int hisi_qm_sort_devices(int node, struct list_head *head, struct hisi_qm_list *qm_list) { … } /** * hisi_qm_alloc_qps_node() - Create multiple queue pairs. * @qm_list: The list of all available devices. * @qp_num: The number of queue pairs need created. * @alg_type: The algorithm type. * @node: The numa node. * @qps: The queue pairs need created. * * This function will sort all available device according to numa distance. * Then try to create all queue pairs from one device, if all devices do * not meet the requirements will return error. */ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, u8 alg_type, int node, struct hisi_qp **qps) { … } EXPORT_SYMBOL_GPL(…); static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) { … } static int qm_clear_vft_config(struct hisi_qm *qm) { … } static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) { … } static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) { … } static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) { … } static int qm_vf_read_qos(struct hisi_qm *qm) { … } static ssize_t qm_algqos_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { … } static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, unsigned long *val, unsigned int *fun_index) { … } static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { … } static const struct file_operations qm_algqos_fops = …; /** * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. * @qm: The qm for which we want to add debugfs files. * * Create function qos debugfs files, VF ping PF to get function qos. */ void hisi_qm_set_algqos_init(struct hisi_qm *qm) { … } static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) { … } /** * hisi_qm_sriov_enable() - enable virtual functions * @pdev: the PCIe device * @max_vfs: the number of virtual functions to enable * * Returns the number of enabled VFs. If there are VFs enabled already or * max_vfs is more than the total number of device can be enabled, returns * failure. */ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_sriov_disable - disable virtual functions * @pdev: the PCI device. * @is_frozen: true when all the VFs are frozen. * * Return failure if there are VFs assigned already or VF is in used. */ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_sriov_configure - configure the number of VFs * @pdev: The PCI device * @num_vfs: The number of VFs need enabled * * Enable SR-IOV according to num_vfs, 0 means disable. */ int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { … } EXPORT_SYMBOL_GPL(…); static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) { … } static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) { … } /** * hisi_qm_dev_err_detected() - Get device and qm error status then log it. * @pdev: The PCI device which need report error. * @state: The connectivity between CPU and device. * * We register this function into PCIe AER handlers, It will report device or * qm hardware error status when error occur. */ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { … } EXPORT_SYMBOL_GPL(…); static int qm_check_req_recv(struct hisi_qm *qm) { … } static int qm_set_pf_mse(struct hisi_qm *qm, bool set) { … } static int qm_set_vf_mse(struct hisi_qm *qm, bool set) { … } static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) { … } static int qm_vf_reset_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { … } static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, enum qm_stop_reason stop_reason) { … } static int qm_controller_reset_prepare(struct hisi_qm *qm) { … } static int qm_master_ooo_check(struct hisi_qm *qm) { … } static int qm_soft_reset_prepare(struct hisi_qm *qm) { … } static int qm_reset_device(struct hisi_qm *qm) { … } static int qm_soft_reset(struct hisi_qm *qm) { … } static int qm_vf_reset_done(struct hisi_qm *qm) { … } static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) { … } static int qm_dev_hw_init(struct hisi_qm *qm) { … } static void qm_restart_prepare(struct hisi_qm *qm) { … } static void qm_restart_done(struct hisi_qm *qm) { … } static int qm_controller_reset_done(struct hisi_qm *qm) { … } static int qm_controller_reset(struct hisi_qm *qm) { … } /** * hisi_qm_dev_slot_reset() - slot reset * @pdev: the PCIe device * * This function offers QM relate PCIe device reset interface. Drivers which * use QM can use this function as slot_reset in its struct pci_error_handlers. */ pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) { … } EXPORT_SYMBOL_GPL(…); void hisi_qm_reset_prepare(struct pci_dev *pdev) { … } EXPORT_SYMBOL_GPL(…); static bool qm_flr_reset_complete(struct pci_dev *pdev) { … } void hisi_qm_reset_done(struct pci_dev *pdev) { … } EXPORT_SYMBOL_GPL(…); static irqreturn_t qm_abnormal_irq(int irq, void *data) { … } /** * hisi_qm_dev_shutdown() - Shutdown device. * @pdev: The device will be shutdown. * * This function will stop qm when OS shutdown or rebooting. */ void hisi_qm_dev_shutdown(struct pci_dev *pdev) { … } EXPORT_SYMBOL_GPL(…); static void hisi_qm_controller_reset(struct work_struct *rst_work) { … } static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { … } static void qm_pf_reset_vf_done(struct hisi_qm *qm) { … } static int qm_wait_pf_reset_finish(struct hisi_qm *qm) { … } static void qm_pf_reset_vf_process(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { … } static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) { … } static void qm_cmd_process(struct work_struct *cmd_process) { … } /** * hisi_qm_alg_register() - Register alg to crypto. * @qm: The qm needs add. * @qm_list: The qm list. * @guard: Guard of qp_num. * * Register algorithm to crypto when the function is satisfy guard. */ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_alg_unregister() - Unregister alg from crypto. * @qm: The qm needs delete. * @qm_list: The qm list. * @guard: Guard of qp_num. * * Unregister algorithm from crypto when the last function is satisfy guard. */ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) { … } EXPORT_SYMBOL_GPL(…); static void qm_unregister_abnormal_irq(struct hisi_qm *qm) { … } static int qm_register_abnormal_irq(struct hisi_qm *qm) { … } static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) { … } static int qm_register_mb_cmd_irq(struct hisi_qm *qm) { … } static void qm_unregister_aeq_irq(struct hisi_qm *qm) { … } static int qm_register_aeq_irq(struct hisi_qm *qm) { … } static void qm_unregister_eq_irq(struct hisi_qm *qm) { … } static int qm_register_eq_irq(struct hisi_qm *qm) { … } static void qm_irqs_unregister(struct hisi_qm *qm) { … } static int qm_irqs_register(struct hisi_qm *qm) { … } static int qm_get_qp_num(struct hisi_qm *qm) { … } static int qm_pre_store_irq_type_caps(struct hisi_qm *qm) { … } static int qm_get_hw_caps(struct hisi_qm *qm) { … } static int qm_get_pci_res(struct hisi_qm *qm) { … } static int qm_clear_device(struct hisi_qm *qm) { … } static int hisi_qm_pci_init(struct hisi_qm *qm) { … } static int hisi_qm_init_work(struct hisi_qm *qm) { … } static int hisi_qp_alloc_memory(struct hisi_qm *qm) { … } static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm) { … } static int hisi_qm_memory_init(struct hisi_qm *qm) { … } /** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. * * This function init qm, then we can call hisi_qm_start to put qm into work. */ int hisi_qm_init(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_get_dfx_access() - Try to get dfx access. * @qm: pointer to accelerator device. * * Try to get dfx access, then user can get message. * * If device is in suspended, return failure, otherwise * bump up the runtime PM usage counter. */ int hisi_qm_get_dfx_access(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_put_dfx_access() - Put dfx access. * @qm: pointer to accelerator device. * * Put dfx access, drop runtime PM usage counter. */ void hisi_qm_put_dfx_access(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_pm_init() - Initialize qm runtime PM. * @qm: pointer to accelerator device. * * Function that initialize qm runtime PM. */ void hisi_qm_pm_init(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_pm_uninit() - Uninitialize qm runtime PM. * @qm: pointer to accelerator device. * * Function that uninitialize qm runtime PM. */ void hisi_qm_pm_uninit(struct hisi_qm *qm) { … } EXPORT_SYMBOL_GPL(…); static int qm_prepare_for_suspend(struct hisi_qm *qm) { … } static int qm_rebuild_for_resume(struct hisi_qm *qm) { … } /** * hisi_qm_suspend() - Runtime suspend of given device. * @dev: device to suspend. * * Function that suspend the device. */ int hisi_qm_suspend(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); /** * hisi_qm_resume() - Runtime resume of given device. * @dev: device to resume. * * Function that resume the device. */ int hisi_qm_resume(struct device *dev) { … } EXPORT_SYMBOL_GPL(…); MODULE_LICENSE(…) …; MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …;
Codebase Browser
linux