// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2019 HabanaLabs, Ltd. * All Rights Reserved. */ #include "habanalabs.h" #include <linux/slab.h> /* * hl_queue_add_ptr - add to pi or ci and checks if it wraps around * * @ptr: the current pi/ci value * @val: the amount to add * * Add val to ptr. It can go until twice the queue length. */ inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val) { … } static inline int queue_ci_get(atomic_t *ci, u32 queue_len) { … } static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len) { … } void hl_hw_queue_update_ci(struct hl_cs *cs) { … } /* * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a * H/W queue. * @hdev: pointer to habanalabs device structure * @q: pointer to habanalabs queue structure * @ctl: BD's control word * @len: BD's length * @ptr: BD's pointer * * This function assumes there is enough space on the queue to submit a new * BD to it. It initializes the next BD and calls the device specific * function to set the pi (and doorbell) * * This function must be called when the scheduler mutex is taken * */ void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q, u32 ctl, u32 len, u64 ptr) { … } /* * ext_queue_sanity_checks - perform some sanity checks on external queue * * @hdev : pointer to hl_device structure * @q : pointer to hl_hw_queue structure * @num_of_entries : how many entries to check for space * @reserve_cq_entry : whether to reserve an entry in the cq * * H/W queues spinlock should be taken before calling this function * * Perform the following: * - Make sure we have enough space in the h/w queue * - Make sure we have enough space in the completion queue * - Reserve space in the completion queue (needs to be reversed if there * is a failure down the road before the actual submission of work). Only * do this action if reserve_cq_entry is true * */ static int ext_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, int num_of_entries, bool reserve_cq_entry) { … } /* * int_queue_sanity_checks - perform some sanity checks on internal queue * * @hdev : pointer to hl_device structure * @q : pointer to hl_hw_queue structure * @num_of_entries : how many entries to check for space * * H/W queues spinlock should be taken before calling this function * * Perform the following: * - Make sure we have enough space in the h/w queue * */ static int int_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, int num_of_entries) { … } /* * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue * @hdev: Pointer to hl_device structure. * @q: Pointer to hl_hw_queue structure. * @num_of_entries: How many entries to check for space. * * Notice: We do not reserve queue entries so this function mustn't be called * more than once per CS for the same queue * */ static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, int num_of_entries) { … } /* * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion * * @hdev: pointer to hl_device structure * @hw_queue_id: Queue's type * @cb_size: size of CB * @cb_ptr: pointer to CB location * * This function sends a single CB, that must NOT generate a completion entry. * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()' */ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, u32 cb_size, u64 cb_ptr) { … } /* * ext_queue_schedule_job - submit a JOB to an external queue * * @job: pointer to the job that needs to be submitted to the queue * * This function must be called when the scheduler mutex is taken * */ static void ext_queue_schedule_job(struct hl_cs_job *job) { … } /* * int_queue_schedule_job - submit a JOB to an internal queue * * @job: pointer to the job that needs to be submitted to the queue * * This function must be called when the scheduler mutex is taken * */ static void int_queue_schedule_job(struct hl_cs_job *job) { … } /* * hw_queue_schedule_job - submit a JOB to a H/W queue * * @job: pointer to the job that needs to be submitted to the queue * * This function must be called when the scheduler mutex is taken * */ static void hw_queue_schedule_job(struct hl_cs_job *job) { … } static int init_signal_cs(struct hl_device *hdev, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { … } void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, struct hl_cs *cs, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { … } static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { … } /* * init_signal_wait_cs - initialize a signal/wait CS * @cs: pointer to the signal/wait CS * * H/W queues spinlock should be taken before calling this function */ static int init_signal_wait_cs(struct hl_cs *cs) { … } static int encaps_sig_first_staged_cs_handler (struct hl_device *hdev, struct hl_cs *cs) { … } /* * hl_hw_queue_schedule_cs - schedule a command submission * @cs: pointer to the CS */ int hl_hw_queue_schedule_cs(struct hl_cs *cs) { … } /* * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue * * @hdev: pointer to hl_device structure * @hw_queue_id: which queue to increment its ci */ void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id) { … } static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q, bool is_cpu_queue) { … } static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) { … } static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) { … } static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) { … } static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) { … } static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) { … } static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx) { … } /* * queue_init - main initialization function for H/W queue object * * @hdev: pointer to hl_device device structure * @q: pointer to hl_hw_queue queue structure * @hw_queue_id: The id of the H/W queue * * Allocate dma-able memory for the queue and initialize fields * Returns 0 on success */ static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q, u32 hw_queue_id) { … } /* * hw_queue_fini - destroy queue * * @hdev: pointer to hl_device device structure * @q: pointer to hl_hw_queue queue structure * * Free the queue memory */ static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q) { … } int hl_hw_queues_create(struct hl_device *hdev) { … } void hl_hw_queues_destroy(struct hl_device *hdev) { … } void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset) { … }
Codebase Browser
linux