linux/drivers/accel/habanalabs/common/irq.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2022 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "habanalabs.h"

#include <linux/slab.h>

/**
 * struct hl_eqe_work - This structure is used to schedule work of EQ
 *                      entry and cpucp_reset event
 *
 * @eq_work:          workqueue object to run when EQ entry is received
 * @hdev:             pointer to device structure
 * @eq_entry:         copy of the EQ entry
 */
struct hl_eqe_work { … };

/**
 * hl_cq_inc_ptr - increment ci or pi of cq
 *
 * @ptr: the current ci or pi value of the completion queue
 *
 * Increment ptr by 1. If it reaches the number of completion queue
 * entries, set it to 0
 */
inline u32 hl_cq_inc_ptr(u32 ptr)
{ … }

/**
 * hl_eq_inc_ptr - increment ci of eq
 *
 * @ptr: the current ci value of the event queue
 *
 * Increment ptr by 1. If it reaches the number of event queue
 * entries, set it to 0
 */
static inline u32 hl_eq_inc_ptr(u32 ptr)
{ … }

static void irq_handle_eqe(struct work_struct *work)
{ … }

/**
 * job_finish - queue job finish work
 *
 * @hdev: pointer to device structure
 * @cs_seq: command submission sequence
 * @cq: completion queue
 * @timestamp: interrupt timestamp
 *
 */
static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp)
{ … }

/**
 * cs_finish - queue all cs jobs finish work
 *
 * @hdev: pointer to device structure
 * @cs_seq: command submission sequence
 * @timestamp: interrupt timestamp
 *
 */
static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp)
{ … }

/**
 * hl_irq_handler_cq - irq handler for completion queue
 *
 * @irq: irq number
 * @arg: pointer to completion queue structure
 *
 */
irqreturn_t hl_irq_handler_cq(int irq, void *arg)
{ … }

/*
 * hl_ts_free_objects - handler of the free objects workqueue.
 * This function should put refcount to objects that the registration node
 * took refcount to them.
 * @work: workqueue object pointer
 */
static void hl_ts_free_objects(struct work_struct *work)
{ … }

/*
 * This function called with spin_lock of wait_list_lock taken
 * This function will set timestamp and delete the registration node from the
 * wait_list_lock.
 * and since we're protected with spin_lock here, so we cannot just put the refcount
 * for the objects here, since the release function may be called and it's also a long
 * logic (which might sleep also) that cannot be handled in irq context.
 * so here we'll be filling a list with nodes of "put" jobs and then will send this
 * list to a dedicated workqueue to do the actual put.
 */
static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
						struct list_head **free_list,
						struct list_head **dynamic_alloc_list,
						struct hl_user_interrupt *intr)
{ … }

static void handle_user_interrupt_ts_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
{ … }

static void handle_user_interrupt_wait_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
{ … }

static void handle_tpc_interrupt(struct hl_device *hdev)
{ … }

static void handle_unexpected_user_interrupt(struct hl_device *hdev)
{ … }

/**
 * hl_irq_user_interrupt_handler - irq handler for user interrupts.
 *
 * @irq: irq number
 * @arg: pointer to user interrupt structure
 */
irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg)
{ … }

/**
 * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
 * This function is invoked by threaded irq mechanism
 *
 * @irq: irq number
 * @arg: pointer to user interrupt structure
 *
 */
irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
{ … }

irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg)
{ … }

/**
 * hl_irq_handler_eq - irq handler for event queue
 *
 * @irq: irq number
 * @arg: pointer to event queue structure
 *
 */
irqreturn_t hl_irq_handler_eq(int irq, void *arg)
{ … }

/**
 * hl_irq_handler_dec_abnrm - Decoder error interrupt handler
 * @irq: IRQ number
 * @arg: pointer to decoder structure.
 */
irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
{ … }

/**
 * hl_cq_init - main initialization function for an cq object
 *
 * @hdev: pointer to device structure
 * @q: pointer to cq structure
 * @hw_queue_id: The H/W queue ID this completion queue belongs to
 *               HL_INVALID_QUEUE if cq is not attached to any specific queue
 *
 * Allocate dma-able memory for the completion queue and initialize fields
 * Returns 0 on success
 */
int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
{ … }

/**
 * hl_cq_fini - destroy completion queue
 *
 * @hdev: pointer to device structure
 * @q: pointer to cq structure
 *
 * Free the completion queue memory
 */
void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
{ … }

void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
{ … }

/**
 * hl_eq_init - main initialization function for an event queue object
 *
 * @hdev: pointer to device structure
 * @q: pointer to eq structure
 *
 * Allocate dma-able memory for the event queue and initialize fields
 * Returns 0 on success
 */
int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
{ … }

/**
 * hl_eq_fini - destroy event queue
 *
 * @hdev: pointer to device structure
 * @q: pointer to eq structure
 *
 * Free the event queue memory
 */
void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
{ … }

void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
{ … }

void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q)
{ … }