// SPDX-License-Identifier: GPL-2.0-only /* * IBM Accelerator Family 'GenWQE' * * (C) Copyright IBM Corp. 2013 * * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> * Author: Michael Jung <mijung@gmx.net> * Author: Michael Ruettger <michael@ibmra.de> */ /* * Device Driver Control Block (DDCB) queue support. Definition of * interrupt handlers for queue support as well as triggering the * health monitor code in case of problems. The current hardware uses * an MSI interrupt which is shared between error handling and * functional code. */ #include <linux/types.h> #include <linux/sched.h> #include <linux/wait.h> #include <linux/pci.h> #include <linux/string.h> #include <linux/dma-mapping.h> #include <linux/delay.h> #include <linux/module.h> #include <linux/interrupt.h> #include <linux/crc-itu-t.h> #include "card_base.h" #include "card_ddcb.h" /* * N: next DDCB, this is where the next DDCB will be put. * A: active DDCB, this is where the code will look for the next completion. * x: DDCB is enqueued, we are waiting for its completion. * Situation (1): Empty queue * +---+---+---+---+---+---+---+---+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | * | | | | | | | | | * +---+---+---+---+---+---+---+---+ * A/N * enqueued_ddcbs = A - N = 2 - 2 = 0 * * Situation (2): Wrapped, N > A * +---+---+---+---+---+---+---+---+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | * | | | x | x | | | | | * +---+---+---+---+---+---+---+---+ * A N * enqueued_ddcbs = N - A = 4 - 2 = 2 * * Situation (3): Queue wrapped, A > N * +---+---+---+---+---+---+---+---+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | * | x | x | | | x | x | x | x | * +---+---+---+---+---+---+---+---+ * N A * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 * * Situation (4a): Queue full N > A * +---+---+---+---+---+---+---+---+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | * | x | x | x | x | x | x | x | | * +---+---+---+---+---+---+---+---+ * A N * * enqueued_ddcbs = N - A = 7 - 0 = 7 * * Situation (4a): Queue full A > N * +---+---+---+---+---+---+---+---+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | * | x | x | x | | x | x | x | x | * +---+---+---+---+---+---+---+---+ * N A * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 */ static int queue_empty(struct ddcb_queue *queue) { … } static int queue_enqueued_ddcbs(struct ddcb_queue *queue) { … } static int queue_free_ddcbs(struct ddcb_queue *queue) { … } /* * Use of the PRIV field in the DDCB for queue debugging: * * (1) Trying to get rid of a DDCB which saw a timeout: * pddcb->priv[6] = 0xcc; # cleared * * (2) Append a DDCB via NEXT bit: * pddcb->priv[7] = 0xaa; # appended * * (3) DDCB needed tapping: * pddcb->priv[7] = 0xbb; # tapped * * (4) DDCB marked as correctly finished: * pddcb->priv[6] = 0xff; # finished */ static inline void ddcb_mark_tapped(struct ddcb *pddcb) { … } static inline void ddcb_mark_appended(struct ddcb *pddcb) { … } static inline void ddcb_mark_cleared(struct ddcb *pddcb) { … } static inline void ddcb_mark_finished(struct ddcb *pddcb) { … } static inline void ddcb_mark_unused(struct ddcb *pddcb) { … } /** * genwqe_crc16() - Generate 16-bit crc as required for DDCBs * @buff: pointer to data buffer * @len: length of data for calculation * @init: initial crc (0xffff at start) * * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff * should result in a crc16 of 0x89c3 * * Return: crc16 checksum in big endian format ! */ static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) { … } static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) { … } struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) { … } void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) { … } static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) { … } static inline void ddcb_requ_set_state(struct ddcb_requ *req, enum genwqe_requ_state new_state) { … } static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) { … } /** * ddcb_requ_finished() - Returns the hardware state of the associated DDCB * @cd: pointer to genwqe device descriptor * @req: DDCB work request * * Status of ddcb_requ mirrors this hardware state, but is copied in * the ddcb_requ on interrupt/polling function. The lowlevel code * should check the hardware state directly, the higher level code * should check the copy. * * This function will also return true if the state of the queue is * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the * shutdown case. */ static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) { … } #define RET_DDCB_APPENDED … #define RET_DDCB_TAPPED … /** * enqueue_ddcb() - Enqueue a DDCB * @cd: pointer to genwqe device descriptor * @queue: queue this operation should be done on * @pddcb: pointer to ddcb structure * @ddcb_no: pointer to ddcb number being tapped * * Start execution of DDCB by tapping or append to queue via NEXT * bit. This is done by an atomic 'compare and swap' instruction and * checking SHI and HSI of the previous DDCB. * * This function must only be called with ddcb_lock held. * * Return: 1 if new DDCB is appended to previous * 2 if DDCB queue is tapped via register/simulation */ static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, struct ddcb *pddcb, int ddcb_no) { … } /** * copy_ddcb_results() - Copy output state from real DDCB to request * @req: pointer to requested DDCB parameters * @ddcb_no: pointer to ddcb number being tapped * * Copy DDCB ASV to request struct. There is no endian * conversion made, since data structure in ASV is still * unknown here. * * This is needed by: * - genwqe_purge_ddcb() * - genwqe_check_ddcb_queue() */ static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) { … } /** * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests. * @cd: pointer to genwqe device descriptor * @queue: queue to be checked * * Return: Number of DDCBs which were finished */ static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) { … } /** * __genwqe_wait_ddcb(): Waits until DDCB is completed * @cd: pointer to genwqe device descriptor * @req: pointer to requsted DDCB parameters * * The Service Layer will update the RETC in DDCB when processing is * pending or done. * * Return: > 0 remaining jiffies, DDCB completed * -ETIMEDOUT when timeout * -ERESTARTSYS when ^C * -EINVAL when unknown error condition * * When an error is returned the called needs to ensure that * purge_ddcb() is being called to get the &req removed from the * queue. */ int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) { … } /** * get_next_ddcb() - Get next available DDCB * @cd: pointer to genwqe device descriptor * @queue: DDCB queue * @num: internal DDCB number * * DDCB's content is completely cleared but presets for PRE and * SEQNUM. This function must only be called when ddcb_lock is held. * * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. */ static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, int *num) { … } /** * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue * @cd: genwqe device descriptor * @req: DDCB request * * This will fail when the request was already FETCHED. In this case * we need to wait until it is finished. Else the DDCB can be * reused. This function also ensures that the request data structure * is removed from ddcb_req[]. * * Do not forget to call this function when genwqe_wait_ddcb() fails, * such that the request gets really removed from ddcb_req[]. * * Return: 0 success */ int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) { … } int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) { … } /** * __genwqe_enqueue_ddcb() - Enqueue a DDCB * @cd: pointer to genwqe device descriptor * @req: pointer to DDCB execution request * @f_flags: file mode: blocking, non-blocking * * Return: 0 if enqueuing succeeded * -EIO if card is unusable/PCIe problems * -EBUSY if enqueuing failed */ int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, unsigned int f_flags) { … } /** * __genwqe_execute_raw_ddcb() - Setup and execute DDCB * @cd: pointer to genwqe device descriptor * @cmd: user provided DDCB command * @f_flags: file mode: blocking, non-blocking */ int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, struct genwqe_ddcb_cmd *cmd, unsigned int f_flags) { … } /** * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished * @cd: pointer to genwqe device descriptor * * We use this as condition for our wait-queue code. */ static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) { … } /** * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight * @cd: pointer to genwqe device descriptor * * Keep track on the number of DDCBs which ware currently in the * queue. This is needed for statistics as well as condition if we want * to wait or better do polling in case of no interrupts available. */ int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) { … } static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) { … } static int ddcb_queue_initialized(struct ddcb_queue *queue) { … } static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) { … } static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) { … } static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) { … } /** * genwqe_card_thread() - Work thread for the DDCB queue * @data: pointer to genwqe device descriptor * * The idea is to check if there are DDCBs in processing. If there are * some finished DDCBs, we process them and wakeup the * requestors. Otherwise we give other processes time using * cond_resched(). */ static int genwqe_card_thread(void *data) { … } /** * genwqe_setup_service_layer() - Setup DDCB queue * @cd: pointer to genwqe device descriptor * * Allocate DDCBs. Configure Service Layer Controller (SLC). * * Return: 0 success */ int genwqe_setup_service_layer(struct genwqe_dev *cd) { … } /** * queue_wake_up_all() - Handles fatal error case * @cd: pointer to genwqe device descriptor * * The PCI device got unusable and we have to stop all pending * requests as fast as we can. The code after this must purge the * DDCBs in question and ensure that all mappings are freed. */ static int queue_wake_up_all(struct genwqe_dev *cd) { … } /** * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces * @cd: pointer to genwqe device descriptor * * Relies on the pre-condition that there are no users of the card * device anymore e.g. with open file-descriptors. * * This function must be robust enough to be called twice. */ int genwqe_finish_queue(struct genwqe_dev *cd) { … } /** * genwqe_release_service_layer() - Shutdown DDCB queue * @cd: genwqe device descriptor * * This function must be robust enough to be called twice. */ int genwqe_release_service_layer(struct genwqe_dev *cd) { … }