linux/drivers/dma/ioat/dma.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel I/OAT DMA Linux driver
 * Copyright(c) 2004 - 2015 Intel Corporation.
 */

/*
 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
 * copy operations.
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/prefetch.h>
#include <linux/sizes.h>
#include "dma.h"
#include "registers.h"
#include "hw.h"

#include "../dmaengine.h"

static int completion_timeout = …;
module_param(completion_timeout, int, 0644);
MODULE_PARM_DESC(…) …;
static int idle_timeout = …;
module_param(idle_timeout, int, 0644);
MODULE_PARM_DESC(…) …;

#define IDLE_TIMEOUT …
#define COMPLETION_TIMEOUT …

static char *chanerr_str[] = …;

static void ioat_eh(struct ioatdma_chan *ioat_chan);

static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
{ … }

/**
 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
 * @irq: interrupt id
 * @data: interrupt data
 */
irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{ … }

/**
 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
 * @irq: interrupt id
 * @data: interrupt data
 */
irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{ … }

void ioat_stop(struct ioatdma_chan *ioat_chan)
{ … }

static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
{ … }

void ioat_issue_pending(struct dma_chan *c)
{ … }

/**
 * ioat_update_pending - log pending descriptors
 * @ioat_chan: ioat+ channel
 *
 * Check if the number of unsubmitted descriptors has exceeded the
 * watermark.  Called with prep_lock held
 */
static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
{ … }

static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
{ … }

void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
{ … }

static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
{ … }

static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
{ … }

static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
{ … }

static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
	__releases(&ioat_chan->prep_lock)
{ … }

static struct ioat_ring_ent *
ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
{ … }

void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
{ … }

struct ioat_ring_ent **
ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
{ … }

/**
 * ioat_check_space_lock - verify space and grab ring producer lock
 * @ioat_chan: ioat,3 channel (ring) to operate on
 * @num_descs: allocation length
 */
int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
	__acquires(&ioat_chan->prep_lock)
{ … }

static bool desc_has_ext(struct ioat_ring_ent *desc)
{ … }

static void
ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
{ … }

static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
{ … }

static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
				   u64 *phys_complete)
{ … }

static void
desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
{ … }

/**
 * __ioat_cleanup - reclaim used descriptors
 * @ioat_chan: channel (ring) to clean
 * @phys_complete: zeroed (or not) completion address (from status)
 */
static void __ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
{ … }

static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
{ … }

void ioat_cleanup_event(struct tasklet_struct *t)
{ … }

static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
{ … }


static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
{ … }

static void ioat_eh(struct ioatdma_chan *ioat_chan)
{ … }

static void check_active(struct ioatdma_chan *ioat_chan)
{ … }

static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan)
{ … }

void ioat_timer_event(struct timer_list *t)
{ … }

enum dma_status
ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
		struct dma_tx_state *txstate)
{ … }

int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
{ … }