linux/drivers/dma/ioat/init.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel I/OAT DMA Linux driver
 * Copyright(c) 2004 - 2015 Intel Corporation.
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/prefetch.h>
#include <linux/dca.h>
#include <linux/sizes.h>
#include "dma.h"
#include "registers.h"
#include "hw.h"

#include "../dmaengine.h"

MODULE_VERSION(…);
MODULE_DESCRIPTION(…) …;
MODULE_LICENSE(…) …;
MODULE_AUTHOR(…) …;

static const struct pci_device_id ioat_pci_tbl[] = …;
MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);

static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
static void ioat_remove(struct pci_dev *pdev);
static void
ioat_init_channel(struct ioatdma_device *ioat_dma,
		  struct ioatdma_chan *ioat_chan, int idx);
static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);

static int ioat_dca_enabled = …;
module_param(ioat_dca_enabled, int, 0644);
MODULE_PARM_DESC(…) …;
int ioat_pending_level = …;
module_param(ioat_pending_level, int, 0644);
MODULE_PARM_DESC(…) …;
static char ioat_interrupt_style[32] = …;
module_param_string(…);
MODULE_PARM_DESC(…) …;

struct kmem_cache *ioat_cache;
struct kmem_cache *ioat_sed_cache;

static bool is_jf_ioat(struct pci_dev *pdev)
{ … }

static bool is_snb_ioat(struct pci_dev *pdev)
{ … }

static bool is_ivb_ioat(struct pci_dev *pdev)
{ … }

static bool is_hsw_ioat(struct pci_dev *pdev)
{ … }

static bool is_bdx_ioat(struct pci_dev *pdev)
{ … }

static inline bool is_skx_ioat(struct pci_dev *pdev)
{ … }

static bool is_xeon_cb32(struct pci_dev *pdev)
{ … }

bool is_bwd_ioat(struct pci_dev *pdev)
{ … }

static bool is_bwd_noraid(struct pci_dev *pdev)
{ … }

/*
 * Perform a IOAT transaction to verify the HW works.
 */
#define IOAT_TEST_SIZE …

static void ioat_dma_test_callback(void *dma_async_param)
{ … }

/**
 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
 * @ioat_dma: dma device to be tested
 */
static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
{ … }

/**
 * ioat_dma_setup_interrupts - setup interrupt handler
 * @ioat_dma: ioat dma device
 */
int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
{ … }

static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
{ … }

static int ioat_probe(struct ioatdma_device *ioat_dma)
{ … }

static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
{ … }

/**
 * ioat_enumerate_channels - find and initialize the device's channels
 * @ioat_dma: the ioat dma device to be enumerated
 */
static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
{ … }

/**
 * ioat_free_chan_resources - release all the descriptors
 * @c: the channel to be cleaned
 */
static void ioat_free_chan_resources(struct dma_chan *c)
{ … }

/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring
 * @chan: channel to be initialized
 */
static int ioat_alloc_chan_resources(struct dma_chan *c)
{ … }

/* common channel initialization */
static void
ioat_init_channel(struct ioatdma_device *ioat_dma,
		  struct ioatdma_chan *ioat_chan, int idx)
{ … }

#define IOAT_NUM_SRC_TEST …
static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
{ … }

static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)
{ … }

static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)
{ … }

static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
{ … }

static void ioat_shutdown(struct pci_dev *pdev)
{ … }

static void ioat_resume(struct ioatdma_device *ioat_dma)
{ … }

#define DRV_NAME …

static pci_ers_result_t ioat_pcie_error_detected(struct pci_dev *pdev,
						 pci_channel_state_t error)
{ … }

static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev)
{ … }

static void ioat_pcie_error_resume(struct pci_dev *pdev)
{ … }

static const struct pci_error_handlers ioat_err_handler = …;

static struct pci_driver ioat_pci_driver = …;

static void release_ioatdma(struct dma_device *device)
{ … }

static struct ioatdma_device *
alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
{ … }

static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{ … }

static void ioat_remove(struct pci_dev *pdev)
{ … }

static int __init ioat_init_module(void)
{ … }
module_init(…) …;

static void __exit ioat_exit_module(void)
{ … }
module_exit(ioat_exit_module);