/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. */ #ifndef IOATDMA_H #define IOATDMA_H #include <linux/dmaengine.h> #include <linux/init.h> #include <linux/dmapool.h> #include <linux/cache.h> #include <linux/pci_ids.h> #include <linux/circ_buf.h> #include <linux/interrupt.h> #include "registers.h" #include "hw.h" #define IOAT_DMA_VERSION … #define IOAT_DMA_DCA_ANY_CPU … #define to_ioatdma_device(dev) … #define to_dev(ioat_chan) … #define to_pdev(ioat_chan) … #define chan_num(ch) … /* ioat hardware assumes at least two sources for raid operations */ #define src_cnt_to_sw(x) … #define src_cnt_to_hw(x) … #define ndest_to_sw(x) … #define ndest_to_hw(x) … #define src16_cnt_to_sw(x) … #define src16_cnt_to_hw(x) … /* * workaround for IOAT ver.3.0 null descriptor issue * (channel returns error when size is 0) */ #define NULL_DESC_BUFFER_SIZE … enum ioat_irq_mode { … }; /** * struct ioatdma_device - internal representation of a IOAT device * @pdev: PCI-Express device * @reg_base: MMIO register space base address * @completion_pool: DMA buffers for completion ops * @sed_hw_pool: DMA super descriptor pools * @dma_dev: embedded struct dma_device * @version: version of ioatdma device * @msix_entries: irq handlers * @idx: per channel data * @dca: direct cache access context * @irq_mode: interrupt mode (INTX, MSI, MSIX) * @cap: read DMA capabilities register */ struct ioatdma_device { … }; #define IOAT_MAX_ORDER … #define IOAT_MAX_DESCS … #define IOAT_CHUNK_SIZE … #define IOAT_DESCS_PER_CHUNK … struct ioat_descs { … }; struct ioatdma_chan { … }; struct ioat_sysfs_entry { … }; /** * struct ioat_sed_ent - wrapper around super extended hardware descriptor * @hw: hardware SED * @dma: dma address for the SED * @parent: point to the dma descriptor that's the parent * @hw_pool: descriptor pool index */ struct ioat_sed_ent { … }; /** * struct ioat_ring_ent - wrapper around hardware descriptor * @hw: hardware DMA descriptor (for memcpy) * @xor: hardware xor descriptor * @xor_ex: hardware xor extension descriptor * @pq: hardware pq descriptor * @pq_ex: hardware pq extension descriptor * @pqu: hardware pq update descriptor * @raw: hardware raw (un-typed) descriptor * @txd: the generic software descriptor for all engines * @len: total transaction length for unmap * @result: asynchronous result of validate operations * @id: identifier for debug * @sed: pointer to super extended descriptor sw desc */ struct ioat_ring_ent { … }; extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; extern int ioat_pending_level; extern struct kobj_type ioat_ktype; extern struct kmem_cache *ioat_cache; extern struct kmem_cache *ioat_sed_cache; static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) { … } /* wrapper around hardware descriptor format + additional software fields */ #ifdef DEBUG #define set_desc_id(desc, i) … #define desc_id(desc) … #else #define set_desc_id … #define desc_id … #endif static inline void __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw, struct dma_async_tx_descriptor *tx, int id) { … } #define dump_desc_dbg(c, d) … static inline struct ioatdma_chan * ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index) { … } static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan) { … } static inline u64 ioat_chansts_to_addr(u64 status) { … } static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan) { … } static inline void ioat_suspend(struct ioatdma_chan *ioat_chan) { … } static inline void ioat_reset(struct ioatdma_chan *ioat_chan) { … } static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan) { … } static inline bool is_ioat_active(unsigned long status) { … } static inline bool is_ioat_idle(unsigned long status) { … } static inline bool is_ioat_halted(unsigned long status) { … } static inline bool is_ioat_suspended(unsigned long status) { … } /* channel was fatally programmed */ static inline bool is_ioat_bug(unsigned long err) { … } static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan) { … } /* count of descriptors in flight with the engine */ static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan) { … } /* count of descriptors pending submission to hardware */ static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan) { … } static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan) { … } static inline u16 ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len) { … } static inline struct ioat_ring_ent * ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx) { … } static inline void ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr) { … } /* IOAT Prep functions */ struct dma_async_tx_descriptor * ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, dma_addr_t dma_src, size_t len, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, size_t len, enum sum_check_flags *result, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, enum sum_check_flags *pqres, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags); struct dma_async_tx_descriptor * ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, size_t len, enum sum_check_flags *result, unsigned long flags); /* IOAT Operation functions */ irqreturn_t ioat_dma_do_interrupt(int irq, void *data); irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data); struct ioat_ring_ent ** ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags); void ioat_start_null_desc(struct ioatdma_chan *ioat_chan); void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan); int ioat_reset_hw(struct ioatdma_chan *ioat_chan); enum dma_status ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate); void ioat_cleanup_event(struct tasklet_struct *t); void ioat_timer_event(struct timer_list *t); int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs); void ioat_issue_pending(struct dma_chan *chan); /* IOAT Init functions */ bool is_bwd_ioat(struct pci_dev *pdev); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *ioat_dma); int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); void ioat_stop(struct ioatdma_chan *ioat_chan); #endif /* IOATDMA_H */
Codebase Browser
linux