// SPDX-License-Identifier: GPL-2.0-or-later /* * DMA driver for Xilinx ZynqMP DMA Engine * * Copyright (C) 2016 Xilinx, Inc. All rights reserved. */ #include <linux/bitops.h> #include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/pm_runtime.h> #include "../dmaengine.h" /* Register Offsets */ #define ZYNQMP_DMA_ISR … #define ZYNQMP_DMA_IMR … #define ZYNQMP_DMA_IER … #define ZYNQMP_DMA_IDS … #define ZYNQMP_DMA_CTRL0 … #define ZYNQMP_DMA_CTRL1 … #define ZYNQMP_DMA_DATA_ATTR … #define ZYNQMP_DMA_DSCR_ATTR … #define ZYNQMP_DMA_SRC_DSCR_WRD0 … #define ZYNQMP_DMA_SRC_DSCR_WRD1 … #define ZYNQMP_DMA_SRC_DSCR_WRD2 … #define ZYNQMP_DMA_SRC_DSCR_WRD3 … #define ZYNQMP_DMA_DST_DSCR_WRD0 … #define ZYNQMP_DMA_DST_DSCR_WRD1 … #define ZYNQMP_DMA_DST_DSCR_WRD2 … #define ZYNQMP_DMA_DST_DSCR_WRD3 … #define ZYNQMP_DMA_SRC_START_LSB … #define ZYNQMP_DMA_SRC_START_MSB … #define ZYNQMP_DMA_DST_START_LSB … #define ZYNQMP_DMA_DST_START_MSB … #define ZYNQMP_DMA_TOTAL_BYTE … #define ZYNQMP_DMA_RATE_CTRL … #define ZYNQMP_DMA_IRQ_SRC_ACCT … #define ZYNQMP_DMA_IRQ_DST_ACCT … #define ZYNQMP_DMA_CTRL2 … /* Interrupt registers bit field definitions */ #define ZYNQMP_DMA_DONE … #define ZYNQMP_DMA_AXI_WR_DATA … #define ZYNQMP_DMA_AXI_RD_DATA … #define ZYNQMP_DMA_AXI_RD_DST_DSCR … #define ZYNQMP_DMA_AXI_RD_SRC_DSCR … #define ZYNQMP_DMA_IRQ_DST_ACCT_ERR … #define ZYNQMP_DMA_IRQ_SRC_ACCT_ERR … #define ZYNQMP_DMA_BYTE_CNT_OVRFL … #define ZYNQMP_DMA_DST_DSCR_DONE … #define ZYNQMP_DMA_INV_APB … /* Control 0 register bit field definitions */ #define ZYNQMP_DMA_OVR_FETCH … #define ZYNQMP_DMA_POINT_TYPE_SG … #define ZYNQMP_DMA_RATE_CTRL_EN … /* Control 1 register bit field definitions */ #define ZYNQMP_DMA_SRC_ISSUE … /* Data Attribute register bit field definitions */ #define ZYNQMP_DMA_ARBURST … #define ZYNQMP_DMA_ARCACHE … #define ZYNQMP_DMA_ARCACHE_OFST … #define ZYNQMP_DMA_ARQOS … #define ZYNQMP_DMA_ARQOS_OFST … #define ZYNQMP_DMA_ARLEN … #define ZYNQMP_DMA_ARLEN_OFST … #define ZYNQMP_DMA_AWBURST … #define ZYNQMP_DMA_AWCACHE … #define ZYNQMP_DMA_AWCACHE_OFST … #define ZYNQMP_DMA_AWQOS … #define ZYNQMP_DMA_AWQOS_OFST … #define ZYNQMP_DMA_AWLEN … #define ZYNQMP_DMA_AWLEN_OFST … /* Descriptor Attribute register bit field definitions */ #define ZYNQMP_DMA_AXCOHRNT … #define ZYNQMP_DMA_AXCACHE … #define ZYNQMP_DMA_AXCACHE_OFST … #define ZYNQMP_DMA_AXQOS … #define ZYNQMP_DMA_AXQOS_OFST … /* Control register 2 bit field definitions */ #define ZYNQMP_DMA_ENABLE … /* Buffer Descriptor definitions */ #define ZYNQMP_DMA_DESC_CTRL_STOP … #define ZYNQMP_DMA_DESC_CTRL_COMP_INT … #define ZYNQMP_DMA_DESC_CTRL_SIZE_256 … #define ZYNQMP_DMA_DESC_CTRL_COHRNT … /* Interrupt Mask specific definitions */ #define ZYNQMP_DMA_INT_ERR … #define ZYNQMP_DMA_INT_OVRFL … #define ZYNQMP_DMA_INT_DONE … #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK … /* Max number of descriptors per channel */ #define ZYNQMP_DMA_NUM_DESCS … /* Max transfer size per descriptor */ #define ZYNQMP_DMA_MAX_TRANS_LEN … /* Max burst lengths */ #define ZYNQMP_DMA_MAX_DST_BURST_LEN … #define ZYNQMP_DMA_MAX_SRC_BURST_LEN … /* Reset values for data attributes */ #define ZYNQMP_DMA_AXCACHE_VAL … #define ZYNQMP_DMA_SRC_ISSUE_RST_VAL … #define ZYNQMP_DMA_IDS_DEFAULT_MASK … /* Bus width in bits */ #define ZYNQMP_DMA_BUS_WIDTH_64 … #define ZYNQMP_DMA_BUS_WIDTH_128 … #define ZDMA_PM_TIMEOUT … #define ZYNQMP_DMA_DESC_SIZE(chan) … #define to_chan(chan) … #define tx_to_desc(tx) … /* IRQ Register offset for Versal Gen 2 */ #define IRQ_REG_OFFSET … /** * struct zynqmp_dma_desc_ll - Hw linked list descriptor * @addr: Buffer address * @size: Size of the buffer * @ctrl: Control word * @nxtdscraddr: Next descriptor base address * @rsvd: Reserved field and for Hw internal use. */ struct zynqmp_dma_desc_ll { … }; /** * struct zynqmp_dma_desc_sw - Per Transaction structure * @src: Source address for simple mode dma * @dst: Destination address for simple mode dma * @len: Transfer length for simple mode dma * @node: Node in the channel descriptor list * @tx_list: List head for the current transfer * @async_tx: Async transaction descriptor * @src_v: Virtual address of the src descriptor * @src_p: Physical address of the src descriptor * @dst_v: Virtual address of the dst descriptor * @dst_p: Physical address of the dst descriptor */ struct zynqmp_dma_desc_sw { … }; /** * struct zynqmp_dma_chan - Driver specific DMA channel structure * @zdev: Driver specific device structure * @regs: Control registers offset * @lock: Descriptor operation lock * @pending_list: Descriptors waiting * @free_list: Descriptors free * @active_list: Descriptors active * @sw_desc_pool: SW descriptor pool * @done_list: Complete descriptors * @common: DMA common channel * @desc_pool_v: Statically allocated descriptor base * @desc_pool_p: Physical allocated descriptor base * @desc_free_cnt: Descriptor available count * @dev: The dma device * @irq: Channel IRQ * @is_dmacoherent: Tells whether dma operations are coherent or not * @tasklet: Cleanup work after irq * @idle : Channel status; * @desc_size: Size of the low level descriptor * @err: Channel has errors * @bus_width: Bus width * @src_burst_len: Source burst length * @dst_burst_len: Dest burst length * @irq_offset: Irq register offset */ struct zynqmp_dma_chan { … }; /** * struct zynqmp_dma_device - DMA device structure * @dev: Device Structure * @common: DMA device structure * @chan: Driver specific DMA channel * @clk_main: Pointer to main clock * @clk_apb: Pointer to apb clock */ struct zynqmp_dma_device { … }; struct zynqmp_dma_config { … }; static const struct zynqmp_dma_config versal2_dma_config = …; static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, u64 value) { … } /** * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller * @chan: ZynqMP DMA DMA channel pointer * @desc: Transaction descriptor pointer */ static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan, struct zynqmp_dma_desc_sw *desc) { … } /** * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor * @chan: ZynqMP DMA channel pointer * @desc: Hw descriptor pointer */ static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan, void *desc) { … } /** * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor * @chan: ZynqMP DMA channel pointer * @sdesc: Hw descriptor pointer * @src: Source buffer address * @dst: Destination buffer address * @len: Transfer length * @prev: Previous hw descriptor pointer */ static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan, struct zynqmp_dma_desc_ll *sdesc, dma_addr_t src, dma_addr_t dst, size_t len, struct zynqmp_dma_desc_ll *prev) { … } /** * zynqmp_dma_init - Initialize the channel * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_tx_submit - Submit DMA transaction * @tx: Async transaction descriptor pointer * * Return: cookie value */ static dma_cookie_t zynqmp_dma_tx_submit(struct dma_async_tx_descriptor *tx) { … } /** * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool * @chan: ZynqMP DMA channel pointer * * Return: The sw descriptor */ static struct zynqmp_dma_desc_sw * zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_free_descriptor - Issue pending transactions * @chan: ZynqMP DMA channel pointer * @sdesc: Transaction descriptor pointer */ static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan, struct zynqmp_dma_desc_sw *sdesc) { … } /** * zynqmp_dma_free_desc_list - Free descriptors list * @chan: ZynqMP DMA channel pointer * @list: List to parse and delete the descriptor */ static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan, struct list_head *list) { … } /** * zynqmp_dma_alloc_chan_resources - Allocate channel resources * @dchan: DMA channel * * Return: Number of descriptors on success and failure value on error */ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) { … } /** * zynqmp_dma_start - Start DMA channel * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_start(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt * @chan: ZynqMP DMA channel pointer * @status: Interrupt status value */ static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status) { … } static void zynqmp_dma_config(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_device_config - Zynqmp dma device configuration * @dchan: DMA channel * @config: DMA device config * * Return: 0 always */ static int zynqmp_dma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) { … } /** * zynqmp_dma_start_transfer - Initiate the new transfer * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors * @chan: ZynqMP DMA channel */ static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_issue_pending - Issue pending transactions * @dchan: DMA channel pointer */ static void zynqmp_dma_issue_pending(struct dma_chan *dchan) { … } /** * zynqmp_dma_free_descriptors - Free channel descriptors * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_free_chan_resources - Free channel resources * @dchan: DMA channel pointer */ static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan) { … } /** * zynqmp_dma_reset - Reset the channel * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler * @irq: IRQ number * @data: Pointer to the ZynqMP DMA channel structure * * Return: IRQ_HANDLED/IRQ_NONE */ static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data) { … } /** * zynqmp_dma_do_tasklet - Schedule completion tasklet * @t: Pointer to the ZynqMP DMA channel structure */ static void zynqmp_dma_do_tasklet(struct tasklet_struct *t) { … } /** * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel * @dchan: DMA channel pointer * * Return: Always '0' */ static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan) { … } /** * zynqmp_dma_synchronize - Synchronizes the termination of a transfers to the current context. * @dchan: DMA channel pointer */ static void zynqmp_dma_synchronize(struct dma_chan *dchan) { … } /** * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction * @dchan: DMA channel * @dma_dst: Destination buffer address * @dma_src: Source buffer address * @len: Transfer length * @flags: transfer ack flags * * Return: Async transaction descriptor on success and NULL on failure */ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, ulong flags) { … } /** * zynqmp_dma_chan_remove - Channel remove function * @chan: ZynqMP DMA channel pointer */ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) { … } /** * zynqmp_dma_chan_probe - Per Channel Probing * @zdev: Driver specific device structure * @pdev: Pointer to the platform_device structure * * Return: '0' on success and failure value on error */ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, struct platform_device *pdev) { … } /** * of_zynqmp_dma_xlate - Translation function * @dma_spec: Pointer to DMA specifier as found in the device tree * @ofdma: Pointer to DMA controller data * * Return: DMA channel pointer on success and NULL on error */ static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { … } /** * zynqmp_dma_suspend - Suspend method for the driver * @dev: Address of the device structure * * Put the driver into low power mode. * Return: 0 on success and failure value on error */ static int __maybe_unused zynqmp_dma_suspend(struct device *dev) { … } /** * zynqmp_dma_resume - Resume from suspend * @dev: Address of the device structure * * Resume operation after suspend. * Return: 0 on success and failure value on error */ static int __maybe_unused zynqmp_dma_resume(struct device *dev) { … } /** * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver * @dev: Address of the device structure * * Put the driver into low power mode. * Return: 0 always */ static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev) { … } /** * zynqmp_dma_runtime_resume - Runtime suspend method for the driver * @dev: Address of the device structure * * Put the driver into low power mode. * Return: 0 always */ static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev) { … } static const struct dev_pm_ops zynqmp_dma_dev_pm_ops = …; /** * zynqmp_dma_probe - Driver probe function * @pdev: Pointer to the platform_device structure * * Return: '0' on success and failure value on error */ static int zynqmp_dma_probe(struct platform_device *pdev) { … } /** * zynqmp_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure * * Return: Always '0' */ static void zynqmp_dma_remove(struct platform_device *pdev) { … } static const struct of_device_id zynqmp_dma_of_match[] = …; MODULE_DEVICE_TABLE(of, zynqmp_dma_of_match); static struct platform_driver zynqmp_dma_driver = …; module_platform_driver(…) …; MODULE_LICENSE(…) …; MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …;
Codebase Browser
linux