// SPDX-License-Identifier: GPL-2.0 /* * Renesas R-Car Gen2/Gen3 DMA Controller Driver * * Copyright (C) 2014-2019 Renesas Electronics Inc. * * Author: Laurent Pinchart <[email protected]> */ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/list.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> #include <linux/spinlock.h> #include "../dmaengine.h" /* * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer * @node: entry in the parent's chunks list * @src_addr: device source address * @dst_addr: device destination address * @size: transfer size in bytes */ struct rcar_dmac_xfer_chunk { … }; /* * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk * @sar: value of the SAR register (source address) * @dar: value of the DAR register (destination address) * @tcr: value of the TCR register (transfer count) */ struct rcar_dmac_hw_desc { … } __attribute__((packed)); /* * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor * @async_tx: base DMA asynchronous transaction descriptor * @direction: direction of the DMA transfer * @xfer_shift: log2 of the transfer size * @chcr: value of the channel configuration register for this transfer * @node: entry in the channel's descriptors lists * @chunks: list of transfer chunks for this transfer * @running: the transfer chunk being currently processed * @nchunks: number of transfer chunks for this transfer * @hwdescs.use: whether the transfer descriptor uses hardware descriptors * @hwdescs.mem: hardware descriptors memory for the transfer * @hwdescs.dma: device address of the hardware descriptors memory * @hwdescs.size: size of the hardware descriptors in bytes * @size: transfer size in bytes * @cyclic: when set indicates that the DMA transfer is cyclic */ struct rcar_dmac_desc { … }; #define to_rcar_dmac_desc(d) … /* * struct rcar_dmac_desc_page - One page worth of descriptors * @node: entry in the channel's pages list * @descs: array of DMA descriptors * @chunks: array of transfer chunk descriptors */ struct rcar_dmac_desc_page { … }; #define RCAR_DMAC_DESCS_PER_PAGE … #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE … /* * struct rcar_dmac_chan_slave - Slave configuration * @slave_addr: slave memory address * @xfer_size: size (in bytes) of hardware transfers */ struct rcar_dmac_chan_slave { … }; /* * struct rcar_dmac_chan_map - Map of slave device phys to dma address * @addr: slave dma address * @dir: direction of mapping * @slave: slave configuration that is mapped */ struct rcar_dmac_chan_map { … }; /* * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel * @chan: base DMA channel object * @iomem: channel I/O memory base * @index: index of this channel in the controller * @irq: channel IRQ * @src: slave memory address and size on the source side * @dst: slave memory address and size on the destination side * @mid_rid: hardware MID/RID for the DMA client using this channel * @lock: protects the channel CHCR register and the desc members * @desc.free: list of free descriptors * @desc.pending: list of pending descriptors (submitted with tx_submit) * @desc.active: list of active descriptors (activated with issue_pending) * @desc.done: list of completed descriptors * @desc.wait: list of descriptors waiting for an ack * @desc.running: the descriptor being processed (a member of the active list) * @desc.chunks_free: list of free transfer chunk descriptors * @desc.pages: list of pages used by allocated descriptors */ struct rcar_dmac_chan { … }; #define to_rcar_dmac_chan(c) … /* * struct rcar_dmac - R-Car Gen2 DMA Controller * @engine: base DMA engine object * @dev: the hardware device * @dmac_base: remapped base register block * @chan_base: remapped channel register block (optional) * @n_channels: number of available channels * @channels: array of DMAC channels * @channels_mask: bitfield of which DMA channels are managed by this driver * @modules: bitmask of client modules in use */ struct rcar_dmac { … }; #define to_rcar_dmac(d) … #define for_each_rcar_dmac_chan(i, dmac, chan) … /* * struct rcar_dmac_of_data - This driver's OF data * @chan_offset_base: DMAC channels base offset * @chan_offset_stride: DMAC channels offset stride */ struct rcar_dmac_of_data { … }; /* ----------------------------------------------------------------------------- * Registers */ #define RCAR_DMAISTA … #define RCAR_DMASEC … #define RCAR_DMAOR … #define RCAR_DMAOR_PRI_FIXED … #define RCAR_DMAOR_PRI_ROUND_ROBIN … #define RCAR_DMAOR_AE … #define RCAR_DMAOR_DME … #define RCAR_DMACHCLR … #define RCAR_DMADPSEC … #define RCAR_DMASAR … #define RCAR_DMADAR … #define RCAR_DMATCR … #define RCAR_DMATCR_MASK … #define RCAR_DMATSR … #define RCAR_DMACHCR … #define RCAR_DMACHCR_CAE … #define RCAR_DMACHCR_CAIE … #define RCAR_DMACHCR_DPM_DISABLED … #define RCAR_DMACHCR_DPM_ENABLED … #define RCAR_DMACHCR_DPM_REPEAT … #define RCAR_DMACHCR_DPM_INFINITE … #define RCAR_DMACHCR_RPT_SAR … #define RCAR_DMACHCR_RPT_DAR … #define RCAR_DMACHCR_RPT_TCR … #define RCAR_DMACHCR_DPB … #define RCAR_DMACHCR_DSE … #define RCAR_DMACHCR_DSIE … #define RCAR_DMACHCR_TS_1B … #define RCAR_DMACHCR_TS_2B … #define RCAR_DMACHCR_TS_4B … #define RCAR_DMACHCR_TS_16B … #define RCAR_DMACHCR_TS_32B … #define RCAR_DMACHCR_TS_64B … #define RCAR_DMACHCR_TS_8B … #define RCAR_DMACHCR_DM_FIXED … #define RCAR_DMACHCR_DM_INC … #define RCAR_DMACHCR_DM_DEC … #define RCAR_DMACHCR_SM_FIXED … #define RCAR_DMACHCR_SM_INC … #define RCAR_DMACHCR_SM_DEC … #define RCAR_DMACHCR_RS_AUTO … #define RCAR_DMACHCR_RS_DMARS … #define RCAR_DMACHCR_IE … #define RCAR_DMACHCR_TE … #define RCAR_DMACHCR_DE … #define RCAR_DMATCRB … #define RCAR_DMATSRB … #define RCAR_DMACHCRB … #define RCAR_DMACHCRB_DCNT(n) … #define RCAR_DMACHCRB_DPTR_MASK … #define RCAR_DMACHCRB_DPTR_SHIFT … #define RCAR_DMACHCRB_DRST … #define RCAR_DMACHCRB_DTS … #define RCAR_DMACHCRB_SLM_NORMAL … #define RCAR_DMACHCRB_SLM_CLK(n) … #define RCAR_DMACHCRB_PRI(n) … #define RCAR_DMARS … #define RCAR_DMABUFCR … #define RCAR_DMABUFCR_MBU(n) … #define RCAR_DMABUFCR_ULB(n) … #define RCAR_DMADPBASE … #define RCAR_DMADPBASE_MASK … #define RCAR_DMADPBASE_SEL … #define RCAR_DMADPCR … #define RCAR_DMADPCR_DIPT(n) … #define RCAR_DMAFIXSAR … #define RCAR_DMAFIXDAR … #define RCAR_DMAFIXDPBASE … /* For R-Car Gen4 */ #define RCAR_GEN4_DMACHCLR … /* Hardcode the MEMCPY transfer size to 4 bytes. */ #define RCAR_DMAC_MEMCPY_XFER_SIZE … /* ----------------------------------------------------------------------------- * Device access */ static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) { … } static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) { … } static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) { … } static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) { … } static void rcar_dmac_chan_clear(struct rcar_dmac *dmac, struct rcar_dmac_chan *chan) { … } static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac) { … } /* ----------------------------------------------------------------------------- * Initialization and configuration */ static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) { … } static int rcar_dmac_init(struct rcar_dmac *dmac) { … } /* ----------------------------------------------------------------------------- * Descriptors submission */ static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) { … } /* ----------------------------------------------------------------------------- * Descriptors allocation and free */ /* * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors * @chan: the DMA channel * @gfp: allocation flags */ static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) { … } /* * rcar_dmac_desc_put - Release a DMA transfer descriptor * @chan: the DMA channel * @desc: the descriptor * * Put the descriptor and its transfer chunk descriptors back in the channel's * free descriptors lists. The descriptor's chunks list will be reinitialized to * an empty list as a result. * * The descriptor must have been removed from the channel's lists before calling * this function. */ static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc) { … } static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) { … } /* * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer * @chan: the DMA channel * * Locking: This function must be called in a non-atomic context. * * Return: A pointer to the allocated descriptor or NULL if no descriptor can * be allocated. */ static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) { … } /* * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks * @chan: the DMA channel * @gfp: allocation flags */ static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) { … } /* * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer * @chan: the DMA channel * * Locking: This function must be called in a non-atomic context. * * Return: A pointer to the allocated transfer chunk descriptor or NULL if no * descriptor can be allocated. */ static struct rcar_dmac_xfer_chunk * rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc, size_t size) { … } static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc) { … } /* ----------------------------------------------------------------------------- * Stop and reset */ static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) { … } static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) { … } static int rcar_dmac_chan_pause(struct dma_chan *chan) { … } /* ----------------------------------------------------------------------------- * Descriptors preparation */ static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc) { … } /* * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list * * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also * converted to scatter-gather to guarantee consistent locking and a correct * list manipulation. For slave DMA direction carries the usual meaning, and, * logically, the SG list is RAM and the addr variable contains slave address, * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM * and the SG list contains only one element and points at the source buffer. */ static struct dma_async_tx_descriptor * rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, unsigned int sg_len, dma_addr_t dev_addr, enum dma_transfer_direction dir, unsigned long dma_flags, bool cyclic) { … } /* ----------------------------------------------------------------------------- * DMA engine operations */ static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) { … } static void rcar_dmac_free_chan_resources(struct dma_chan *chan) { … } static struct dma_async_tx_descriptor * rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, size_t len, unsigned long flags) { … } static int rcar_dmac_map_slave_addr(struct dma_chan *chan, enum dma_transfer_direction dir) { … } static struct dma_async_tx_descriptor * rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { … } #define RCAR_DMAC_MAX_SG_LEN … static struct dma_async_tx_descriptor * rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { … } static int rcar_dmac_device_config(struct dma_chan *chan, struct dma_slave_config *cfg) { … } static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) { … } static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, dma_cookie_t cookie) { … } static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { … } static void rcar_dmac_issue_pending(struct dma_chan *chan) { … } static void rcar_dmac_device_synchronize(struct dma_chan *chan) { … } /* ----------------------------------------------------------------------------- * IRQ handling */ static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) { … } static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) { … } static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) { … } static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) { … } /* ----------------------------------------------------------------------------- * OF xlate and channel filter */ static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) { … } static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { … } /* ----------------------------------------------------------------------------- * Power management */ #ifdef CONFIG_PM static int rcar_dmac_runtime_suspend(struct device *dev) { … } static int rcar_dmac_runtime_resume(struct device *dev) { … } #endif static const struct dev_pm_ops rcar_dmac_pm = …; /* ----------------------------------------------------------------------------- * Probe and remove */ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, struct rcar_dmac_chan *rchan) { … } #define RCAR_DMAC_MAX_CHANNELS … static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) { … } static int rcar_dmac_probe(struct platform_device *pdev) { … } static void rcar_dmac_remove(struct platform_device *pdev) { … } static void rcar_dmac_shutdown(struct platform_device *pdev) { … } static const struct rcar_dmac_of_data rcar_dmac_data = …; static const struct rcar_dmac_of_data rcar_gen4_dmac_data = …; static const struct of_device_id rcar_dmac_of_ids[] = …; MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); static struct platform_driver rcar_dmac_driver = …; module_platform_driver(…) …; MODULE_DESCRIPTION(…) …; MODULE_AUTHOR(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux