// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2013-2015 Analog Devices Inc. * Author: Lars-Peter Clausen <[email protected]> */ #include <linux/atomic.h> #include <linux/cleanup.h> #include <linux/slab.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/device.h> #include <linux/workqueue.h> #include <linux/mutex.h> #include <linux/sched.h> #include <linux/poll.h> #include <linux/iio/buffer_impl.h> #include <linux/iio/buffer-dma.h> #include <linux/dma-buf.h> #include <linux/dma-fence.h> #include <linux/dma-mapping.h> #include <linux/sizes.h> /* * For DMA buffers the storage is sub-divided into so called blocks. Each block * has its own memory buffer. The size of the block is the granularity at which * memory is exchanged between the hardware and the application. Increasing the * basic unit of data exchange from one sample to one block decreases the * management overhead that is associated with each sample. E.g. if we say the * management overhead for one exchange is x and the unit of exchange is one * sample the overhead will be x for each sample. Whereas when using a block * which contains n samples the overhead per sample is reduced to x/n. This * allows to achieve much higher samplerates than what can be sustained with * the one sample approach. * * Blocks are exchanged between the DMA controller and the application via the * means of two queues. The incoming queue and the outgoing queue. Blocks on the * incoming queue are waiting for the DMA controller to pick them up and fill * them with data. Block on the outgoing queue have been filled with data and * are waiting for the application to dequeue them and read the data. * * A block can be in one of the following states: * * Owned by the application. In this state the application can read data from * the block. * * On the incoming list: Blocks on the incoming list are queued up to be * processed by the DMA controller. * * Owned by the DMA controller: The DMA controller is processing the block * and filling it with data. * * On the outgoing list: Blocks on the outgoing list have been successfully * processed by the DMA controller and contain data. They can be dequeued by * the application. * * Dead: A block that is dead has been marked as to be freed. It might still * be owned by either the application or the DMA controller at the moment. * But once they are done processing it instead of going to either the * incoming or outgoing queue the block will be freed. * * In addition to this blocks are reference counted and the memory associated * with both the block structure as well as the storage memory for the block * will be freed when the last reference to the block is dropped. This means a * block must not be accessed without holding a reference. * * The iio_dma_buffer implementation provides a generic infrastructure for * managing the blocks. * * A driver for a specific piece of hardware that has DMA capabilities need to * implement the submit() callback from the iio_dma_buffer_ops structure. This * callback is supposed to initiate the DMA transfer copying data from the * converter to the memory region of the block. Once the DMA transfer has been * completed the driver must call iio_dma_buffer_block_done() for the completed * block. * * Prior to this it must set the bytes_used field of the block contains * the actual number of bytes in the buffer. Typically this will be equal to the * size of the block, but if the DMA hardware has certain alignment requirements * for the transfer length it might choose to use less than the full size. In * either case it is expected that bytes_used is a multiple of the bytes per * datum, i.e. the block must not contain partial samples. * * The driver must call iio_dma_buffer_block_done() for each block it has * received through its submit_block() callback, even if it does not actually * perform a DMA transfer for the block, e.g. because the buffer was disabled * before the block transfer was started. In this case it should set bytes_used * to 0. * * In addition it is recommended that a driver implements the abort() callback. * It will be called when the buffer is disabled and can be used to cancel * pending and stop active transfers. * * The specific driver implementation should use the default callback * implementations provided by this module for the iio_buffer_access_funcs * struct. It may overload some callbacks with custom variants if the hardware * has special requirements that are not handled by the generic functions. If a * driver chooses to overload a callback it has to ensure that the generic * callback is called from within the custom callback. */ static void iio_buffer_block_release(struct kref *kref) { … } static void iio_buffer_block_get(struct iio_dma_buffer_block *block) { … } static void iio_buffer_block_put(struct iio_dma_buffer_block *block) { … } /* * dma_free_coherent can sleep, hence we need to take some special care to be * able to drop a reference from an atomic context. */ static LIST_HEAD(iio_dma_buffer_dead_blocks); static DEFINE_SPINLOCK(iio_dma_buffer_dead_blocks_lock); static void iio_dma_buffer_cleanup_worker(struct work_struct *work) { … } static DECLARE_WORK(iio_dma_buffer_cleanup_work, iio_dma_buffer_cleanup_worker); static void iio_buffer_block_release_atomic(struct kref *kref) { … } /* * Version of iio_buffer_block_put() that can be called from atomic context */ static void iio_buffer_block_put_atomic(struct iio_dma_buffer_block *block) { … } static struct iio_dma_buffer_queue *iio_buffer_to_queue(struct iio_buffer *buf) { … } static struct iio_dma_buffer_block *iio_dma_buffer_alloc_block( struct iio_dma_buffer_queue *queue, size_t size, bool fileio) { … } static void _iio_dma_buffer_block_done(struct iio_dma_buffer_block *block) { … } static void iio_dma_buffer_queue_wake(struct iio_dma_buffer_queue *queue) { … } /** * iio_dma_buffer_block_done() - Indicate that a block has been completed * @block: The completed block * * Should be called when the DMA controller has finished handling the block to * pass back ownership of the block to the queue. */ void iio_dma_buffer_block_done(struct iio_dma_buffer_block *block) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_block_list_abort() - Indicate that a list block has been * aborted * @queue: Queue for which to complete blocks. * @list: List of aborted blocks. All blocks in this list must be from @queue. * * Typically called from the abort() callback after the DMA controller has been * stopped. This will set bytes_used to 0 for each block in the list and then * hand the blocks back to the queue. */ void iio_dma_buffer_block_list_abort(struct iio_dma_buffer_queue *queue, struct list_head *list) { … } EXPORT_SYMBOL_GPL(…); static bool iio_dma_block_reusable(struct iio_dma_buffer_block *block) { … } static bool iio_dma_buffer_can_use_fileio(struct iio_dma_buffer_queue *queue) { … } /** * iio_dma_buffer_request_update() - DMA buffer request_update callback * @buffer: The buffer which to request an update * * Should be used as the iio_dma_buffer_request_update() callback for * iio_buffer_access_ops struct for DMA buffers. */ int iio_dma_buffer_request_update(struct iio_buffer *buffer) { … } EXPORT_SYMBOL_GPL(…); static void iio_dma_buffer_fileio_free(struct iio_dma_buffer_queue *queue) { … } static void iio_dma_buffer_submit_block(struct iio_dma_buffer_queue *queue, struct iio_dma_buffer_block *block) { … } /** * iio_dma_buffer_enable() - Enable DMA buffer * @buffer: IIO buffer to enable * @indio_dev: IIO device the buffer is attached to * * Needs to be called when the device that the buffer is attached to starts * sampling. Typically should be the iio_buffer_access_ops enable callback. * * This will allocate the DMA buffers and start the DMA transfers. */ int iio_dma_buffer_enable(struct iio_buffer *buffer, struct iio_dev *indio_dev) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_disable() - Disable DMA buffer * @buffer: IIO DMA buffer to disable * @indio_dev: IIO device the buffer is attached to * * Needs to be called when the device that the buffer is attached to stops * sampling. Typically should be the iio_buffer_access_ops disable callback. */ int iio_dma_buffer_disable(struct iio_buffer *buffer, struct iio_dev *indio_dev) { … } EXPORT_SYMBOL_GPL(…); static void iio_dma_buffer_enqueue(struct iio_dma_buffer_queue *queue, struct iio_dma_buffer_block *block) { … } static struct iio_dma_buffer_block *iio_dma_buffer_dequeue( struct iio_dma_buffer_queue *queue) { … } static int iio_dma_buffer_io(struct iio_buffer *buffer, size_t n, char __user *user_buffer, bool is_from_user) { … } /** * iio_dma_buffer_read() - DMA buffer read callback * @buffer: Buffer to read form * @n: Number of bytes to read * @user_buffer: Userspace buffer to copy the data to * * Should be used as the read callback for iio_buffer_access_ops * struct for DMA buffers. */ int iio_dma_buffer_read(struct iio_buffer *buffer, size_t n, char __user *user_buffer) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_write() - DMA buffer write callback * @buffer: Buffer to read form * @n: Number of bytes to read * @user_buffer: Userspace buffer to copy the data from * * Should be used as the write callback for iio_buffer_access_ops * struct for DMA buffers. */ int iio_dma_buffer_write(struct iio_buffer *buffer, size_t n, const char __user *user_buffer) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_usage() - DMA buffer data_available and * space_available callback * @buf: Buffer to check for data availability * * Should be used as the data_available and space_available callbacks for * iio_buffer_access_ops struct for DMA buffers. */ size_t iio_dma_buffer_usage(struct iio_buffer *buf) { … } EXPORT_SYMBOL_GPL(…); struct iio_dma_buffer_block * iio_dma_buffer_attach_dmabuf(struct iio_buffer *buffer, struct dma_buf_attachment *attach) { … } EXPORT_SYMBOL_GPL(…); void iio_dma_buffer_detach_dmabuf(struct iio_buffer *buffer, struct iio_dma_buffer_block *block) { … } EXPORT_SYMBOL_GPL(…); static int iio_dma_can_enqueue_block(struct iio_dma_buffer_block *block) { … } int iio_dma_buffer_enqueue_dmabuf(struct iio_buffer *buffer, struct iio_dma_buffer_block *block, struct dma_fence *fence, struct sg_table *sgt, size_t size, bool cyclic) { … } EXPORT_SYMBOL_GPL(…); void iio_dma_buffer_lock_queue(struct iio_buffer *buffer) { … } EXPORT_SYMBOL_GPL(…); void iio_dma_buffer_unlock_queue(struct iio_buffer *buffer) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_set_bytes_per_datum() - DMA buffer set_bytes_per_datum callback * @buffer: Buffer to set the bytes-per-datum for * @bpd: The new bytes-per-datum value * * Should be used as the set_bytes_per_datum callback for iio_buffer_access_ops * struct for DMA buffers. */ int iio_dma_buffer_set_bytes_per_datum(struct iio_buffer *buffer, size_t bpd) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_set_length - DMA buffer set_length callback * @buffer: Buffer to set the length for * @length: The new buffer length * * Should be used as the set_length callback for iio_buffer_access_ops * struct for DMA buffers. */ int iio_dma_buffer_set_length(struct iio_buffer *buffer, unsigned int length) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_init() - Initialize DMA buffer queue * @queue: Buffer to initialize * @dev: DMA device * @ops: DMA buffer queue callback operations * * The DMA device will be used by the queue to do DMA memory allocations. So it * should refer to the device that will perform the DMA to ensure that * allocations are done from a memory region that can be accessed by the device. */ int iio_dma_buffer_init(struct iio_dma_buffer_queue *queue, struct device *dev, const struct iio_dma_buffer_ops *ops) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_exit() - Cleanup DMA buffer queue * @queue: Buffer to cleanup * * After this function has completed it is safe to free any resources that are * associated with the buffer and are accessed inside the callback operations. */ void iio_dma_buffer_exit(struct iio_dma_buffer_queue *queue) { … } EXPORT_SYMBOL_GPL(…); /** * iio_dma_buffer_release() - Release final buffer resources * @queue: Buffer to release * * Frees resources that can't yet be freed in iio_dma_buffer_exit(). Should be * called in the buffers release callback implementation right before freeing * the memory associated with the buffer. */ void iio_dma_buffer_release(struct iio_dma_buffer_queue *queue) { … } EXPORT_SYMBOL_GPL(…); MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux