// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. * Copyright (C) 2019-2024 Linaro Ltd. */ #include <linux/bitfield.h> #include <linux/bits.h> #include <linux/dma-direction.h> #include <linux/refcount.h> #include <linux/scatterlist.h> #include <linux/types.h> #include "gsi.h" #include "gsi_private.h" #include "gsi_trans.h" #include "ipa_cmd.h" #include "ipa_data.h" #include "ipa_gsi.h" /** * DOC: GSI Transactions * * A GSI transaction abstracts the behavior of a GSI channel by representing * everything about a related group of IPA operations in a single structure. * (A "operation" in this sense is either a data transfer or an IPA immediate * command.) Most details of interaction with the GSI hardware are managed * by the GSI transaction core, allowing users to simply describe operations * to be performed. When a transaction has completed a callback function * (dependent on the type of endpoint associated with the channel) allows * cleanup of resources associated with the transaction. * * To perform an operation (or set of them), a user of the GSI transaction * interface allocates a transaction, indicating the number of TREs required * (one per operation). If sufficient TREs are available, they are reserved * for use in the transaction and the allocation succeeds. This way * exhaustion of the available TREs in a channel ring is detected as early * as possible. Any other resources that might be needed to complete a * transaction are also allocated when the transaction is allocated. * * Operations performed as part of a transaction are represented in an array * of Linux scatterlist structures, allocated with the transaction. These * scatterlist structures are initialized by "adding" operations to the * transaction. If a buffer in an operation must be mapped for DMA, this is * done at the time it is added to the transaction. It is possible for a * mapping error to occur when an operation is added. In this case the * transaction should simply be freed; this correctly releases resources * associated with the transaction. * * Once all operations have been successfully added to a transaction, the * transaction is committed. Committing transfers ownership of the entire * transaction to the GSI transaction core. The GSI transaction code * formats the content of the scatterlist array into the channel ring * buffer and informs the hardware that new TREs are available to process. * * The last TRE in each transaction is marked to interrupt the AP when the * GSI hardware has completed it. Because transfers described by TREs are * performed strictly in order, signaling the completion of just the last * TRE in the transaction is sufficient to indicate the full transaction * is complete. * * When a transaction is complete, ipa_gsi_trans_complete() is called by the * GSI code into the IPA layer, allowing it to perform any final cleanup * required before the transaction is freed. */ /* Hardware values representing a transfer element type */ enum gsi_tre_type { … }; /* An entry in a channel ring */ struct gsi_tre { … }; /* gsi_tre->flags mask values (in CPU byte order) */ #define TRE_FLAGS_CHAIN_FMASK … #define TRE_FLAGS_IEOT_FMASK … #define TRE_FLAGS_BEI_FMASK … #define TRE_FLAGS_TYPE_FMASK … int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count, u32 max_alloc) { … } void gsi_trans_pool_exit(struct gsi_trans_pool *pool) { … } /* Home-grown DMA pool. This way we can preallocate the pool, and guarantee * allocations will succeed. The immediate commands in a transaction can * require up to max_alloc elements from the pool. But we only allow * allocation of a single element from a DMA pool at a time. */ int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool, size_t size, u32 count, u32 max_alloc) { … } void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool) { … } /* Return the byte offset of the next free entry in the pool */ static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count) { … } /* Allocate a contiguous block of zeroed entries from a pool */ void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count) { … } /* Allocate a single zeroed entry from a DMA pool */ void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr) { … } /* Map a TRE ring entry index to the transaction it is associated with */ static void gsi_trans_map(struct gsi_trans *trans, u32 index) { … } /* Return the transaction mapped to a given ring entry */ struct gsi_trans * gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) { … } /* Return the oldest completed transaction for a channel (or null) */ struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) { … } /* Move a transaction from allocated to committed state */ static void gsi_trans_move_committed(struct gsi_trans *trans) { … } /* Move committed transactions to pending state */ static void gsi_trans_move_pending(struct gsi_trans *trans) { … } /* Move pending transactions to completed state */ void gsi_trans_move_complete(struct gsi_trans *trans) { … } /* Move a transaction from completed to polled state */ void gsi_trans_move_polled(struct gsi_trans *trans) { … } /* Reserve some number of TREs on a channel. Returns true if successful */ static bool gsi_trans_tre_reserve(struct gsi_trans_info *trans_info, u32 tre_count) { … } /* Release previously-reserved TRE entries to a channel */ static void gsi_trans_tre_release(struct gsi_trans_info *trans_info, u32 tre_count) { … } /* Return true if no transactions are allocated, false otherwise */ bool gsi_channel_trans_idle(struct gsi *gsi, u32 channel_id) { … } /* Allocate a GSI transaction on a channel */ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, u32 tre_count, enum dma_data_direction direction) { … } /* Free a previously-allocated transaction */ void gsi_trans_free(struct gsi_trans *trans) { … } /* Add an immediate command to a transaction */ void gsi_trans_cmd_add(struct gsi_trans *trans, void *buf, u32 size, dma_addr_t addr, enum ipa_cmd_opcode opcode) { … } /* Add a page transfer to a transaction. It will fill the only TRE. */ int gsi_trans_page_add(struct gsi_trans *trans, struct page *page, u32 size, u32 offset) { … } /* Add an SKB transfer to a transaction. No other TREs will be used. */ int gsi_trans_skb_add(struct gsi_trans *trans, struct sk_buff *skb) { … } /* Compute the length/opcode value to use for a TRE */ static __le16 gsi_tre_len_opcode(enum ipa_cmd_opcode opcode, u32 len) { … } /* Compute the flags value to use for a given TRE */ static __le32 gsi_tre_flags(bool last_tre, bool bei, enum ipa_cmd_opcode opcode) { … } static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr, u32 len, bool last_tre, bool bei, enum ipa_cmd_opcode opcode) { … } /** * __gsi_trans_commit() - Common GSI transaction commit code * @trans: Transaction to commit * @ring_db: Whether to tell the hardware about these queued transfers * * Formats channel ring TRE entries based on the content of the scatterlist. * Maps a transaction pointer to the last ring entry used for the transaction, * so it can be recovered when it completes. Moves the transaction to * pending state. Finally, updates the channel ring pointer and optionally * rings the doorbell. */ static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db) { … } /* Commit a GSI transaction */ void gsi_trans_commit(struct gsi_trans *trans, bool ring_db) { … } /* Commit a GSI transaction and wait for it to complete */ void gsi_trans_commit_wait(struct gsi_trans *trans) { … } /* Process the completion of a transaction; called while polling */ void gsi_trans_complete(struct gsi_trans *trans) { … } /* Cancel a channel's pending transactions */ void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) { … } /* Issue a command to read a single byte from a channel */ int gsi_trans_read_byte(struct gsi *gsi, u32 channel_id, dma_addr_t addr) { … } /* Mark a gsi_trans_read_byte() request done */ void gsi_trans_read_byte_done(struct gsi *gsi, u32 channel_id) { … } /* Initialize a channel's GSI transaction info */ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) { … } /* Inverse of gsi_channel_trans_init() */ void gsi_channel_trans_exit(struct gsi_channel *channel) { … }
Codebase Browser
linux