// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. * Copyright (C) 2018-2024 Linaro Ltd. */ #include <linux/bits.h> #include <linux/bug.h> #include <linux/completion.h> #include <linux/interrupt.h> #include <linux/mutex.h> #include <linux/netdevice.h> #include <linux/platform_device.h> #include <linux/types.h> #include "gsi.h" #include "gsi_private.h" #include "gsi_reg.h" #include "gsi_trans.h" #include "ipa_data.h" #include "ipa_gsi.h" #include "ipa_version.h" #include "reg.h" /** * DOC: The IPA Generic Software Interface * * The generic software interface (GSI) is an integral component of the IPA, * providing a well-defined communication layer between the AP subsystem * and the IPA core. The modem uses the GSI layer as well. * * -------- --------- * | | | | * | AP +<---. .----+ Modem | * | +--. | | .->+ | * | | | | | | | | * -------- | | | | --------- * v | v | * --+-+---+-+-- * | GSI | * |-----------| * | | * | IPA | * | | * ------------- * * In the above diagram, the AP and Modem represent "execution environments" * (EEs), which are independent operating environments that use the IPA for * data transfer. * * Each EE uses a set of unidirectional GSI "channels," which allow transfer * of data to or from the IPA. A channel is implemented as a ring buffer, * with a DRAM-resident array of "transfer elements" (TREs) available to * describe transfers to or from other EEs through the IPA. A transfer * element can also contain an immediate command, requesting the IPA perform * actions other than data transfer. * * Each TRE refers to a block of data--also located in DRAM. After writing * one or more TREs to a channel, the writer (either the IPA or an EE) writes * a doorbell register to inform the receiving side how many elements have * been written. * * Each channel has a GSI "event ring" associated with it. An event ring * is implemented very much like a channel ring, but is always directed from * the IPA to an EE. The IPA notifies an EE (such as the AP) about channel * events by adding an entry to the event ring associated with the channel. * The GSI then writes its doorbell for the event ring, causing the target * EE to be interrupted. Each entry in an event ring contains a pointer * to the channel TRE whose completion the event represents. * * Each TRE in a channel ring has a set of flags. One flag indicates whether * the completion of the transfer operation generates an entry (and possibly * an interrupt) in the channel's event ring. Other flags allow transfer * elements to be chained together, forming a single logical transaction. * TRE flags are used to control whether and when interrupts are generated * to signal completion of channel transfers. * * Elements in channel and event rings are completed (or consumed) strictly * in order. Completion of one entry implies the completion of all preceding * entries. A single completion interrupt can therefore communicate the * completion of many transfers. * * Note that all GSI registers are little-endian, which is the assumed * endianness of I/O space accesses. The accessor functions perform byte * swapping if needed (i.e., for a big endian CPU). */ /* Delay period for interrupt moderation (in 32KHz IPA internal timer ticks) */ #define GSI_EVT_RING_INT_MODT … #define GSI_CMD_TIMEOUT … #define GSI_CHANNEL_STOP_RETRIES … #define GSI_CHANNEL_MODEM_HALT_RETRIES … #define GSI_CHANNEL_MODEM_FLOW_RETRIES … #define GSI_MHI_EVENT_ID_START … #define GSI_MHI_EVENT_ID_END … #define GSI_ISR_MAX_ITER … /* An entry in an event ring */ struct gsi_event { … }; /** gsi_channel_scratch_gpi - GPI protocol scratch register * @max_outstanding_tre: * Defines the maximum number of TREs allowed in a single transaction * on a channel (in bytes). This determines the amount of prefetch * performed by the hardware. We configure this to equal the size of * the TLV FIFO for the channel. * @outstanding_threshold: * Defines the threshold (in bytes) determining when the sequencer * should update the channel doorbell. We configure this to equal * the size of two TREs. */ struct gsi_channel_scratch_gpi { … }; /** gsi_channel_scratch - channel scratch configuration area * * The exact interpretation of this register is protocol-specific. * We only use GPI channels; see struct gsi_channel_scratch_gpi, above. */ gsi_channel_scratch; /* Check things that can be validated at build time. */ static void gsi_validate_build(void) { … } /* Return the channel id associated with a given channel */ static u32 gsi_channel_id(struct gsi_channel *channel) { … } /* An initialized channel has a non-null GSI pointer */ static bool gsi_channel_initialized(struct gsi_channel *channel) { … } /* Encode the channel protocol for the CH_C_CNTXT_0 register */ static u32 ch_c_cntxt_0_type_encode(enum ipa_version version, const struct reg *reg, enum gsi_channel_type type) { … } /* Update the GSI IRQ type register with the cached value */ static void gsi_irq_type_update(struct gsi *gsi, u32 val) { … } static void gsi_irq_type_enable(struct gsi *gsi, enum gsi_irq_type_id type_id) { … } static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id) { … } /* Event ring commands are performed one at a time. Their completion * is signaled by the event ring control GSI interrupt type, which is * only enabled when we issue an event ring command. Only the event * ring being operated on has this interrupt enabled. */ static void gsi_irq_ev_ctrl_enable(struct gsi *gsi, u32 evt_ring_id) { … } /* Disable event ring control interrupts */ static void gsi_irq_ev_ctrl_disable(struct gsi *gsi) { … } /* Channel commands are performed one at a time. Their completion is * signaled by the channel control GSI interrupt type, which is only * enabled when we issue a channel command. Only the channel being * operated on has this interrupt enabled. */ static void gsi_irq_ch_ctrl_enable(struct gsi *gsi, u32 channel_id) { … } /* Disable channel control interrupts */ static void gsi_irq_ch_ctrl_disable(struct gsi *gsi) { … } static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id) { … } static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask) { … } static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id) { … } /* Enable all GSI_interrupt types */ static void gsi_irq_enable(struct gsi *gsi) { … } /* Disable all GSI interrupt types */ static void gsi_irq_disable(struct gsi *gsi) { … } /* Return the virtual address associated with a ring index */ void *gsi_ring_virt(struct gsi_ring *ring, u32 index) { … } /* Return the 32-bit DMA address associated with a ring index */ static u32 gsi_ring_addr(struct gsi_ring *ring, u32 index) { … } /* Return the ring index of a 32-bit ring offset */ static u32 gsi_ring_index(struct gsi_ring *ring, u32 offset) { … } /* Issue a GSI command by writing a value to a register, then wait for * completion to be signaled. Returns true if the command completes * or false if it times out. */ static bool gsi_command(struct gsi *gsi, u32 reg, u32 val) { … } /* Return the hardware's notion of the current state of an event ring */ static enum gsi_evt_ring_state gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id) { … } /* Issue an event ring command and wait for it to complete */ static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, enum gsi_evt_cmd_opcode opcode) { … } /* Allocate an event ring in NOT_ALLOCATED state */ static int gsi_evt_ring_alloc_command(struct gsi *gsi, u32 evt_ring_id) { … } /* Reset a GSI event ring in ALLOCATED or ERROR state. */ static void gsi_evt_ring_reset_command(struct gsi *gsi, u32 evt_ring_id) { … } /* Issue a hardware de-allocation request for an allocated event ring */ static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id) { … } /* Fetch the current state of a channel from hardware */ static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel) { … } /* Issue a channel command and wait for it to complete */ static void gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode) { … } /* Allocate GSI channel in NOT_ALLOCATED state */ static int gsi_channel_alloc_command(struct gsi *gsi, u32 channel_id) { … } /* Start an ALLOCATED channel */ static int gsi_channel_start_command(struct gsi_channel *channel) { … } /* Stop a GSI channel in STARTED state */ static int gsi_channel_stop_command(struct gsi_channel *channel) { … } /* Reset a GSI channel in ALLOCATED or ERROR state. */ static void gsi_channel_reset_command(struct gsi_channel *channel) { … } /* Deallocate an ALLOCATED GSI channel */ static void gsi_channel_de_alloc_command(struct gsi *gsi, u32 channel_id) { … } /* Ring an event ring doorbell, reporting the last entry processed by the AP. * The index argument (modulo the ring count) is the first unfilled entry, so * we supply one less than that with the doorbell. Update the event ring * index field with the value provided. */ static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) { … } /* Program an event ring for use */ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) { … } /* Find the transaction whose completion indicates a channel is quiesced */ static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel) { … } /* Wait for transaction activity on a channel to complete */ static void gsi_channel_trans_quiesce(struct gsi_channel *channel) { … } /* Program a channel for use; there is no gsi_channel_deprogram() */ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) { … } static int __gsi_channel_start(struct gsi_channel *channel, bool resume) { … } /* Start an allocated GSI channel */ int gsi_channel_start(struct gsi *gsi, u32 channel_id) { … } static int gsi_channel_stop_retry(struct gsi_channel *channel) { … } static int __gsi_channel_stop(struct gsi_channel *channel, bool suspend) { … } /* Stop a started channel */ int gsi_channel_stop(struct gsi *gsi, u32 channel_id) { … } /* Reset and reconfigure a channel, (possibly) enabling the doorbell engine */ void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool doorbell) { … } /* Stop a started channel for suspend */ int gsi_channel_suspend(struct gsi *gsi, u32 channel_id) { … } /* Resume a suspended channel (starting if stopped) */ int gsi_channel_resume(struct gsi *gsi, u32 channel_id) { … } /* Prevent all GSI interrupts while suspended */ void gsi_suspend(struct gsi *gsi) { … } /* Allow all GSI interrupts again when resuming */ void gsi_resume(struct gsi *gsi) { … } void gsi_trans_tx_committed(struct gsi_trans *trans) { … } void gsi_trans_tx_queued(struct gsi_trans *trans) { … } /** * gsi_trans_tx_completed() - Report completed TX transactions * @trans: TX channel transaction that has completed * * Report that a transaction on a TX channel has completed. At the time a * transaction is committed, we record *in the transaction* its channel's * committed transaction and byte counts. Transactions are completed in * order, and the difference between the channel's byte/transaction count * when the transaction was committed and when it completes tells us * exactly how much data has been transferred while the transaction was * pending. * * We report this information to the network stack, which uses it to manage * the rate at which data is sent to hardware. */ static void gsi_trans_tx_completed(struct gsi_trans *trans) { … } /* Channel control interrupt handler */ static void gsi_isr_chan_ctrl(struct gsi *gsi) { … } /* Event ring control interrupt handler */ static void gsi_isr_evt_ctrl(struct gsi *gsi) { … } /* Global channel error interrupt handler */ static void gsi_isr_glob_chan_err(struct gsi *gsi, u32 err_ee, u32 channel_id, u32 code) { … } /* Global event error interrupt handler */ static void gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code) { … } /* Global error interrupt handler */ static void gsi_isr_glob_err(struct gsi *gsi) { … } /* Generic EE interrupt handler */ static void gsi_isr_gp_int1(struct gsi *gsi) { … } /* Inter-EE interrupt handler */ static void gsi_isr_glob_ee(struct gsi *gsi) { … } /* I/O completion interrupt event */ static void gsi_isr_ieob(struct gsi *gsi) { … } /* General event interrupts represent serious problems, so report them */ static void gsi_isr_general(struct gsi *gsi) { … } /** * gsi_isr() - Top level GSI interrupt service routine * @irq: Interrupt number (ignored) * @dev_id: GSI pointer supplied to request_irq() * * This is the main handler function registered for the GSI IRQ. Each type * of interrupt has a separate handler function that is called from here. */ static irqreturn_t gsi_isr(int irq, void *dev_id) { … } /* Init function for GSI IRQ lookup; there is no gsi_irq_exit() */ static int gsi_irq_init(struct gsi *gsi, struct platform_device *pdev) { … } /* Return the transaction associated with a transfer completion event */ static struct gsi_trans * gsi_event_trans(struct gsi *gsi, struct gsi_event *event) { … } /** * gsi_evt_ring_update() - Update transaction state from hardware * @gsi: GSI pointer * @evt_ring_id: Event ring ID * @index: Event index in ring reported by hardware * * Events for RX channels contain the actual number of bytes received into * the buffer. Every event has a transaction associated with it, and here * we update transactions to record their actual received lengths. * * When an event for a TX channel arrives we use information in the * transaction to report the number of requests and bytes that have * been transferred. * * This function is called whenever we learn that the GSI hardware has filled * new events since the last time we checked. The ring's index field tells * the first entry in need of processing. The index provided is the * first *unfilled* event in the ring (following the last filled one). * * Events are sequential within the event ring, and transactions are * sequential within the transaction array. * * Note that @index always refers to an element *within* the event ring. */ static void gsi_evt_ring_update(struct gsi *gsi, u32 evt_ring_id, u32 index) { … } /* Initialize a ring, including allocating DMA memory for its entries */ static int gsi_ring_alloc(struct gsi *gsi, struct gsi_ring *ring, u32 count) { … } /* Free a previously-allocated ring */ static void gsi_ring_free(struct gsi *gsi, struct gsi_ring *ring) { … } /* Allocate an available event ring id */ static int gsi_evt_ring_id_alloc(struct gsi *gsi) { … } /* Free a previously-allocated event ring id */ static void gsi_evt_ring_id_free(struct gsi *gsi, u32 evt_ring_id) { … } /* Ring a channel doorbell, reporting the first un-filled entry */ void gsi_channel_doorbell(struct gsi_channel *channel) { … } /* Consult hardware, move newly completed transactions to completed state */ void gsi_channel_update(struct gsi_channel *channel) { … } /** * gsi_channel_poll_one() - Return a single completed transaction on a channel * @channel: Channel to be polled * * Return: Transaction pointer, or null if none are available * * This function returns the first of a channel's completed transactions. * If no transactions are in completed state, the hardware is consulted to * determine whether any new transactions have completed. If so, they're * moved to completed state and the first such transaction is returned. * If there are no more completed transactions, a null pointer is returned. */ static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel) { … } /** * gsi_channel_poll() - NAPI poll function for a channel * @napi: NAPI structure for the channel * @budget: Budget supplied by NAPI core * * Return: Number of items polled (<= budget) * * Single transactions completed by hardware are polled until either * the budget is exhausted, or there are no more. Each transaction * polled is passed to gsi_trans_complete(), to perform remaining * completion processing and retire/free the transaction. */ static int gsi_channel_poll(struct napi_struct *napi, int budget) { … } /* The event bitmap represents which event ids are available for allocation. * Set bits are not available, clear bits can be used. This function * initializes the map so all events supported by the hardware are available, * then precludes any reserved events from being allocated. */ static u32 gsi_event_bitmap_init(u32 evt_ring_max) { … } /* Setup function for a single channel */ static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id) { … } /* Inverse of gsi_channel_setup_one() */ static void gsi_channel_teardown_one(struct gsi *gsi, u32 channel_id) { … } /* We use generic commands only to operate on modem channels. We don't have * the ability to determine channel state for a modem channel, so we simply * issue the command and wait for it to complete. */ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, enum gsi_generic_cmd_opcode opcode, u8 params) { … } static int gsi_modem_channel_alloc(struct gsi *gsi, u32 channel_id) { … } static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) { … } /* Enable or disable flow control for a modem GSI TX channel (IPA v4.2+) */ void gsi_modem_channel_flow_control(struct gsi *gsi, u32 channel_id, bool enable) { … } /* Setup function for channels */ static int gsi_channel_setup(struct gsi *gsi) { … } /* Inverse of gsi_channel_setup() */ static void gsi_channel_teardown(struct gsi *gsi) { … } /* Turn off all GSI interrupts initially */ static int gsi_irq_setup(struct gsi *gsi) { … } static void gsi_irq_teardown(struct gsi *gsi) { … } /* Get # supported channel and event rings; there is no gsi_ring_teardown() */ static int gsi_ring_setup(struct gsi *gsi) { … } /* Setup function for GSI. GSI firmware must be loaded and initialized */ int gsi_setup(struct gsi *gsi) { … } /* Inverse of gsi_setup() */ void gsi_teardown(struct gsi *gsi) { … } /* Initialize a channel's event ring */ static int gsi_channel_evt_ring_init(struct gsi_channel *channel) { … } /* Inverse of gsi_channel_evt_ring_init() */ static void gsi_channel_evt_ring_exit(struct gsi_channel *channel) { … } static bool gsi_channel_data_valid(struct gsi *gsi, bool command, const struct ipa_gsi_endpoint_data *data) { … } /* Init function for a single channel */ static int gsi_channel_init_one(struct gsi *gsi, const struct ipa_gsi_endpoint_data *data, bool command) { … } /* Inverse of gsi_channel_init_one() */ static void gsi_channel_exit_one(struct gsi_channel *channel) { … } /* Init function for channels */ static int gsi_channel_init(struct gsi *gsi, u32 count, const struct ipa_gsi_endpoint_data *data) { … } /* Inverse of gsi_channel_init() */ static void gsi_channel_exit(struct gsi *gsi) { … } /* Init function for GSI. GSI hardware does not need to be "ready" */ int gsi_init(struct gsi *gsi, struct platform_device *pdev, enum ipa_version version, u32 count, const struct ipa_gsi_endpoint_data *data) { … } /* Inverse of gsi_init() */ void gsi_exit(struct gsi *gsi) { … } /* The maximum number of outstanding TREs on a channel. This limits * a channel's maximum number of transactions outstanding (worst case * is one TRE per transaction). * * The absolute limit is the number of TREs in the channel's TRE ring, * and in theory we should be able use all of them. But in practice, * doing that led to the hardware reporting exhaustion of event ring * slots for writing completion information. So the hardware limit * would be (tre_count - 1). * * We reduce it a bit further though. Transaction resource pools are * sized to be a little larger than this maximum, to allow resource * allocations to always be contiguous. The number of entries in a * TRE ring buffer is a power of 2, and the extra resources in a pool * tends to nearly double the memory allocated for it. Reducing the * maximum number of outstanding TREs allows the number of entries in * a pool to avoid crossing that power-of-2 boundary, and this can * substantially reduce pool memory requirements. The number we * reduce it by matches the number added in gsi_trans_pool_init(). */ u32 gsi_channel_tre_max(struct gsi *gsi, u32 channel_id) { … }
Codebase Browser
linux