// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) … #include <linux/atomic.h> #include <linux/cpu_pm.h> #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/ktime.h> #include <linux/list.h> #include <linux/module.h> #include <linux/notifier.h> #include <linux/of.h> #include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_domain.h> #include <linux/pm_runtime.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/wait.h> #include <clocksource/arm_arch_timer.h> #include <soc/qcom/cmd-db.h> #include <soc/qcom/tcs.h> #include <dt-bindings/soc/qcom,rpmh-rsc.h> #include "rpmh-internal.h" #define CREATE_TRACE_POINTS #include "trace-rpmh.h" #define RSC_DRV_ID … #define MAJOR_VER_MASK … #define MAJOR_VER_SHIFT … #define MINOR_VER_MASK … #define MINOR_VER_SHIFT … enum { … }; /* DRV HW Solver Configuration Information Register */ #define DRV_HW_SOLVER_MASK … #define DRV_HW_SOLVER_SHIFT … /* DRV TCS Configuration Information Register */ #define DRV_NUM_TCS_MASK … #define DRV_NUM_TCS_SHIFT … #define DRV_NCPT_MASK … #define DRV_NCPT_SHIFT … /* Offsets for CONTROL TCS Registers */ #define RSC_DRV_CTL_TCS_DATA_HI … #define RSC_DRV_CTL_TCS_DATA_HI_MASK … #define RSC_DRV_CTL_TCS_DATA_HI_VALID … #define RSC_DRV_CTL_TCS_DATA_LO … #define RSC_DRV_CTL_TCS_DATA_LO_MASK … #define RSC_DRV_CTL_TCS_DATA_SIZE … #define TCS_AMC_MODE_ENABLE … #define TCS_AMC_MODE_TRIGGER … /* TCS CMD register bit mask */ #define CMD_MSGID_LEN … #define CMD_MSGID_RESP_REQ … #define CMD_MSGID_WRITE … #define CMD_STATUS_ISSUED … #define CMD_STATUS_COMPL … /* * Here's a high level overview of how all the registers in RPMH work * together: * * - The main rpmh-rsc address is the base of a register space that can * be used to find overall configuration of the hardware * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register * space are all the TCS blocks. The offset of the TCS blocks is * specified in the device tree by "qcom,tcs-offset" and used to * compute tcs_base. * - TCS blocks come one after another. Type, count, and order are * specified by the device tree as "qcom,tcs-config". * - Each TCS block has some registers, then space for up to 16 commands. * Note that though address space is reserved for 16 commands, fewer * might be present. See ncpt (num cmds per TCS). * * Here's a picture: * * +---------------------------------------------------+ * |RSC | * | ctrl | * | | * | Drvs: | * | +-----------------------------------------------+ | * | |DRV0 | | * | | ctrl/config | | * | | IRQ | | * | | | | * | | TCSes: | | * | | +------------------------------------------+ | | * | | |TCS0 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | +------------------------------------------+ | | * | | |TCS1 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | +------------------------------------------+ | | * | | |TCS2 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | ...... | | * | +-----------------------------------------------+ | * | +-----------------------------------------------+ | * | |DRV1 | | * | | (same as DRV0) | | * | +-----------------------------------------------+ | * | ...... | * +---------------------------------------------------+ */ #define USECS_TO_CYCLES(time_usecs) … static inline unsigned long xloops_to_cycles(u64 xloops) { … } static u32 rpmh_rsc_reg_offset_ver_2_7[] = …; static u32 rpmh_rsc_reg_offset_ver_3_0[] = …; static inline void __iomem * tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) { … } static inline void __iomem * tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { … } static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { … } static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) { … } static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id, u32 data) { … } static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, u32 data) { … } static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, u32 data) { … } /** * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). * @drv: The RSC controller. * @type: SLEEP_TCS or WAKE_TCS * * This will clear the "slots" variable of the given tcs_group and also * tell the hardware to forget about all entries. * * The caller must ensure that no other RPMH actions are happening when this * function is called, since otherwise the device may immediately become * used again even before this function exits. */ static void tcs_invalidate(struct rsc_drv *drv, int type) { … } /** * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. * @drv: The RSC controller. * * The caller must ensure that no other RPMH actions are happening when this * function is called, since otherwise the device may immediately become * used again even before this function exits. */ void rpmh_rsc_invalidate(struct rsc_drv *drv) { … } /** * get_tcs_for_msg() - Get the tcs_group used to send the given message. * @drv: The RSC controller. * @msg: The message we want to send. * * This is normally pretty straightforward except if we are trying to send * an ACTIVE_ONLY message but don't have any active_only TCSes. * * Return: A pointer to a tcs_group or an ERR_PTR. */ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, const struct tcs_request *msg) { … } /** * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. * @drv: The RSC controller. * @tcs_id: The global ID of this TCS. * * For ACTIVE_ONLY transfers we want to call back into the client when the * transfer finishes. To do this we need the "request" that the client * originally provided us. This function grabs the request that we stashed * when we started the transfer. * * This only makes sense for ACTIVE_ONLY transfers since those are the only * ones we track sending (the only ones we enable interrupts for and the only * ones we call back to the client for). * * Return: The stashed request. */ static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, int tcs_id) { … } /** * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS * @drv: The controller. * @tcs_id: The global ID of this TCS. * @trigger: If true then untrigger/retrigger. If false then just untrigger. * * In the normal case we only ever call with "trigger=true" to start a * transfer. That will un-trigger/disable the TCS from the last transfer * then trigger/enable for this transfer. * * If we borrowed a wake TCS for an active-only transfer we'll also call * this function with "trigger=false" to just do the un-trigger/disable * before using the TCS for wake purposes again. * * Note that the AP is only in charge of triggering active-only transfers. * The AP never triggers sleep/wake values using this function. */ static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) { … } /** * enable_tcs_irq() - Enable or disable interrupts on the given TCS. * @drv: The controller. * @tcs_id: The global ID of this TCS. * @enable: If true then enable; if false then disable * * We only ever call this when we borrow a wake TCS for an active-only * transfer. For active-only TCSes interrupts are always left enabled. */ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) { … } /** * tcs_tx_done() - TX Done interrupt handler. * @irq: The IRQ number (ignored). * @p: Pointer to "struct rsc_drv". * * Called for ACTIVE_ONLY transfers (those are the only ones we enable the * IRQ for) when a transfer is done. * * Return: IRQ_HANDLED */ static irqreturn_t tcs_tx_done(int irq, void *p) { … } /** * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. * @drv: The controller. * @tcs_id: The global ID of this TCS. * @cmd_id: The index within the TCS to start writing. * @msg: The message we want to send, which will contain several addr/data * pairs to program (but few enough that they all fit in one TCS). * * This is used for all types of transfers (active, sleep, and wake). */ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, const struct tcs_request *msg) { … } /** * check_for_req_inflight() - Look to see if conflicting cmds are in flight. * @drv: The controller. * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. * @msg: The message we want to send, which will contain several addr/data * pairs to program (but few enough that they all fit in one TCS). * * This will walk through the TCSes in the group and check if any of them * appear to be sending to addresses referenced in the message. If it finds * one it'll return -EBUSY. * * Only for use for active-only transfers. * * Must be called with the drv->lock held since that protects tcs_in_use. * * Return: 0 if nothing in flight or -EBUSY if we should try again later. * The caller must re-enable interrupts between tries since that's * the only way tcs_in_use will ever be updated and the only way * RSC_DRV_CMD_ENABLE will ever be cleared. */ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, const struct tcs_request *msg) { … } /** * find_free_tcs() - Find free tcs in the given tcs_group; only for active. * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if * we borrowed it because there are zero active-only ones). * * Must be called with the drv->lock held since that protects tcs_in_use. * * Return: The first tcs that's free or -EBUSY if all in use. */ static int find_free_tcs(struct tcs_group *tcs) { … } /** * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. * @drv: The controller. * @tcs: The tcs_group used for ACTIVE_ONLY transfers. * @msg: The data to be sent. * * Claims a tcs in the given tcs_group while making sure that no existing cmd * is in flight that would conflict with the one in @msg. * * Context: Must be called with the drv->lock held since that protects * tcs_in_use. * * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight * or the tcs_group is full. */ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, const struct tcs_request *msg) { … } /** * rpmh_rsc_send_data() - Write / trigger active-only message. * @drv: The controller. * @msg: The data to be sent. * * NOTES: * - This is only used for "ACTIVE_ONLY" since the limitations of this * function don't make sense for sleep/wake cases. * - To do the transfer, we will grab a whole TCS for ourselves--we don't * try to share. If there are none available we'll wait indefinitely * for a free one. * - This function will not wait for the commands to be finished, only for * data to be programmed into the RPMh. See rpmh_tx_done() which will * be called when the transfer is fully complete. * - This function must be called with interrupts enabled. If the hardware * is busy doing someone else's transfer we need that transfer to fully * finish so that we can have the hardware, and to fully finish it needs * the interrupt handler to run. If the interrupts is set to run on the * active CPU this can never happen if interrupts are disabled. * * Return: 0 on success, -EINVAL on error. */ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) { … } /** * find_slots() - Find a place to write the given message. * @tcs: The tcs group to search. * @msg: The message we want to find room for. * @tcs_id: If we return 0 from the function, we return the global ID of the * TCS to write to here. * @cmd_id: If we return 0 from the function, we return the index of * the command array of the returned TCS where the client should * start writing the message. * * Only for use on sleep/wake TCSes since those are the only ones we maintain * tcs->slots for. * * Return: -ENOMEM if there was no room, else 0. */ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, int *tcs_id, int *cmd_id) { … } /** * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. * @drv: The controller. * @msg: The data to be written to the controller. * * This should only be called for sleep/wake state, never active-only * state. * * The caller must ensure that no other RPMH actions are happening and the * controller is idle when this function is called since it runs lockless. * * Return: 0 if no error; else -error. */ int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) { … } /** * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. * @drv: The controller * * Checks if any of the AMCs are busy in handling ACTIVE sets. * This is called from the last cpu powering down before flushing * SLEEP and WAKE sets. If AMCs are busy, controller can not enter * power collapse, so deny from the last cpu's pm notification. * * Context: Must be called with the drv->lock held. * * Return: * * False - AMCs are idle * * True - AMCs are busy */ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) { … } /** * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS. * @drv: The controller * * Writes maximum wakeup cycles when called from suspend. * Writes earliest hrtimer wakeup when called from idle. */ void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) { … } /** * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. * @nfb: Pointer to the notifier block in struct rsc_drv. * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. * @v: Unused * * This function is given to cpu_pm_register_notifier so we can be informed * about when CPUs go down. When all CPUs go down we know no more active * transfers will be started so we write sleep/wake sets. This function gets * called from cpuidle code paths and also at system suspend time. * * If its last CPU going down and AMCs are not busy then writes cached sleep * and wake messages to TCSes. The firmware then takes care of triggering * them when entering deepest low power modes. * * Return: See cpu_pm_register_notifier() */ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, unsigned long action, void *v) { … } /** * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy. * @nfb: Pointer to the genpd notifier block in struct rsc_drv. * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON. * @v: Unused * * This function is given to dev_pm_genpd_add_notifier() so we can be informed * about when cluster-pd is going down. When cluster go down we know no more active * transfers will be started so we write sleep/wake sets. This function gets * called from cpuidle code paths and also at system suspend time. * * If AMCs are not busy then writes cached sleep and wake messages to TCSes. * The firmware then takes care of triggering them when entering deepest low power modes. * * Return: * * NOTIFY_OK - success * * NOTIFY_BAD - failure */ static int rpmh_rsc_pd_callback(struct notifier_block *nfb, unsigned long action, void *v) { … } static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) { … } static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) { … } static int rpmh_rsc_probe(struct platform_device *pdev) { … } static const struct of_device_id rpmh_drv_match[] = …; MODULE_DEVICE_TABLE(of, rpmh_drv_match); static struct platform_driver rpmh_driver = …; static int __init rpmh_driver_init(void) { … } core_initcall(rpmh_driver_init); MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux