// SPDX-License-Identifier: GPL-2.0+ /* * SSH packet transport layer. * * Copyright (C) 2019-2022 Maximilian Luz <[email protected]> */ #include <asm/unaligned.h> #include <linux/atomic.h> #include <linux/error-injection.h> #include <linux/jiffies.h> #include <linux/kfifo.h> #include <linux/kref.h> #include <linux/kthread.h> #include <linux/ktime.h> #include <linux/limits.h> #include <linux/list.h> #include <linux/lockdep.h> #include <linux/serdev.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/workqueue.h> #include <linux/surface_aggregator/serial_hub.h> #include "ssh_msgb.h" #include "ssh_packet_layer.h" #include "ssh_parser.h" #include "trace.h" /* * To simplify reasoning about the code below, we define a few concepts. The * system below is similar to a state-machine for packets, however, there are * too many states to explicitly write them down. To (somewhat) manage the * states and packets we rely on flags, reference counting, and some simple * concepts. State transitions are triggered by actions. * * >> Actions << * * - submit * - transmission start (process next item in queue) * - transmission finished (guaranteed to never be parallel to transmission * start) * - ACK received * - NAK received (this is equivalent to issuing re-submit for all pending * packets) * - timeout (this is equivalent to re-issuing a submit or canceling) * - cancel (non-pending and pending) * * >> Data Structures, Packet Ownership, General Overview << * * The code below employs two main data structures: The packet queue, * containing all packets scheduled for transmission, and the set of pending * packets, containing all packets awaiting an ACK. * * Shared ownership of a packet is controlled via reference counting. Inside * the transport system are a total of five packet owners: * * - the packet queue, * - the pending set, * - the transmitter thread, * - the receiver thread (via ACKing), and * - the timeout work item. * * Normal operation is as follows: The initial reference of the packet is * obtained by submitting the packet and queuing it. The receiver thread takes * packets from the queue. By doing this, it does not increment the refcount * but takes over the reference (removing it from the queue). If the packet is * sequenced (i.e. needs to be ACKed by the client), the transmitter thread * sets-up the timeout and adds the packet to the pending set before starting * to transmit it. As the timeout is handled by a reaper task, no additional * reference for it is needed. After the transmit is done, the reference held * by the transmitter thread is dropped. If the packet is unsequenced (i.e. * does not need an ACK), the packet is completed by the transmitter thread * before dropping that reference. * * On receival of an ACK, the receiver thread removes and obtains the * reference to the packet from the pending set. The receiver thread will then * complete the packet and drop its reference. * * On receival of a NAK, the receiver thread re-submits all currently pending * packets. * * Packet timeouts are detected by the timeout reaper. This is a task, * scheduled depending on the earliest packet timeout expiration date, * checking all currently pending packets if their timeout has expired. If the * timeout of a packet has expired, it is re-submitted and the number of tries * of this packet is incremented. If this number reaches its limit, the packet * will be completed with a failure. * * On transmission failure (such as repeated packet timeouts), the completion * callback is immediately run by on thread on which the error was detected. * * To ensure that a packet eventually leaves the system it is marked as * "locked" directly before it is going to be completed or when it is * canceled. Marking a packet as "locked" has the effect that passing and * creating new references of the packet is disallowed. This means that the * packet cannot be added to the queue, the pending set, and the timeout, or * be picked up by the transmitter thread or receiver thread. To remove a * packet from the system it has to be marked as locked and subsequently all * references from the data structures (queue, pending) have to be removed. * References held by threads will eventually be dropped automatically as * their execution progresses. * * Note that the packet completion callback is, in case of success and for a * sequenced packet, guaranteed to run on the receiver thread, thus providing * a way to reliably identify responses to the packet. The packet completion * callback is only run once and it does not indicate that the packet has * fully left the system (for this, one should rely on the release method, * triggered when the reference count of the packet reaches zero). In case of * re-submission (and with somewhat unlikely timing), it may be possible that * the packet is being re-transmitted while the completion callback runs. * Completion will occur both on success and internal error, as well as when * the packet is canceled. * * >> Flags << * * Flags are used to indicate the state and progression of a packet. Some flags * have stricter guarantees than other: * * - locked * Indicates if the packet is locked. If the packet is locked, passing and/or * creating additional references to the packet is forbidden. The packet thus * may not be queued, dequeued, or removed or added to the pending set. Note * that the packet state flags may still change (e.g. it may be marked as * ACKed, transmitted, ...). * * - completed * Indicates if the packet completion callback has been executed or is about * to be executed. This flag is used to ensure that the packet completion * callback is only run once. * * - queued * Indicates if a packet is present in the submission queue or not. This flag * must only be modified with the queue lock held, and must be coherent to the * presence of the packet in the queue. * * - pending * Indicates if a packet is present in the set of pending packets or not. * This flag must only be modified with the pending lock held, and must be * coherent to the presence of the packet in the pending set. * * - transmitting * Indicates if the packet is currently transmitting. In case of * re-transmissions, it is only safe to wait on the "transmitted" completion * after this flag has been set. The completion will be set both in success * and error case. * * - transmitted * Indicates if the packet has been transmitted. This flag is not cleared by * the system, thus it indicates the first transmission only. * * - acked * Indicates if the packet has been acknowledged by the client. There are no * other guarantees given. For example, the packet may still be canceled * and/or the completion may be triggered an error even though this bit is * set. Rely on the status provided to the completion callback instead. * * - canceled * Indicates if the packet has been canceled from the outside. There are no * other guarantees given. Specifically, the packet may be completed by * another part of the system before the cancellation attempts to complete it. * * >> General Notes << * * - To avoid deadlocks, if both queue and pending locks are required, the * pending lock must be acquired before the queue lock. * * - The packet priority must be accessed only while holding the queue lock. * * - The packet timestamp must be accessed only while holding the pending * lock. */ /* * SSH_PTL_MAX_PACKET_TRIES - Maximum transmission attempts for packet. * * Maximum number of transmission attempts per sequenced packet in case of * time-outs. Must be smaller than 16. If the packet times out after this * amount of tries, the packet will be completed with %-ETIMEDOUT as status * code. */ #define SSH_PTL_MAX_PACKET_TRIES … /* * SSH_PTL_TX_TIMEOUT - Packet transmission timeout. * * Timeout in jiffies for packet transmission via the underlying serial * device. If transmitting the packet takes longer than this timeout, the * packet will be completed with -ETIMEDOUT. It will not be re-submitted. */ #define SSH_PTL_TX_TIMEOUT … /* * SSH_PTL_PACKET_TIMEOUT - Packet response timeout. * * Timeout as ktime_t delta for ACKs. If we have not received an ACK in this * time-frame after starting transmission, the packet will be re-submitted. */ #define SSH_PTL_PACKET_TIMEOUT … /* * SSH_PTL_PACKET_TIMEOUT_RESOLUTION - Packet timeout granularity. * * Time-resolution for timeouts. Should be larger than one jiffy to avoid * direct re-scheduling of reaper work_struct. */ #define SSH_PTL_PACKET_TIMEOUT_RESOLUTION … /* * SSH_PTL_MAX_PENDING - Maximum number of pending packets. * * Maximum number of sequenced packets concurrently waiting for an ACK. * Packets marked as blocking will not be transmitted while this limit is * reached. */ #define SSH_PTL_MAX_PENDING … /* * SSH_PTL_RX_BUF_LEN - Evaluation-buffer size in bytes. */ #define SSH_PTL_RX_BUF_LEN … /* * SSH_PTL_RX_FIFO_LEN - Fifo input-buffer size in bytes. */ #define SSH_PTL_RX_FIFO_LEN … #ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION /** * ssh_ptl_should_drop_ack_packet() - Error injection hook to drop ACK packets. * * Useful to test detection and handling of automated re-transmits by the EC. * Specifically of packets that the EC considers not-ACKed but the driver * already considers ACKed (due to dropped ACK). In this case, the EC * re-transmits the packet-to-be-ACKed and the driver should detect it as * duplicate/already handled. Note that the driver should still send an ACK * for the re-transmitted packet. */ static noinline bool ssh_ptl_should_drop_ack_packet(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_drop_nak_packet() - Error injection hook to drop NAK packets. * * Useful to test/force automated (timeout-based) re-transmit by the EC. * Specifically, packets that have not reached the driver completely/with valid * checksums. Only useful in combination with receival of (injected) bad data. */ static noinline bool ssh_ptl_should_drop_nak_packet(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_drop_dsq_packet() - Error injection hook to drop sequenced * data packet. * * Useful to test re-transmit timeout of the driver. If the data packet has not * been ACKed after a certain time, the driver should re-transmit the packet up * to limited number of times defined in SSH_PTL_MAX_PACKET_TRIES. */ static noinline bool ssh_ptl_should_drop_dsq_packet(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_fail_write() - Error injection hook to make * serdev_device_write() fail. * * Hook to simulate errors in serdev_device_write when transmitting packets. */ static noinline int ssh_ptl_should_fail_write(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_corrupt_tx_data() - Error injection hook to simulate invalid * data being sent to the EC. * * Hook to simulate corrupt/invalid data being sent from host (driver) to EC. * Causes the packet data to be actively corrupted by overwriting it with * pre-defined values, such that it becomes invalid, causing the EC to respond * with a NAK packet. Useful to test handling of NAK packets received by the * driver. */ static noinline bool ssh_ptl_should_corrupt_tx_data(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_corrupt_rx_syn() - Error injection hook to simulate invalid * data being sent by the EC. * * Hook to simulate invalid SYN bytes, i.e. an invalid start of messages and * test handling thereof in the driver. */ static noinline bool ssh_ptl_should_corrupt_rx_syn(void) { … } ALLOW_ERROR_INJECTION(…); /** * ssh_ptl_should_corrupt_rx_data() - Error injection hook to simulate invalid * data being sent by the EC. * * Hook to simulate invalid data/checksum of the message frame and test handling * thereof in the driver. */ static noinline bool ssh_ptl_should_corrupt_rx_data(void) { … } ALLOW_ERROR_INJECTION(…); static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet) { … } static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet) { … } static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet) { … } static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) { … } static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet, const unsigned char *buf, size_t count) { … } static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) { … } static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl, struct ssam_span *data) { … } static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl, struct ssam_span *frame) { … } #else /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */ static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) { return false; } static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet, const unsigned char *buf, size_t count) { return serdev_device_write_buf(ptl->serdev, buf, count); } static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) { } static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl, struct ssam_span *data) { } static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl, struct ssam_span *frame) { } #endif /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */ static void __ssh_ptl_packet_release(struct kref *kref) { … } /** * ssh_packet_get() - Increment reference count of packet. * @packet: The packet to increment the reference count of. * * Increments the reference count of the given packet. See ssh_packet_put() * for the counter-part of this function. * * Return: Returns the packet provided as input. */ struct ssh_packet *ssh_packet_get(struct ssh_packet *packet) { … } EXPORT_SYMBOL_GPL(…); /** * ssh_packet_put() - Decrement reference count of packet. * @packet: The packet to decrement the reference count of. * * If the reference count reaches zero, the ``release`` callback specified in * the packet's &struct ssh_packet_ops, i.e. ``packet->ops->release``, will be * called. * * See ssh_packet_get() for the counter-part of this function. */ void ssh_packet_put(struct ssh_packet *packet) { … } EXPORT_SYMBOL_GPL(…); static u8 ssh_packet_get_seq(struct ssh_packet *packet) { … } /** * ssh_packet_init() - Initialize SSH packet. * @packet: The packet to initialize. * @type: Type-flags of the packet. * @priority: Priority of the packet. See SSH_PACKET_PRIORITY() for details. * @ops: Packet operations. * * Initializes the given SSH packet. Sets the transmission buffer pointer to * %NULL and the transmission buffer length to zero. For data-type packets, * this buffer has to be set separately via ssh_packet_set_data() before * submission, and must contain a valid SSH message, i.e. frame with optional * payload of any type. */ void ssh_packet_init(struct ssh_packet *packet, unsigned long type, u8 priority, const struct ssh_packet_ops *ops) { … } static struct kmem_cache *ssh_ctrl_packet_cache; /** * ssh_ctrl_packet_cache_init() - Initialize the control packet cache. */ int ssh_ctrl_packet_cache_init(void) { … } /** * ssh_ctrl_packet_cache_destroy() - Deinitialize the control packet cache. */ void ssh_ctrl_packet_cache_destroy(void) { … } /** * ssh_ctrl_packet_alloc() - Allocate packet from control packet cache. * @packet: Where the pointer to the newly allocated packet should be stored. * @buffer: The buffer corresponding to this packet. * @flags: Flags used for allocation. * * Allocates a packet and corresponding transport buffer from the control * packet cache. Sets the packet's buffer reference to the allocated buffer. * The packet must be freed via ssh_ctrl_packet_free(), which will also free * the corresponding buffer. The corresponding buffer must not be freed * separately. Intended to be used with %ssh_ptl_ctrl_packet_ops as packet * operations. * * Return: Returns zero on success, %-ENOMEM if the allocation failed. */ static int ssh_ctrl_packet_alloc(struct ssh_packet **packet, struct ssam_span *buffer, gfp_t flags) { … } /** * ssh_ctrl_packet_free() - Free packet allocated from control packet cache. * @p: The packet to free. */ static void ssh_ctrl_packet_free(struct ssh_packet *p) { … } static const struct ssh_packet_ops ssh_ptl_ctrl_packet_ops = …; static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now, ktime_t expires) { … } /* Must be called with queue lock held. */ static void ssh_packet_next_try(struct ssh_packet *p) { … } /* Must be called with queue lock held. */ static struct list_head *__ssh_ptl_queue_find_entrypoint(struct ssh_packet *p) { … } /* Must be called with queue lock held. */ static int __ssh_ptl_queue_push(struct ssh_packet *packet) { … } static int ssh_ptl_queue_push(struct ssh_packet *packet) { … } static void ssh_ptl_queue_remove(struct ssh_packet *packet) { … } static void ssh_ptl_pending_push(struct ssh_packet *p) { … } static void ssh_ptl_pending_remove(struct ssh_packet *packet) { … } /* Warning: Does not check/set "completed" bit. */ static void __ssh_ptl_complete(struct ssh_packet *p, int status) { … } static void ssh_ptl_remove_and_complete(struct ssh_packet *p, int status) { … } static bool ssh_ptl_tx_can_process(struct ssh_packet *packet) { … } static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl) { … } static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl) { … } static void ssh_ptl_tx_compl_success(struct ssh_packet *packet) { … } static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status) { … } static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl) { … } static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout) { … } static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet) { … } static int ssh_ptl_tx_threadfn(void *data) { … } /** * ssh_ptl_tx_wakeup_packet() - Wake up packet transmitter thread for new * packet. * @ptl: The packet transport layer. * * Wakes up the packet transmitter thread, notifying it that a new packet has * arrived and is ready for transfer. If the packet transport layer has been * shut down, calls to this function will be ignored. */ static void ssh_ptl_tx_wakeup_packet(struct ssh_ptl *ptl) { … } /** * ssh_ptl_tx_start() - Start packet transmitter thread. * @ptl: The packet transport layer. * * Return: Returns zero on success, a negative error code on failure. */ int ssh_ptl_tx_start(struct ssh_ptl *ptl) { … } /** * ssh_ptl_tx_stop() - Stop packet transmitter thread. * @ptl: The packet transport layer. * * Return: Returns zero on success, a negative error code on failure. */ int ssh_ptl_tx_stop(struct ssh_ptl *ptl) { … } static struct ssh_packet *ssh_ptl_ack_pop(struct ssh_ptl *ptl, u8 seq_id) { … } static void ssh_ptl_wait_until_transmitted(struct ssh_packet *packet) { … } static void ssh_ptl_acknowledge(struct ssh_ptl *ptl, u8 seq) { … } /** * ssh_ptl_submit() - Submit a packet to the transport layer. * @ptl: The packet transport layer to submit the packet to. * @p: The packet to submit. * * Submits a new packet to the transport layer, queuing it to be sent. This * function should not be used for re-submission. * * Return: Returns zero on success, %-EINVAL if a packet field is invalid or * the packet has been canceled prior to submission, %-EALREADY if the packet * has already been submitted, or %-ESHUTDOWN if the packet transport layer * has been shut down. */ int ssh_ptl_submit(struct ssh_ptl *ptl, struct ssh_packet *p) { … } /* * __ssh_ptl_resubmit() - Re-submit a packet to the transport layer. * @packet: The packet to re-submit. * * Re-submits the given packet: Checks if it can be re-submitted and queues it * if it can, resetting the packet timestamp in the process. Must be called * with the pending lock held. * * Return: Returns %-ECANCELED if the packet has exceeded its number of tries, * %-EINVAL if the packet has been locked, %-EALREADY if the packet is already * on the queue, and %-ESHUTDOWN if the transmission layer has been shut down. */ static int __ssh_ptl_resubmit(struct ssh_packet *packet) { … } static void ssh_ptl_resubmit_pending(struct ssh_ptl *ptl) { … } /** * ssh_ptl_cancel() - Cancel a packet. * @p: The packet to cancel. * * Cancels a packet. There are no guarantees on when completion and release * callbacks will be called. This may occur during execution of this function * or may occur at any point later. * * Note that it is not guaranteed that the packet will actually be canceled if * the packet is concurrently completed by another process. The only guarantee * of this function is that the packet will be completed (with success, * failure, or cancellation) and released from the transport layer in a * reasonable time-frame. * * May be called before the packet has been submitted, in which case any later * packet submission fails. */ void ssh_ptl_cancel(struct ssh_packet *p) { … } /* Must be called with pending lock held */ static ktime_t ssh_packet_get_expiration(struct ssh_packet *p, ktime_t timeout) { … } static void ssh_ptl_timeout_reap(struct work_struct *work) { … } static bool ssh_ptl_rx_retransmit_check(struct ssh_ptl *ptl, const struct ssh_frame *frame) { … } static void ssh_ptl_rx_dataframe(struct ssh_ptl *ptl, const struct ssh_frame *frame, const struct ssam_span *payload) { … } static void ssh_ptl_send_ack(struct ssh_ptl *ptl, u8 seq) { … } static void ssh_ptl_send_nak(struct ssh_ptl *ptl) { … } static size_t ssh_ptl_rx_eval(struct ssh_ptl *ptl, struct ssam_span *source) { … } static int ssh_ptl_rx_threadfn(void *data) { … } static void ssh_ptl_rx_wakeup(struct ssh_ptl *ptl) { … } /** * ssh_ptl_rx_start() - Start packet transport layer receiver thread. * @ptl: The packet transport layer. * * Return: Returns zero on success, a negative error code on failure. */ int ssh_ptl_rx_start(struct ssh_ptl *ptl) { … } /** * ssh_ptl_rx_stop() - Stop packet transport layer receiver thread. * @ptl: The packet transport layer. * * Return: Returns zero on success, a negative error code on failure. */ int ssh_ptl_rx_stop(struct ssh_ptl *ptl) { … } /** * ssh_ptl_rx_rcvbuf() - Push data from lower-layer transport to the packet * layer. * @ptl: The packet transport layer. * @buf: Pointer to the data to push to the layer. * @n: Size of the data to push to the layer, in bytes. * * Pushes data from a lower-layer transport to the receiver fifo buffer of the * packet layer and notifies the receiver thread. Calls to this function are * ignored once the packet layer has been shut down. * * Return: Returns the number of bytes transferred (positive or zero) on * success. Returns %-ESHUTDOWN if the packet layer has been shut down. */ ssize_t ssh_ptl_rx_rcvbuf(struct ssh_ptl *ptl, const u8 *buf, size_t n) { … } /** * ssh_ptl_shutdown() - Shut down the packet transport layer. * @ptl: The packet transport layer. * * Shuts down the packet transport layer, removing and canceling all queued * and pending packets. Packets canceled by this operation will be completed * with %-ESHUTDOWN as status. Receiver and transmitter threads will be * stopped. * * As a result of this function, the transport layer will be marked as shut * down. Submission of packets after the transport layer has been shut down * will fail with %-ESHUTDOWN. */ void ssh_ptl_shutdown(struct ssh_ptl *ptl) { … } /** * ssh_ptl_init() - Initialize packet transport layer. * @ptl: The packet transport layer to initialize. * @serdev: The underlying serial device, i.e. the lower-level transport. * @ops: Packet layer operations. * * Initializes the given packet transport layer. Transmitter and receiver * threads must be started separately via ssh_ptl_tx_start() and * ssh_ptl_rx_start(), after the packet-layer has been initialized and the * lower-level transport layer has been set up. * * Return: Returns zero on success and a nonzero error code on failure. */ int ssh_ptl_init(struct ssh_ptl *ptl, struct serdev_device *serdev, struct ssh_ptl_ops *ops) { … } /** * ssh_ptl_destroy() - Deinitialize packet transport layer. * @ptl: The packet transport layer to deinitialize. * * Deinitializes the given packet transport layer and frees resources * associated with it. If receiver and/or transmitter threads have been * started, the layer must first be shut down via ssh_ptl_shutdown() before * this function can be called. */ void ssh_ptl_destroy(struct ssh_ptl *ptl) { … }
Codebase Browser
linux