// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ #include <linux/types.h> #include <linux/completion.h> #include <linux/mutex.h> #include <linux/bitops.h> #include <linux/kthread.h> #include <linux/wait.h> #include <linux/delay.h> #include <linux/slab.h> #include <linux/kref.h> #include <linux/rcupdate.h> #include <linux/sched/signal.h> #include "vchiq_arm.h" #include "vchiq_core.h" #define VCHIQ_SLOT_HANDLER_STACK … #define VCHIQ_MSG_PADDING … #define VCHIQ_MSG_CONNECT … #define VCHIQ_MSG_OPEN … #define VCHIQ_MSG_OPENACK … #define VCHIQ_MSG_CLOSE … #define VCHIQ_MSG_DATA … #define VCHIQ_MSG_BULK_RX … #define VCHIQ_MSG_BULK_TX … #define VCHIQ_MSG_BULK_RX_DONE … #define VCHIQ_MSG_BULK_TX_DONE … #define VCHIQ_MSG_PAUSE … #define VCHIQ_MSG_RESUME … #define VCHIQ_MSG_REMOTE_USE … #define VCHIQ_MSG_REMOTE_RELEASE … #define VCHIQ_MSG_REMOTE_USE_ACTIVE … #define TYPE_SHIFT … #define VCHIQ_PORT_MAX … #define VCHIQ_PORT_FREE … #define VCHIQ_PORT_IS_VALID(port) … #define VCHIQ_MAKE_MSG(type, srcport, dstport) … #define VCHIQ_MSG_TYPE(msgid) … #define VCHIQ_MSG_SRCPORT(msgid) … #define VCHIQ_MSG_DSTPORT(msgid) … #define MAKE_CONNECT … #define MAKE_OPEN(srcport) … #define MAKE_OPENACK(srcport, dstport) … #define MAKE_CLOSE(srcport, dstport) … #define MAKE_DATA(srcport, dstport) … #define MAKE_PAUSE … #define MAKE_RESUME … #define MAKE_REMOTE_USE … #define MAKE_REMOTE_USE_ACTIVE … /* Ensure the fields are wide enough */ static_assert(…); static_assert(…); static_assert(…); #define VCHIQ_MSGID_PADDING … #define VCHIQ_MSGID_CLAIMED … #define VCHIQ_FOURCC_INVALID … #define VCHIQ_FOURCC_IS_LEGAL(fourcc) … #define VCHIQ_BULK_ACTUAL_ABORTED … #if VCHIQ_ENABLE_STATS #define VCHIQ_STATS_INC(state, stat) … #define VCHIQ_SERVICE_STATS_INC(service, stat) … #define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) … #else #define VCHIQ_STATS_INC … #define VCHIQ_SERVICE_STATS_INC … #define VCHIQ_SERVICE_STATS_ADD … #endif #define HANDLE_STATE_SHIFT … #define SLOT_INFO_FROM_INDEX(state, index) … #define SLOT_DATA_FROM_INDEX(state, index) … #define SLOT_INDEX_FROM_DATA(state, data) … #define SLOT_INDEX_FROM_INFO(state, info) … #define SLOT_QUEUE_INDEX_FROM_POS(pos) … #define SLOT_QUEUE_INDEX_FROM_POS_MASKED(pos) … #define BULK_INDEX(x) … #define NO_CLOSE_RECVD … #define CLOSE_RECVD … #define NO_RETRY_POLL … #define RETRY_POLL … struct vchiq_open_payload { … }; struct vchiq_openack_payload { … }; enum { … }; enum { … }; /* we require this for consistency between endpoints */ static_assert(…); static_assert(…); static inline void check_sizes(void) { … } static unsigned int handle_seq; static const char *const srvstate_names[] = …; static const char *const reason_names[] = …; static const char *const conn_state_names[] = …; static void release_message_sync(struct vchiq_state *state, struct vchiq_header *header); static const char *msg_type_str(unsigned int msg_type) { … } static inline void set_service_state(struct vchiq_service *service, int newstate) { … } struct vchiq_service *handle_to_service(struct vchiq_instance *instance, unsigned int handle) { … } struct vchiq_service * find_service_by_handle(struct vchiq_instance *instance, unsigned int handle) { … } struct vchiq_service * find_service_by_port(struct vchiq_state *state, unsigned int localport) { … } struct vchiq_service * find_service_for_instance(struct vchiq_instance *instance, unsigned int handle) { … } struct vchiq_service * find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle) { … } struct vchiq_service * __next_service_by_instance(struct vchiq_state *state, struct vchiq_instance *instance, int *pidx) { … } struct vchiq_service * next_service_by_instance(struct vchiq_state *state, struct vchiq_instance *instance, int *pidx) { … } void vchiq_service_get(struct vchiq_service *service) { … } static void service_release(struct kref *kref) { … } void vchiq_service_put(struct vchiq_service *service) { … } int vchiq_get_client_id(struct vchiq_instance *instance, unsigned int handle) { … } void * vchiq_get_service_userdata(struct vchiq_instance *instance, unsigned int handle) { … } EXPORT_SYMBOL(…); static void mark_service_closing_internal(struct vchiq_service *service, int sh_thread) { … } static void mark_service_closing(struct vchiq_service *service) { … } static inline int make_service_callback(struct vchiq_service *service, enum vchiq_reason reason, struct vchiq_header *header, void *bulk_userdata) { … } inline void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate) { … } /* This initialises a single remote_event, and the associated wait_queue. */ static inline void remote_event_create(wait_queue_head_t *wq, struct remote_event *event) { … } /* * All the event waiting routines in VCHIQ used a custom semaphore * implementation that filtered most signals. This achieved a behaviour similar * to the "killable" family of functions. While cleaning up this code all the * routines where switched to the "interruptible" family of functions, as the * former was deemed unjustified and the use "killable" set all VCHIQ's * threads in D state. * * Returns: 0 on success, a negative error code on failure */ static inline int remote_event_wait(wait_queue_head_t *wq, struct remote_event *event) { … } /* * Acknowledge that the event has been signalled, and wake any waiters. Usually * called as a result of the doorbell being rung. */ static inline void remote_event_signal_local(wait_queue_head_t *wq, struct remote_event *event) { … } /* Check if a single event has been signalled, waking the waiters if it has. */ static inline void remote_event_poll(wait_queue_head_t *wq, struct remote_event *event) { … } /* * VCHIQ used a small, fixed number of remote events. It is simplest to * enumerate them here for polling. */ void remote_event_pollall(struct vchiq_state *state) { … } /* * Round up message sizes so that any space at the end of a slot is always big * enough for a header. This relies on header size being a power of two, which * has been verified earlier by a static assertion. */ static inline size_t calc_stride(size_t size) { … } /* Called by the slot handler thread */ static struct vchiq_service * get_listening_service(struct vchiq_state *state, int fourcc) { … } /* Called by the slot handler thread */ static struct vchiq_service * get_connected_service(struct vchiq_state *state, unsigned int port) { … } inline void request_poll(struct vchiq_state *state, struct vchiq_service *service, int poll_type) { … } /* * Called from queue_message, by the slot handler and application threads, * with slot_mutex held */ static struct vchiq_header * reserve_space(struct vchiq_state *state, size_t space, int is_blocking) { … } static void process_free_data_message(struct vchiq_state *state, u32 *service_found, struct vchiq_header *header) { … } /* Called by the recycle thread. */ static void process_free_queue(struct vchiq_state *state, u32 *service_found, size_t length) { … } static ssize_t memcpy_copy_callback(void *context, void *dest, size_t offset, size_t maxsize) { … } static ssize_t copy_message_data(ssize_t (*copy_callback)(void *context, void *dest, size_t offset, size_t maxsize), void *context, void *dest, size_t size) { … } /* Called by the slot handler and application threads */ static int queue_message(struct vchiq_state *state, struct vchiq_service *service, int msgid, ssize_t (*copy_callback)(void *context, void *dest, size_t offset, size_t maxsize), void *context, size_t size, int flags) { … } /* Called by the slot handler and application threads */ static int queue_message_sync(struct vchiq_state *state, struct vchiq_service *service, int msgid, ssize_t (*copy_callback)(void *context, void *dest, size_t offset, size_t maxsize), void *context, int size) { … } static inline void claim_slot(struct vchiq_slot_info *slot) { … } static void release_slot(struct vchiq_state *state, struct vchiq_slot_info *slot_info, struct vchiq_header *header, struct vchiq_service *service) { … } static inline enum vchiq_reason get_bulk_reason(struct vchiq_bulk *bulk) { … } /* Called by the slot handler - don't hold the bulk mutex */ static int notify_bulks(struct vchiq_service *service, struct vchiq_bulk_queue *queue, int retry_poll) { … } static void poll_services_of_group(struct vchiq_state *state, int group) { … } /* Called by the slot handler thread */ static void poll_services(struct vchiq_state *state) { … } /* Called with the bulk_mutex held */ static void abort_outstanding_bulks(struct vchiq_service *service, struct vchiq_bulk_queue *queue) { … } static int parse_open(struct vchiq_state *state, struct vchiq_header *header) { … } /** * parse_message() - parses a single message from the rx slot * @state: vchiq state struct * @header: message header * * Context: Process context * * Return: * * >= 0 - size of the parsed message payload (without header) * * -EINVAL - fatal error occurred, bail out is required */ static int parse_message(struct vchiq_state *state, struct vchiq_header *header) { … } /* Called by the slot handler thread */ static void parse_rx_slots(struct vchiq_state *state) { … } /** * handle_poll() - handle service polling and other rare conditions * @state: vchiq state struct * * Context: Process context * * Return: * * 0 - poll handled successful * * -EAGAIN - retry later */ static int handle_poll(struct vchiq_state *state) { … } /* Called by the slot handler thread */ static int slot_handler_func(void *v) { … } /* Called by the recycle thread */ static int recycle_func(void *v) { … } /* Called by the sync thread */ static int sync_func(void *v) { … } inline const char * get_conn_state_name(enum vchiq_connstate conn_state) { … } struct vchiq_slot_zero * vchiq_init_slots(struct device *dev, void *mem_base, int mem_size) { … } int vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero, struct device *dev) { … } void vchiq_msg_queue_push(struct vchiq_instance *instance, unsigned int handle, struct vchiq_header *header) { … } EXPORT_SYMBOL(…); struct vchiq_header *vchiq_msg_hold(struct vchiq_instance *instance, unsigned int handle) { … } EXPORT_SYMBOL(…); static int vchiq_validate_params(struct vchiq_state *state, const struct vchiq_service_params_kernel *params) { … } /* Called from application thread when a client or server service is created. */ struct vchiq_service * vchiq_add_service_internal(struct vchiq_state *state, const struct vchiq_service_params_kernel *params, int srvstate, struct vchiq_instance *instance, void (*userdata_term)(void *userdata)) { … } int vchiq_open_service_internal(struct vchiq_service *service, int client_id) { … } static void release_service_messages(struct vchiq_service *service) { … } static int do_abort_bulks(struct vchiq_service *service) { … } static int close_service_complete(struct vchiq_service *service, int failstate) { … } /* * Prepares a bulk transfer to be queued. The function is interruptible and is * intended to be called from user threads. It may return -EAGAIN to indicate * that a signal has been received and the call should be retried after being * returned to user context. */ static int vchiq_bulk_xfer_queue_msg_interruptible(struct vchiq_service *service, void *offset, void __user *uoffset, int size, void *userdata, enum vchiq_bulk_mode mode, enum vchiq_bulk_dir dir) { … } /* Called by the slot handler */ int vchiq_close_service_internal(struct vchiq_service *service, int close_recvd) { … } /* Called from the application process upon process death */ void vchiq_terminate_service_internal(struct vchiq_service *service) { … } /* Called from the slot handler */ void vchiq_free_service_internal(struct vchiq_service *service) { … } int vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance) { … } void vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance) { … } int vchiq_close_service(struct vchiq_instance *instance, unsigned int handle) { … } EXPORT_SYMBOL(…); int vchiq_remove_service(struct vchiq_instance *instance, unsigned int handle) { … } int vchiq_bulk_xfer_blocking_interruptible(struct vchiq_instance *instance, unsigned int handle, void *offset, void __user *uoffset, int size, void __user *userdata, enum vchiq_bulk_dir dir) { … } int vchiq_bulk_xfer_callback_interruptible(struct vchiq_instance *instance, unsigned int handle, void *offset, void __user *uoffset, int size, enum vchiq_bulk_mode mode, void *userdata, enum vchiq_bulk_dir dir) { … } /* * This function is called by VCHIQ ioctl interface and is interruptible. * It may receive -EAGAIN to indicate that a signal has been received * and the call should be retried after being returned to user context. */ int vchiq_bulk_xfer_waiting_interruptible(struct vchiq_instance *instance, unsigned int handle, struct bulk_waiter *userdata) { … } int vchiq_queue_message(struct vchiq_instance *instance, unsigned int handle, ssize_t (*copy_callback)(void *context, void *dest, size_t offset, size_t maxsize), void *context, size_t size) { … } int vchiq_queue_kernel_message(struct vchiq_instance *instance, unsigned int handle, void *data, unsigned int size) { … } EXPORT_SYMBOL(…); void vchiq_release_message(struct vchiq_instance *instance, unsigned int handle, struct vchiq_header *header) { … } EXPORT_SYMBOL(…); static void release_message_sync(struct vchiq_state *state, struct vchiq_header *header) { … } int vchiq_get_peer_version(struct vchiq_instance *instance, unsigned int handle, short *peer_version) { … } EXPORT_SYMBOL(…); void vchiq_get_config(struct vchiq_config *config) { … } int vchiq_set_service_option(struct vchiq_instance *instance, unsigned int handle, enum vchiq_service_option option, int value) { … } static void vchiq_dump_shared_state(struct seq_file *f, struct vchiq_state *state, struct vchiq_shared_state *shared, const char *label) { … } static void vchiq_dump_service_state(struct seq_file *f, struct vchiq_service *service) { … } void vchiq_dump_state(struct seq_file *f, struct vchiq_state *state) { … } int vchiq_send_remote_use(struct vchiq_state *state) { … } int vchiq_send_remote_use_active(struct vchiq_state *state) { … } void vchiq_log_dump_mem(struct device *dev, const char *label, u32 addr, const void *void_mem, size_t num_bytes) { … }
Codebase Browser
linux