linux/io_uring/sqpoll.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Contains the core associated with submission side polling of the SQ
 * ring, offloading submissions from the application to a kernel thread.
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/audit.h>
#include <linux/security.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "napi.h"
#include "sqpoll.h"

#define IORING_SQPOLL_CAP_ENTRIES_VALUE …
#define IORING_TW_CAP_ENTRIES_VALUE …

enum { … };

void io_sq_thread_unpark(struct io_sq_data *sqd)
	__releases(&sqd->lock)
{ … }

void io_sq_thread_park(struct io_sq_data *sqd)
	__acquires(&sqd->lock)
{ … }

void io_sq_thread_stop(struct io_sq_data *sqd)
{ … }

void io_put_sq_data(struct io_sq_data *sqd)
{ … }

static __cold void io_sqd_update_thread_idle(struct io_sq_data *sqd)
{ … }

void io_sq_thread_finish(struct io_ring_ctx *ctx)
{ … }

static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
{ … }

static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
					 bool *attached)
{ … }

static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
{ … }

static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
{ … }

static bool io_sqd_handle_event(struct io_sq_data *sqd)
{ … }

/*
 * Run task_work, processing the retry_list first. The retry_list holds
 * entries that we passed on in the previous run, if we had more task_work
 * than we were asked to process. Newly queued task_work isn't run until the
 * retry list has been fully processed.
 */
static unsigned int io_sq_tw(struct llist_node **retry_list, int max_entries)
{ … }

static bool io_sq_tw_pending(struct llist_node *retry_list)
{ … }

static void io_sq_update_worktime(struct io_sq_data *sqd, struct rusage *start)
{ … }

static int io_sq_thread(void *data)
{ … }

void io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
{ … }

__cold int io_sq_offload_create(struct io_ring_ctx *ctx,
				struct io_uring_params *p)
{ … }

__cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx,
				     cpumask_var_t mask)
{ … }