linux/io_uring/timeout.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include <trace/events/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "refs.h"
#include "cancel.h"
#include "timeout.h"

struct io_timeout { … };

struct io_timeout_rem { … };

static inline bool io_is_timeout_noseq(struct io_kiocb *req)
{ … }

static inline void io_put_req(struct io_kiocb *req)
{ … }

static inline bool io_timeout_finish(struct io_timeout *timeout,
				     struct io_timeout_data *data)
{ … }

static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);

static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
{ … }

static bool io_kill_timeout(struct io_kiocb *req, int status)
	__must_hold(&req->ctx->timeout_lock)
{ … }

__cold void io_flush_timeouts(struct io_ring_ctx *ctx)
{ … }

static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
{ … }

static void io_fail_links(struct io_kiocb *req)
	__must_hold(&req->ctx->completion_lock)
{ … }

static inline void io_remove_next_linked(struct io_kiocb *req)
{ … }

void io_disarm_next(struct io_kiocb *req)
	__must_hold(&req->ctx->completion_lock)
{ … }

struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
					    struct io_kiocb *link)
	__must_hold(&req->ctx->completion_lock)
	__must_hold(&req->ctx->timeout_lock)
{ … }

static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{ … }

static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
					   struct io_cancel_data *cd)
	__must_hold(&ctx->timeout_lock)
{ … }

int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
	__must_hold(&ctx->completion_lock)
{ … }

static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
{ … }

static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
{ … }

static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
{ … }

static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
				    struct timespec64 *ts, enum hrtimer_mode mode)
	__must_hold(&ctx->timeout_lock)
{ … }

static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
			     struct timespec64 *ts, enum hrtimer_mode mode)
	__must_hold(&ctx->timeout_lock)
{ … }

int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ … }

static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
{ … }

/*
 * Remove or update an existing timeout command
 */
int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
{ … }

static int __io_timeout_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe,
			     bool is_timeout_link)
{ … }

int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ … }

int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ … }

int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
{ … }

void io_queue_linked_timeout(struct io_kiocb *req)
{ … }

static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
			  bool cancel_all)
	__must_hold(&head->ctx->timeout_lock)
{ … }

/* Returns true if we found and killed one or more timeouts */
__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
			     bool cancel_all)
{ … }