linux/io_uring/rsrc.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/hugetlb.h>
#include <linux/compat.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "alloc_cache.h"
#include "openclose.h"
#include "rsrc.h"
#include "memmap.h"

struct io_rsrc_update { … };

static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
				  struct io_mapped_ubuf **pimu,
				  struct page **last_hpage);

/* only define max */
#define IORING_MAX_FIXED_FILES …
#define IORING_MAX_REG_BUFFERS …

static const struct io_mapped_ubuf dummy_ubuf = …;

int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
{ … }

static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{ … }

static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{ … }

static int io_buffer_validate(struct iovec *iov)
{ … }

static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
{ … }

static void io_rsrc_put_work(struct io_rsrc_node *node)
{ … }

void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
{ … }

void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
	__must_hold(&node->ctx->uring_lock)
{ … }

struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
{ … }

__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
				      struct io_ring_ctx *ctx)
{ … }

static void io_free_page_table(void **table, size_t size)
{ … }

static void io_rsrc_data_free(struct io_rsrc_data *data)
{ … }

static __cold void **io_alloc_page_table(size_t size)
{ … }

__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, int type,
				     u64 __user *utags,
				     unsigned nr, struct io_rsrc_data **pdata)
{ … }

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_rsrc_update2 *up,
				 unsigned nr_args)
{ … }

static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
				   struct io_uring_rsrc_update2 *up,
				   unsigned int nr_args)
{ … }

static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
				     struct io_uring_rsrc_update2 *up,
				     unsigned nr_args)
{ … }

int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
			     unsigned nr_args)
{ … }

int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
			    unsigned size, unsigned type)
{ … }

__cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
			    unsigned int size, unsigned int type)
{ … }

int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ … }

static int io_files_update_with_index_alloc(struct io_kiocb *req,
					    unsigned int issue_flags)
{ … }

int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
{ … }

int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
{ … }

void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{ … }

int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{ … }

int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
			  unsigned nr_args, u64 __user *tags)
{ … }

static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
{ … }

void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
{ … }

int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
{ … }

/*
 * Not super efficient, but this is just a registration time. And we do cache
 * the last compound head, so generally we'll only do a full search if we don't
 * match that one.
 *
 * We check if the given compound head page has already been accounted, to
 * avoid double accounting it. This allows us to account the full size of the
 * page, not just the constituent pages of a huge page.
 */
static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
				  int nr_pages, struct page *hpage)
{ … }

static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
				 int nr_pages, struct io_mapped_ubuf *imu,
				 struct page **last_hpage)
{ … }

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
				  struct io_mapped_ubuf **pimu,
				  struct page **last_hpage)
{ … }

static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
{ … }

int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
			    unsigned int nr_args, u64 __user *tags)
{ … }

int io_import_fixed(int ddir, struct iov_iter *iter,
			   struct io_mapped_ubuf *imu,
			   u64 buf_addr, size_t len)
{ … }