linux/drivers/block/loop.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 1993 by Theodore Ts'o.
 */
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/major.h>
#include <linux/wait.h>
#include <linux/blkpg.h>
#include <linux/init.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/compat.h>
#include <linux/suspend.h>
#include <linux/freezer.h>
#include <linux/mutex.h>
#include <linux/writeback.h>
#include <linux/completion.h>
#include <linux/highmem.h>
#include <linux/splice.h>
#include <linux/sysfs.h>
#include <linux/miscdevice.h>
#include <linux/falloc.h>
#include <linux/uio.h>
#include <linux/ioprio.h>
#include <linux/blk-cgroup.h>
#include <linux/sched/mm.h>
#include <linux/statfs.h>
#include <linux/uaccess.h>
#include <linux/blk-mq.h>
#include <linux/spinlock.h>
#include <uapi/linux/loop.h>

/* Possible states of device */
enum { … };

struct loop_func_table;

struct loop_device { … };

struct loop_cmd { … };

#define LOOP_IDLE_WORKER_TIMEOUT …
#define LOOP_DEFAULT_HW_Q_DEPTH …

static DEFINE_IDR(loop_index_idr);
static DEFINE_MUTEX(loop_ctl_mutex);
static DEFINE_MUTEX(loop_validate_mutex);

/**
 * loop_global_lock_killable() - take locks for safe loop_validate_file() test
 *
 * @lo: struct loop_device
 * @global: true if @lo is about to bind another "struct loop_device", false otherwise
 *
 * Returns 0 on success, -EINTR otherwise.
 *
 * Since loop_validate_file() traverses on other "struct loop_device" if
 * is_loop_device() is true, we need a global lock for serializing concurrent
 * loop_configure()/loop_change_fd()/__loop_clr_fd() calls.
 */
static int loop_global_lock_killable(struct loop_device *lo, bool global)
{ … }

/**
 * loop_global_unlock() - release locks taken by loop_global_lock_killable()
 *
 * @lo: struct loop_device
 * @global: true if @lo was about to bind another "struct loop_device", false otherwise
 */
static void loop_global_unlock(struct loop_device *lo, bool global)
{ … }

static int max_part;
static int part_shift;

static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
{ … }

static loff_t get_loop_size(struct loop_device *lo, struct file *file)
{ … }

/*
 * We support direct I/O only if lo_offset is aligned with the logical I/O size
 * of backing device, and the logical block size of loop is bigger than that of
 * the backing device.
 */
static bool lo_bdev_can_use_dio(struct loop_device *lo,
		struct block_device *backing_bdev)
{ … }

static void __loop_update_dio(struct loop_device *lo, bool dio)
{ … }

/**
 * loop_set_size() - sets device size and notifies userspace
 * @lo: struct loop_device to set the size for
 * @size: new size of the loop device
 *
 * Callers must validate that the size passed into this function fits into
 * a sector_t, eg using loop_validate_size()
 */
static void loop_set_size(struct loop_device *lo, loff_t size)
{ … }

static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
{ … }

static int lo_write_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
{ … }

static int lo_read_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
{ … }

static void loop_clear_limits(struct loop_device *lo, int mode)
{ … }

static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos,
			int mode)
{ … }

static int lo_req_flush(struct loop_device *lo, struct request *rq)
{ … }

static void lo_complete_rq(struct request *rq)
{ … }

static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
{ … }

static void lo_rw_aio_complete(struct kiocb *iocb, long ret)
{ … }

static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
		     loff_t pos, int rw)
{ … }

static int do_req_filebacked(struct loop_device *lo, struct request *rq)
{ … }

static inline void loop_update_dio(struct loop_device *lo)
{ … }

static void loop_reread_partitions(struct loop_device *lo)
{ … }

static inline int is_loop_device(struct file *file)
{ … }

static int loop_validate_file(struct file *file, struct block_device *bdev)
{ … }

/*
 * loop_change_fd switched the backing store of a loopback device to
 * a new file. This is useful for operating system installers to free up
 * the original file and in High Availability environments to switch to
 * an alternative location for the content in case of server meltdown.
 * This can only work if the loop device is used read-only, and if the
 * new backing store is the same size and type as the old backing store.
 */
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
			  unsigned int arg)
{ … }

/* loop sysfs attributes */

static ssize_t loop_attr_show(struct device *dev, char *page,
			      ssize_t (*callback)(struct loop_device *, char *))
{ … }

#define LOOP_ATTR_RO(_name) …

static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
{ … }

static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
{ … }

static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
{ … }

static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
{ … }

static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
{ … }

static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
{ … }

LOOP_ATTR_RO(backing_file);
LOOP_ATTR_RO(offset);
LOOP_ATTR_RO(sizelimit);
LOOP_ATTR_RO(autoclear);
LOOP_ATTR_RO(partscan);
LOOP_ATTR_RO(dio);

static struct attribute *loop_attrs[] = …;

static struct attribute_group loop_attribute_group = …;

static void loop_sysfs_init(struct loop_device *lo)
{ … }

static void loop_sysfs_exit(struct loop_device *lo)
{ … }

static void loop_config_discard(struct loop_device *lo,
		struct queue_limits *lim)
{ … }

struct loop_worker { … };

static void loop_workfn(struct work_struct *work);

#ifdef CONFIG_BLK_CGROUP
static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
{ … }
#else
static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
{
	return !css;
}
#endif

static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)
{ … }

static void loop_set_timer(struct loop_device *lo)
{ … }

static void loop_free_idle_workers(struct loop_device *lo, bool delete_all)
{ … }

static void loop_free_idle_workers_timer(struct timer_list *timer)
{ … }

/**
 * loop_set_status_from_info - configure device from loop_info
 * @lo: struct loop_device to configure
 * @info: struct loop_info64 to configure the device with
 *
 * Configures the loop device parameters according to the passed
 * in loop_info64 configuration.
 */
static int
loop_set_status_from_info(struct loop_device *lo,
			  const struct loop_info64 *info)
{ … }

static unsigned short loop_default_blocksize(struct loop_device *lo,
		struct block_device *backing_bdev)
{ … }

static int loop_reconfigure_limits(struct loop_device *lo, unsigned short bsize)
{ … }

static int loop_configure(struct loop_device *lo, blk_mode_t mode,
			  struct block_device *bdev,
			  const struct loop_config *config)
{ … }

static void __loop_clr_fd(struct loop_device *lo)
{ … }

static int loop_clr_fd(struct loop_device *lo)
{ … }

static int
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
{ … }

static int
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
{ … }

static void
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
{ … }

static int
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
{ … }

static int
loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
{ … }

static int
loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
{ … }

static int
loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { … }

static int
loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { … }

static int loop_set_capacity(struct loop_device *lo)
{ … }

static int loop_set_dio(struct loop_device *lo, unsigned long arg)
{ … }

static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
{ … }

static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
			   unsigned long arg)
{ … }

static int lo_ioctl(struct block_device *bdev, blk_mode_t mode,
	unsigned int cmd, unsigned long arg)
{ … }

#ifdef CONFIG_COMPAT
struct compat_loop_info { … };

/*
 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
loop_info64_from_compat(const struct compat_loop_info __user *arg,
			struct loop_info64 *info64)
{ … }

/*
 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
loop_info64_to_compat(const struct loop_info64 *info64,
		      struct compat_loop_info __user *arg)
{ … }

static int
loop_set_status_compat(struct loop_device *lo,
		       const struct compat_loop_info __user *arg)
{ … }

static int
loop_get_status_compat(struct loop_device *lo,
		       struct compat_loop_info __user *arg)
{ … }

static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
			   unsigned int cmd, unsigned long arg)
{ … }
#endif

static int lo_open(struct gendisk *disk, blk_mode_t mode)
{ … }

static void lo_release(struct gendisk *disk)
{ … }

static void lo_free_disk(struct gendisk *disk)
{ … }

static const struct block_device_operations lo_fops = …;

/*
 * And now the modules code and kernel interface.
 */

/*
 * If max_loop is specified, create that many devices upfront.
 * This also becomes a hard limit. If max_loop is not specified,
 * the default isn't a hard limit (as before commit 85c50197716c
 * changed the default value from 0 for max_loop=0 reasons), just
 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
 * init time. Loop devices can be requested on-demand with the
 * /dev/loop-control interface, or be instantiated by accessing
 * a 'dead' device node.
 */
static int max_loop = …;

#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
static bool max_loop_specified;

static int max_loop_param_set_int(const char *val,
				  const struct kernel_param *kp)
{ … }

static const struct kernel_param_ops max_loop_param_ops = …;

module_param_cb(…);
MODULE_PARM_DESC(…) …;
#else
module_param(max_loop, int, 0444);
MODULE_PARM_DESC(max_loop, "Initial number of loop devices");
#endif

module_param(max_part, int, 0444);
MODULE_PARM_DESC(…) …;

static int hw_queue_depth = …;

static int loop_set_hw_queue_depth(const char *s, const struct kernel_param *p)
{ … }

static const struct kernel_param_ops loop_hw_qdepth_param_ops = …;

device_param_cb(…);
MODULE_PARM_DESC(…) …;

MODULE_DESCRIPTION(…) …;
MODULE_LICENSE(…) …;
MODULE_ALIAS_BLOCKDEV_MAJOR(…);

static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
		const struct blk_mq_queue_data *bd)
{ … }

static void loop_handle_cmd(struct loop_cmd *cmd)
{ … }

static void loop_process_work(struct loop_worker *worker,
			struct list_head *cmd_list, struct loop_device *lo)
{ … }

static void loop_workfn(struct work_struct *work)
{ … }

static void loop_rootcg_workfn(struct work_struct *work)
{ … }

static const struct blk_mq_ops loop_mq_ops = …;

static int loop_add(int i)
{ … }

static void loop_remove(struct loop_device *lo)
{ … }

#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
static void loop_probe(dev_t dev)
{ … }
#else
#define loop_probe …
#endif /* !CONFIG_BLOCK_LEGACY_AUTOLOAD */

static int loop_control_remove(int idx)
{ … }

static int loop_control_get_free(int idx)
{ … }

static long loop_control_ioctl(struct file *file, unsigned int cmd,
			       unsigned long parm)
{ … }

static const struct file_operations loop_ctl_fops = …;

static struct miscdevice loop_misc = …;

MODULE_ALIAS_MISCDEV(…);
MODULE_ALIAS(…) …;

static int __init loop_init(void)
{ … }

static void __exit loop_exit(void)
{ … }

module_init(…) …;
module_exit(loop_exit);

#ifndef MODULE
static int __init max_loop_setup(char *str)
{ … }

__setup(…);
#endif