linux/drivers/md/dm-io.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2003 Sistina Software
 * Copyright (C) 2006 Red Hat GmbH
 *
 * This file is released under the GPL.
 */

#include "dm-core.h"

#include <linux/device-mapper.h>

#include <linux/bio.h>
#include <linux/completion.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/dm-io.h>

#define DM_MSG_PREFIX …

#define DM_IO_MAX_REGIONS …

struct dm_io_client { … };

/*
 * Aligning 'struct io' reduces the number of bits required to store
 * its address.  Refer to store_io_and_region_in_bio() below.
 */
struct io { … } __aligned(…);

static struct kmem_cache *_dm_io_cache;

/*
 * Create a client with mempool and bioset.
 */
struct dm_io_client *dm_io_client_create(void)
{ … }
EXPORT_SYMBOL(…);

void dm_io_client_destroy(struct dm_io_client *client)
{ … }
EXPORT_SYMBOL(…);

/*
 *-------------------------------------------------------------------
 * We need to keep track of which region a bio is doing io for.
 * To avoid a memory allocation to store just 5 or 6 bits, we
 * ensure the 'struct io' pointer is aligned so enough low bits are
 * always zero and then combine it with the region number directly in
 * bi_private.
 *-------------------------------------------------------------------
 */
static void store_io_and_region_in_bio(struct bio *bio, struct io *io,
				       unsigned int region)
{ … }

static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,
				       unsigned int *region)
{ … }

/*
 *--------------------------------------------------------------
 * We need an io object to keep track of the number of bios that
 * have been dispatched for a particular io.
 *--------------------------------------------------------------
 */
static void complete_io(struct io *io)
{ … }

static void dec_count(struct io *io, unsigned int region, blk_status_t error)
{ … }

static void endio(struct bio *bio)
{ … }

/*
 *--------------------------------------------------------------
 * These little objects provide an abstraction for getting a new
 * destination page for io.
 *--------------------------------------------------------------
 */
struct dpages { … };

/*
 * Functions for getting the pages from a list.
 */
static void list_get_page(struct dpages *dp,
		  struct page **p, unsigned long *len, unsigned int *offset)
{ … }

static void list_next_page(struct dpages *dp)
{ … }

static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned int offset)
{ … }

/*
 * Functions for getting the pages from a bvec.
 */
static void bio_get_page(struct dpages *dp, struct page **p,
			 unsigned long *len, unsigned int *offset)
{ … }

static void bio_next_page(struct dpages *dp)
{ … }

static void bio_dp_init(struct dpages *dp, struct bio *bio)
{ … }

/*
 * Functions for getting the pages from a VMA.
 */
static void vm_get_page(struct dpages *dp,
		 struct page **p, unsigned long *len, unsigned int *offset)
{ … }

static void vm_next_page(struct dpages *dp)
{ … }

static void vm_dp_init(struct dpages *dp, void *data)
{ … }

/*
 * Functions for getting the pages from kernel memory.
 */
static void km_get_page(struct dpages *dp, struct page **p, unsigned long *len,
			unsigned int *offset)
{ … }

static void km_next_page(struct dpages *dp)
{ … }

static void km_dp_init(struct dpages *dp, void *data)
{ … }

/*
 *---------------------------------------------------------------
 * IO routines that accept a list of pages.
 *---------------------------------------------------------------
 */
static void do_region(const blk_opf_t opf, unsigned int region,
		      struct dm_io_region *where, struct dpages *dp,
		      struct io *io, unsigned short ioprio)
{ … }

static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
			struct dm_io_region *where, struct dpages *dp,
			struct io *io, unsigned short ioprio)
{ … }

static void async_io(struct dm_io_client *client, unsigned int num_regions,
		     struct dm_io_region *where, blk_opf_t opf,
		     struct dpages *dp, io_notify_fn fn, void *context,
		     unsigned short ioprio)
{ … }

struct sync_io { … };

static void sync_io_complete(unsigned long error, void *context)
{ … }

static int sync_io(struct dm_io_client *client, unsigned int num_regions,
		   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
		   unsigned long *error_bits, unsigned short ioprio)
{ … }

static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
		   unsigned long size)
{ … }

int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
	  struct dm_io_region *where, unsigned long *sync_error_bits,
	  unsigned short ioprio)
{ … }
EXPORT_SYMBOL(…);

int __init dm_io_init(void)
{ … }

void dm_io_exit(void)
{ … }