linux/drivers/md/dm-rq.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2016 Red Hat, Inc. All rights reserved.
 *
 * This file is released under the GPL.
 */

#include "dm-core.h"
#include "dm-rq.h"

#include <linux/blk-mq.h>

#define DM_MSG_PREFIX …

/*
 * One of these is allocated per request.
 */
struct dm_rq_target_io { … };

#define DM_MQ_NR_HW_QUEUES …
#define DM_MQ_QUEUE_DEPTH …
static unsigned int dm_mq_nr_hw_queues = …;
static unsigned int dm_mq_queue_depth = …;

/*
 * Request-based DM's mempools' reserved IOs set by the user.
 */
#define RESERVED_REQUEST_BASED_IOS …
static unsigned int reserved_rq_based_ios = …;

unsigned int dm_get_reserved_rq_based_ios(void)
{ … }

static unsigned int dm_get_blk_mq_nr_hw_queues(void)
{ … }

static unsigned int dm_get_blk_mq_queue_depth(void)
{ … }

int dm_request_based(struct mapped_device *md)
{ … }

void dm_start_queue(struct request_queue *q)
{ … }

void dm_stop_queue(struct request_queue *q)
{ … }

/*
 * Partial completion handling for request-based dm
 */
static void end_clone_bio(struct bio *clone)
{ … }

static struct dm_rq_target_io *tio_from_request(struct request *rq)
{ … }

static void rq_end_stats(struct mapped_device *md, struct request *orig)
{ … }

/*
 * Don't touch any member of the md after calling this function because
 * the md may be freed in dm_put() at the end of this function.
 * Or do dm_get() before calling this function and dm_put() later.
 */
static void rq_completed(struct mapped_device *md)
{ … }

/*
 * Complete the clone and the original request.
 * Must be called without clone's queue lock held,
 * see end_clone_request() for more details.
 */
static void dm_end_request(struct request *clone, blk_status_t error)
{ … }

static void __dm_mq_kick_requeue_list(struct request_queue *q, unsigned long msecs)
{ … }

void dm_mq_kick_requeue_list(struct mapped_device *md)
{ … }
EXPORT_SYMBOL(…);

static void dm_mq_delay_requeue_request(struct request *rq, unsigned long msecs)
{ … }

static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_requeue)
{ … }

static void dm_done(struct request *clone, blk_status_t error, bool mapped)
{ … }

/*
 * Request completion handler for request-based dm
 */
static void dm_softirq_done(struct request *rq)
{ … }

/*
 * Complete the clone and the original request with the error status
 * through softirq context.
 */
static void dm_complete_request(struct request *rq, blk_status_t error)
{ … }

/*
 * Complete the not-mapped clone and the original request with the error status
 * through softirq context.
 * Target's rq_end_io() function isn't called.
 * This may be used when the target's clone_and_map_rq() function fails.
 */
static void dm_kill_unmapped_request(struct request *rq, blk_status_t error)
{ … }

static enum rq_end_io_ret end_clone_request(struct request *clone,
					    blk_status_t error)
{ … }

static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
				 void *data)
{ … }

static int setup_clone(struct request *clone, struct request *rq,
		       struct dm_rq_target_io *tio, gfp_t gfp_mask)
{ … }

static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
		     struct mapped_device *md)
{ … }

/*
 * Returns:
 * DM_MAPIO_*       : the request has been processed as indicated
 * DM_MAPIO_REQUEUE : the original request needs to be immediately requeued
 * < 0              : the request was completed due to failure
 */
static int map_request(struct dm_rq_target_io *tio)
{ … }

/* DEPRECATED: previously used for request-based merge heuristic in dm_request_fn() */
ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf)
{ … }

ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md,
						     const char *buf, size_t count)
{ … }

static void dm_start_request(struct mapped_device *md, struct request *orig)
{ … }

static int dm_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
			      unsigned int hctx_idx, unsigned int numa_node)
{ … }

static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
			  const struct blk_mq_queue_data *bd)
{ … }

static const struct blk_mq_ops dm_mq_ops = …;

int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t)
{ … }

void dm_mq_cleanup_mapped_device(struct mapped_device *md)
{ … }

module_param(reserved_rq_based_ios, uint, 0644);
MODULE_PARM_DESC(…) …;

/* Unused, but preserved for userspace compatibility */
static bool use_blk_mq = …;
module_param(use_blk_mq, bool, 0644);
MODULE_PARM_DESC(…) …;

module_param(dm_mq_nr_hw_queues, uint, 0644);
MODULE_PARM_DESC(…) …;

module_param(dm_mq_queue_depth, uint, 0644);
MODULE_PARM_DESC(…) …;