linux/block/blk-mq-sched.c

// SPDX-License-Identifier: GPL-2.0
/*
 * blk-mq scheduling framework
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/list_sort.h>

#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-sched.h"
#include "blk-wbt.h"

/*
 * Mark a hardware queue as needing a restart.
 */
void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx)
{ … }
EXPORT_SYMBOL_GPL(…);

void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
{ … }

static int sched_rq_cmp(void *priv, const struct list_head *a,
			const struct list_head *b)
{ … }

static bool blk_mq_dispatch_hctx_list(struct list_head *rq_list)
{ … }

#define BLK_MQ_BUDGET_DELAY …

/*
 * Only SCSI implements .get_budget and .put_budget, and SCSI restarts
 * its queue by itself in its completion handler, so we don't need to
 * restart queue if .get_budget() fails to get the budget.
 *
 * Returns -EAGAIN if hctx->dispatch was found non-empty and run_work has to
 * be run again.  This is necessary to avoid starving flushes.
 */
static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
{ … }

static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
{ … }

static struct blk_mq_ctx *blk_mq_next_ctx(struct blk_mq_hw_ctx *hctx,
					  struct blk_mq_ctx *ctx)
{ … }

/*
 * Only SCSI implements .get_budget and .put_budget, and SCSI restarts
 * its queue by itself in its completion handler, so we don't need to
 * restart queue if .get_budget() fails to get the budget.
 *
 * Returns -EAGAIN if hctx->dispatch was found non-empty and run_work has to
 * be run again.  This is necessary to avoid starving flushes.
 */
static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
{ … }

static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
{ … }

void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
{ … }

bool blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
		unsigned int nr_segs)
{ … }

bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq,
				   struct list_head *free)
{ … }
EXPORT_SYMBOL_GPL(…);

static int blk_mq_sched_alloc_map_and_rqs(struct request_queue *q,
					  struct blk_mq_hw_ctx *hctx,
					  unsigned int hctx_idx)
{ … }

static void blk_mq_exit_sched_shared_tags(struct request_queue *queue)
{ … }

/* called in queue's release handler, tagset has gone away */
static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags)
{ … }

static int blk_mq_init_sched_shared_tags(struct request_queue *queue)
{ … }

/* caller must have a reference to @e, will grab another one if successful */
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
{ … }

/*
 * called in either blk_queue_cleanup or elevator_switch, tagset
 * is required for freeing requests
 */
void blk_mq_sched_free_rqs(struct request_queue *q)
{ … }

void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e)
{ … }