// SPDX-License-Identifier: GPL-2.0 /* * Block multiqueue core code * * Copyright (C) 2013-2014 Jens Axboe * Copyright (C) 2013-2014 Christoph Hellwig */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/backing-dev.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/blk-integrity.h> #include <linux/kmemleak.h> #include <linux/mm.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/workqueue.h> #include <linux/smp.h> #include <linux/interrupt.h> #include <linux/llist.h> #include <linux/cpu.h> #include <linux/cache.h> #include <linux/sched/topology.h> #include <linux/sched/signal.h> #include <linux/delay.h> #include <linux/crash_dump.h> #include <linux/prefetch.h> #include <linux/blk-crypto.h> #include <linux/part_stat.h> #include <linux/sched/isolation.h> #include <trace/events/block.h> #include <linux/t10-pi.h> #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" #include "blk-pm.h" #include "blk-stat.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd); static void blk_mq_insert_request(struct request *rq, blk_insert_t flags); static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags); static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, struct list_head *list); static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob, unsigned int flags); /* * Check if any of the ctx, dispatch list or elevator * have pending work in this hardware queue. */ static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) { … } /* * Mark this ctx as having pending work in this hardware queue */ static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx) { … } static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx) { … } struct mq_inflight { … }; static bool blk_mq_check_inflight(struct request *rq, void *priv) { … } unsigned int blk_mq_in_flight(struct request_queue *q, struct block_device *part) { … } void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part, unsigned int inflight[2]) { … } void blk_freeze_queue_start(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_freeze_queue_wait(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, unsigned long timeout) { … } EXPORT_SYMBOL_GPL(…); /* * Guarantee no request is in use, so we can change any data structure of * the queue afterward. */ void blk_freeze_queue(struct request_queue *q) { … } void blk_mq_freeze_queue(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic) { … } void blk_mq_unfreeze_queue(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); /* * FIXME: replace the scsi_internal_device_*block_nowait() calls in the * mpt3sas driver such that this function can be removed. */ void blk_mq_quiesce_queue_nowait(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); /** * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done * @set: tag_set to wait on * * Note: it is driver's responsibility for making sure that quiesce has * been started on or more of the request_queues of the tag_set. This * function only waits for the quiesce on those request_queues that had * the quiesce flag set using blk_mq_quiesce_queue_nowait. */ void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set) { … } EXPORT_SYMBOL_GPL(…); /** * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished * @q: request queue. * * Note: this function does not prevent that the struct request end_io() * callback function is invoked. Once this function is returned, we make * sure no dispatch can happen until the queue is unquiesced via * blk_mq_unquiesce_queue(). */ void blk_mq_quiesce_queue(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); /* * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() * @q: request queue. * * This function recovers queue into the state before quiescing * which is done by blk_mq_quiesce_queue. */ void blk_mq_unquiesce_queue(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_wake_waiters(struct request_queue *q) { … } void blk_rq_init(struct request_queue *q, struct request *rq) { … } EXPORT_SYMBOL(…); /* Set start and alloc time when the allocated request is actually used */ static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns) { … } static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, struct blk_mq_tags *tags, unsigned int tag) { … } static inline struct request * __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) { … } static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) { … } static struct request *blk_mq_rq_cache_fill(struct request_queue *q, struct blk_plug *plug, blk_opf_t opf, blk_mq_req_flags_t flags) { … } static struct request *blk_mq_alloc_cached_request(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags) { … } struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags) { … } EXPORT_SYMBOL(…); struct request *blk_mq_alloc_request_hctx(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx) { … } EXPORT_SYMBOL_GPL(…); static void blk_mq_finish_request(struct request *rq) { … } static void __blk_mq_free_request(struct request *rq) { … } void blk_mq_free_request(struct request *rq) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_free_plug_rqs(struct blk_plug *plug) { … } void blk_dump_rq_flags(struct request *rq, char *msg) { … } EXPORT_SYMBOL(…); static void blk_account_io_completion(struct request *req, unsigned int bytes) { … } static void blk_print_req_error(struct request *req, blk_status_t status) { … } /* * Fully end IO on a request. Does not support partial completions, or * errors. */ static void blk_complete_request(struct request *req) { … } /** * blk_update_request - Complete multiple bytes without completing the request * @req: the request being processed * @error: block status code * @nr_bytes: number of bytes to complete for @req * * Description: * Ends I/O on a number of bytes attached to @req, but doesn't complete * the request structure even if @req doesn't have leftover. * If @req has leftover, sets it up for the next range of segments. * * Passing the result of blk_rq_bytes() as @nr_bytes guarantees * %false return from this function. * * Note: * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function * except in the consistency check at the end of this function. * * Return: * %false - this request doesn't have any more data * %true - this request has more data **/ bool blk_update_request(struct request *req, blk_status_t error, unsigned int nr_bytes) { … } EXPORT_SYMBOL_GPL(…); static inline void blk_account_io_done(struct request *req, u64 now) { … } static inline void blk_account_io_start(struct request *req) { … } static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) { … } inline void __blk_mq_end_request(struct request *rq, blk_status_t error) { … } EXPORT_SYMBOL(…); void blk_mq_end_request(struct request *rq, blk_status_t error) { … } EXPORT_SYMBOL(…); #define TAG_COMP_BATCH … static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, int *tag_array, int nr_tags) { … } void blk_mq_end_request_batch(struct io_comp_batch *iob) { … } EXPORT_SYMBOL_GPL(…); static void blk_complete_reqs(struct llist_head *list) { … } static __latent_entropy void blk_done_softirq(struct softirq_action *h) { … } static int blk_softirq_cpu_dead(unsigned int cpu) { … } static void __blk_mq_complete_request_remote(void *data) { … } static inline bool blk_mq_complete_need_ipi(struct request *rq) { … } static void blk_mq_complete_send_ipi(struct request *rq) { … } static void blk_mq_raise_softirq(struct request *rq) { … } bool blk_mq_complete_request_remote(struct request *rq) { … } EXPORT_SYMBOL_GPL(…); /** * blk_mq_complete_request - end I/O on a request * @rq: the request being processed * * Description: * Complete a request by scheduling the ->complete_rq operation. **/ void blk_mq_complete_request(struct request *rq) { … } EXPORT_SYMBOL(…); /** * blk_mq_start_request - Start processing a request * @rq: Pointer to request to be started * * Function used by device drivers to notify the block layer that a request * is going to be processed now, so blk layer can do proper initializations * such as starting the timeout timer. */ void blk_mq_start_request(struct request *rq) { … } EXPORT_SYMBOL(…); /* * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple * queues. This is important for md arrays to benefit from merging * requests. */ static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug) { … } static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) { … } /** * blk_execute_rq_nowait - insert a request to I/O scheduler for execution * @rq: request to insert * @at_head: insert request at head or tail of queue * * Description: * Insert a fully prepared request at the back of the I/O scheduler queue * for execution. Don't wait for completion. * * Note: * This function will invoke @done directly if the queue is dead. */ void blk_execute_rq_nowait(struct request *rq, bool at_head) { … } EXPORT_SYMBOL_GPL(…); struct blk_rq_wait { … }; static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret) { … } bool blk_rq_is_poll(struct request *rq) { … } EXPORT_SYMBOL_GPL(…); static void blk_rq_poll_completion(struct request *rq, struct completion *wait) { … } /** * blk_execute_rq - insert a request into queue for execution * @rq: request to insert * @at_head: insert request at head or tail of queue * * Description: * Insert a fully prepared request at the back of the I/O scheduler queue * for execution and wait for completion. * Return: The blk_status_t result provided to blk_mq_end_request(). */ blk_status_t blk_execute_rq(struct request *rq, bool at_head) { … } EXPORT_SYMBOL(…); static void __blk_mq_requeue_request(struct request *rq) { … } void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) { … } EXPORT_SYMBOL(…); static void blk_mq_requeue_work(struct work_struct *work) { … } void blk_mq_kick_requeue_list(struct request_queue *q) { … } EXPORT_SYMBOL(…); void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs) { … } EXPORT_SYMBOL(…); static bool blk_is_flush_data_rq(struct request *rq) { … } static bool blk_mq_rq_inflight(struct request *rq, void *priv) { … } bool blk_mq_queue_inflight(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); static void blk_mq_rq_timed_out(struct request *req) { … } struct blk_expired_data { … }; static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired) { … } void blk_mq_put_rq_ref(struct request *rq) { … } static bool blk_mq_check_expired(struct request *rq, void *priv) { … } static bool blk_mq_handle_expired(struct request *rq, void *priv) { … } static void blk_mq_timeout_work(struct work_struct *work) { … } struct flush_busy_ctx_data { … }; static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) { … } /* * Process software queues that have been marked busy, splicing them * to the for-dispatch */ void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) { … } EXPORT_SYMBOL_GPL(…); struct dispatch_rq_data { … }; static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) { … } struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *start) { … } bool __blk_mq_alloc_driver_tag(struct request *rq) { … } static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, int flags, void *key) { … } /* * Mark us waiting for a tag. For shared tags, this involves hooking us into * the tag wakeups. For non-shared tags, we can simply mark us needing a * restart. For both cases, take care to check the condition again after * marking us as waiting. */ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, struct request *rq) { … } #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT … #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR … /* * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): * - EWMA is one simple way to compute running average value * - weight(7/8 and 1/8) is applied so that it can decrease exponentially * - take 4 as factor for avoiding to get too small(0) result, and this * factor doesn't matter because EWMA decreases exponentially */ static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) { … } #define BLK_MQ_RESOURCE_DELAY … static void blk_mq_handle_dev_resource(struct request *rq, struct list_head *list) { … } enum prep_dispatch { … }; static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, bool need_budget) { … } /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */ static void blk_mq_release_budgets(struct request_queue *q, struct list_head *list) { … } /* * blk_mq_commit_rqs will notify driver using bd->last that there is no * more requests. (See comment in struct blk_mq_ops for commit_rqs for * details) * Attention, we should explicitly call this in unusual cases: * 1) did not queue everything initially scheduled to queue * 2) the last attempt to queue a request failed */ static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued, bool from_schedule) { … } /* * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, unsigned int nr_budgets) { … } static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) { … } /* * ->next_cpu is always calculated from hctx->cpumask, so simply use * it for speeding up the check */ static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx) { … } /* * It'd be great if the workqueue API had a way to pass * in a mask and had some smarts for more clever placement. * For now we just round-robin here, switching for every * BLK_MQ_CPU_WORK_BATCH queued items. */ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) { … } /** * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. * @hctx: Pointer to the hardware queue to run. * @msecs: Milliseconds of delay to wait before running the queue. * * Run a hardware queue asynchronously with a delay of @msecs. */ void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) { … } EXPORT_SYMBOL(…); /** * blk_mq_run_hw_queue - Start to run a hardware queue. * @hctx: Pointer to the hardware queue to run. * @async: If we want to run the queue asynchronously. * * Check if the request queue is not in a quiesced state and if there are * pending requests to be sent. If this is true, run the queue to send requests * to hardware. */ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { … } EXPORT_SYMBOL(…); /* * Return prefered queue to dispatch from (if any) for non-mq aware IO * scheduler. */ static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) { … } /** * blk_mq_run_hw_queues - Run all hardware queues in a request queue. * @q: Pointer to the request queue to run. * @async: If we want to run the queue asynchronously. */ void blk_mq_run_hw_queues(struct request_queue *q, bool async) { … } EXPORT_SYMBOL(…); /** * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously. * @q: Pointer to the request queue to run. * @msecs: Milliseconds of delay to wait before running the queues. */ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) { … } EXPORT_SYMBOL(…); /* * This function is often used for pausing .queue_rq() by driver when * there isn't enough resource or some conditions aren't satisfied, and * BLK_STS_RESOURCE is usually returned. * * We do not guarantee that dispatch can be drained or blocked * after blk_mq_stop_hw_queue() returns. Please use * blk_mq_quiesce_queue() for that requirement. */ void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) { … } EXPORT_SYMBOL(…); /* * This function is often used for pausing .queue_rq() by driver when * there isn't enough resource or some conditions aren't satisfied, and * BLK_STS_RESOURCE is usually returned. * * We do not guarantee that dispatch can be drained or blocked * after blk_mq_stop_hw_queues() returns. Please use * blk_mq_quiesce_queue() for that requirement. */ void blk_mq_stop_hw_queues(struct request_queue *q) { … } EXPORT_SYMBOL(…); void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) { … } EXPORT_SYMBOL(…); void blk_mq_start_hw_queues(struct request_queue *q) { … } EXPORT_SYMBOL(…); void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) { … } EXPORT_SYMBOL(…); static void blk_mq_run_work_fn(struct work_struct *work) { … } /** * blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted. * @flags: BLK_MQ_INSERT_* * * Should only be used carefully, when the caller knows we want to * bypass a potential IO scheduler on the target device. */ static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags) { … } static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, struct list_head *list, bool run_queue_async) { … } static void blk_mq_insert_request(struct request *rq, blk_insert_t flags) { … } static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, unsigned int nr_segs) { … } static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, struct request *rq, bool last) { … } static bool blk_mq_get_budget_and_tag(struct request *rq) { … } /** * blk_mq_try_issue_directly - Try to send a request directly to device driver. * @hctx: Pointer of the associated hardware queue. * @rq: Pointer to request to be sent. * * If the device has enough resources to accept a new request now, send the * request directly to device driver. Else, insert at hctx->dispatch queue, so * we can try send it another time in the future. Requests inserted at this * queue have higher priority. */ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, struct request *rq) { … } static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) { … } static void blk_mq_plug_issue_direct(struct blk_plug *plug) { … } static void __blk_mq_flush_plug_list(struct request_queue *q, struct blk_plug *plug) { … } static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) { … } void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) { … } static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, struct list_head *list) { … } static bool blk_mq_attempt_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { … } static struct request *blk_mq_get_new_requests(struct request_queue *q, struct blk_plug *plug, struct bio *bio, unsigned int nsegs) { … } /* * Check if there is a suitable cached request and return it. */ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, struct request_queue *q, blk_opf_t opf) { … } static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, struct bio *bio) { … } static bool bio_unaligned(const struct bio *bio, struct request_queue *q) { … } /** * blk_mq_submit_bio - Create and send a request to block device. * @bio: Bio pointer. * * Builds up a request structure from @q and @bio and send to the device. The * request may not be queued directly to hardware if: * * This request can be merged with another one * * We want to place request at plug queue for possible future merging * * There is an IO scheduler active at this queue * * It will not queue the request if there is an error with the bio, or at the * request creation. */ void blk_mq_submit_bio(struct bio *bio) { … } #ifdef CONFIG_BLK_MQ_STACKING /** * blk_insert_cloned_request - Helper for stacking drivers to submit a request * @rq: the request being queued */ blk_status_t blk_insert_cloned_request(struct request *rq) { … } EXPORT_SYMBOL_GPL(…); /** * blk_rq_unprep_clone - Helper function to free all bios in a cloned request * @rq: the clone request to be cleaned up * * Description: * Free all bios in @rq for a cloned request. */ void blk_rq_unprep_clone(struct request *rq) { … } EXPORT_SYMBOL_GPL(…); /** * blk_rq_prep_clone - Helper function to setup clone request * @rq: the request to be setup * @rq_src: original request to be cloned * @bs: bio_set that bios for clone are allocated from * @gfp_mask: memory allocation mask for bio * @bio_ctr: setup function to be called for each clone bio. * Returns %0 for success, non %0 for failure. * @data: private data to be passed to @bio_ctr * * Description: * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. * Also, pages which the original bios are pointing to are not copied * and the cloned bios just point same pages. * So cloned bios must be completed before original bios, which means * the caller must complete @rq before @rq_src. */ int blk_rq_prep_clone(struct request *rq, struct request *rq_src, struct bio_set *bs, gfp_t gfp_mask, int (*bio_ctr)(struct bio *, struct bio *, void *), void *data) { … } EXPORT_SYMBOL_GPL(…); #endif /* CONFIG_BLK_MQ_STACKING */ /* * Steal bios from a request and add them to a bio list. * The request must not have been partially completed before. */ void blk_steal_bios(struct bio_list *list, struct request *rq) { … } EXPORT_SYMBOL_GPL(…); static size_t order_to_size(unsigned int order) { … } /* called before freeing request pool in @tags */ static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, struct blk_mq_tags *tags) { … } void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) { … } void blk_mq_free_rq_map(struct blk_mq_tags *tags) { … } static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set, unsigned int hctx_idx) { … } static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set, unsigned int hctx_idx) { … } static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, unsigned int hctx_idx, unsigned int nr_tags, unsigned int reserved_tags) { … } static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, int node) { … } static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx, unsigned int depth) { … } struct rq_iter_data { … }; static bool blk_mq_has_request(struct request *rq, void *data) { … } static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) { … } static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx, unsigned int this_cpu) { … } static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) { … } /* * Check if one CPU is mapped to the specified hctx * * Isolated CPUs have been ruled out from hctx->cpumask, which is supposed * to be used for scheduling kworker only. For other usage, please call this * helper for checking if one CPU belongs to the specified hctx */ static bool blk_mq_cpu_mapped_to_hctx(unsigned int cpu, const struct blk_mq_hw_ctx *hctx) { … } static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node) { … } /* * 'cpu' is going away. splice any existing rq_list entries from this * software queue to the hw queue dispatch list, and ensure that it * gets run. */ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) { … } static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) { … } /* * Before freeing hw queue, clearing the flush request reference in * tags->rqs[] for avoiding potential UAF. */ static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, unsigned int queue_depth, struct request *flush_rq) { … } /* hctx->ctxs will be freed in queue's release handler */ static void blk_mq_exit_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { … } static void blk_mq_exit_hw_queues(struct request_queue *q, struct blk_mq_tag_set *set, int nr_queue) { … } static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) { … } static struct blk_mq_hw_ctx * blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, int node) { … } static void blk_mq_init_cpu_queues(struct request_queue *q, unsigned int nr_hw_queues) { … } struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, unsigned int hctx_idx, unsigned int depth) { … } static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, int hctx_idx) { … } void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) { … } static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, unsigned int hctx_idx) { … } static void blk_mq_map_swqueue(struct request_queue *q) { … } /* * Caller needs to ensure that we're either frozen/quiesced, or that * the queue isn't live yet. */ static void queue_set_hctx_shared(struct request_queue *q, bool shared) { … } static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set, bool shared) { … } static void blk_mq_del_queue_tag_set(struct request_queue *q) { … } static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, struct request_queue *q) { … } /* All allocations will be freed in release handler of q->mq_kobj */ static int blk_mq_alloc_ctxs(struct request_queue *q) { … } /* * It is the actual release handler for mq, but we do it from * request queue's release handler for avoiding use-after-free * and headache because q->mq_kobj shouldn't have been introduced, * but we can't group ctx/kctx kobj without it. */ void blk_mq_release(struct request_queue *q) { … } static bool blk_mq_can_poll(struct blk_mq_tag_set *set) { … } struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set, struct queue_limits *lim, void *queuedata) { … } EXPORT_SYMBOL(…); /** * blk_mq_destroy_queue - shutdown a request queue * @q: request queue to shutdown * * This shuts down a request queue allocated by blk_mq_alloc_queue(). All future * requests will be failed with -ENODEV. The caller is responsible for dropping * the reference from blk_mq_alloc_queue() by calling blk_put_queue(). * * Context: can sleep */ void blk_mq_destroy_queue(struct request_queue *q) { … } EXPORT_SYMBOL(…); struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, struct queue_limits *lim, void *queuedata, struct lock_class_key *lkclass) { … } EXPORT_SYMBOL(…); struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, struct lock_class_key *lkclass) { … } EXPORT_SYMBOL(…); static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( struct blk_mq_tag_set *set, struct request_queue *q, int hctx_idx, int node) { … } static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, struct request_queue *q) { … } int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q) { … } EXPORT_SYMBOL(…); /* tags can _not_ be used after returning from blk_mq_exit_queue */ void blk_mq_exit_queue(struct request_queue *q) { … } static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) { … } /* * Allocate the request maps associated with this tag_set. Note that this * may reduce the depth asked for, if memory is tight. set->queue_depth * will be updated to reflect the allocated depth. */ static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set) { … } static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) { … } static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, int new_nr_hw_queues) { … } /* * Alloc a tag set to be associated with one or more request queues. * May fail with EINVAL for various error conditions. May adjust the * requested depth down, if it's too large. In that case, the set * value will be stored in set->queue_depth. */ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) { … } EXPORT_SYMBOL(…); /* allocate and initialize a tagset for a simple single-queue device */ int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set, const struct blk_mq_ops *ops, unsigned int queue_depth, unsigned int set_flags) { … } EXPORT_SYMBOL_GPL(…); void blk_mq_free_tag_set(struct blk_mq_tag_set *set) { … } EXPORT_SYMBOL(…); int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) { … } /* * request_queue and elevator_type pair. * It is just used by __blk_mq_update_nr_hw_queues to cache * the elevator_type associated with a request_queue. */ struct blk_mq_qe_pair { … }; /* * Cache the elevator_type in qe pair list and switch the * io scheduler to 'none' */ static bool blk_mq_elv_switch_none(struct list_head *head, struct request_queue *q) { … } static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head, struct request_queue *q) { … } static void blk_mq_elv_switch_back(struct list_head *head, struct request_queue *q) { … } static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) { … } void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) { … } EXPORT_SYMBOL_GPL(…); static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob, unsigned int flags) { … } int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob, unsigned int flags) { … } int blk_rq_poll(struct request *rq, struct io_comp_batch *iob, unsigned int poll_flags) { … } EXPORT_SYMBOL_GPL(…); unsigned int blk_mq_rq_cpu(struct request *rq) { … } EXPORT_SYMBOL(…); void blk_mq_cancel_work_sync(struct request_queue *q) { … } static int __init blk_mq_init(void) { … } subsys_initcall(blk_mq_init);
Codebase Browser
linux