// SPDX-License-Identifier: GPL-2.0 /* * Functions related to segment and merge handling */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/blk-integrity.h> #include <linux/scatterlist.h> #include <linux/part_stat.h> #include <linux/blk-cgroup.h> #include <trace/events/block.h> #include "blk.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" #include "blk-throttle.h" static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv) { … } static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) { … } static inline bool bio_will_gap(struct request_queue *q, struct request *prev_rq, struct bio *prev, struct bio *next) { … } static inline bool req_gap_back_merge(struct request *req, struct bio *bio) { … } static inline bool req_gap_front_merge(struct request *req, struct bio *bio) { … } /* * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size * is defined as 'unsigned int', meantime it has to be aligned to with the * logical block size, which is the minimum accepted unit by hardware. */ static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) { … } static struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, unsigned *nsegs, struct bio_set *bs) { … } static struct bio *bio_split_write_zeroes(struct bio *bio, const struct queue_limits *lim, unsigned *nsegs, struct bio_set *bs) { … } static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { … } /* * Return the maximum number of sectors from the start of a bio that may be * submitted as a single request to a block device. If enough sectors remain, * align the end to the physical block size. Otherwise align the end to the * logical block size. This approach minimizes the number of non-aligned * requests that are submitted to a block device if the start of a bio is not * aligned to a physical block boundary. */ static inline unsigned get_max_io_size(struct bio *bio, const struct queue_limits *lim) { … } /** * get_max_segment_size() - maximum number of bytes to add as a single segment * @lim: Request queue limits. * @paddr: address of the range to add * @len: maximum length available to add at @paddr * * Returns the maximum number of bytes of the range starting at @paddr that can * be added to a single segment. */ static inline unsigned get_max_segment_size(const struct queue_limits *lim, phys_addr_t paddr, unsigned int len) { … } /** * bvec_split_segs - verify whether or not a bvec should be split in the middle * @lim: [in] queue limits to split based on * @bv: [in] bvec to examine * @nsegs: [in,out] Number of segments in the bio being built. Incremented * by the number of segments from @bv that may be appended to that * bio without exceeding @max_segs * @bytes: [in,out] Number of bytes in the bio being built. Incremented * by the number of bytes from @bv that may be appended to that * bio without exceeding @max_bytes * @max_segs: [in] upper bound for *@nsegs * @max_bytes: [in] upper bound for *@bytes * * When splitting a bio, it can happen that a bvec is encountered that is too * big to fit in a single segment and hence that it has to be split in the * middle. This function verifies whether or not that should happen. The value * %true is returned if and only if appending the entire @bv to a bio with * *@nsegs segments and *@sectors sectors would make that bio unacceptable for * the block driver. */ static bool bvec_split_segs(const struct queue_limits *lim, const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes, unsigned max_segs, unsigned max_bytes) { … } /** * bio_split_rw - split a bio in two bios * @bio: [in] bio to be split * @lim: [in] queue limits to split based on * @segs: [out] number of segments in the bio with the first half of the sectors * @bs: [in] bio set to allocate the clone from * @max_bytes: [in] maximum number of bytes per bio * * Clone @bio, update the bi_iter of the clone to represent the first sectors * of @bio and update @bio->bi_iter to represent the remaining sectors. The * following is guaranteed for the cloned bio: * - That it has at most @max_bytes worth of data * - That it has at most queue_max_segments(@q) segments. * * Except for discard requests the cloned bio will point at the bi_io_vec of * the original bio. It is the responsibility of the caller to ensure that the * original bio is not freed before the cloned bio. The caller is also * responsible for ensuring that @bs is only destroyed after processing of the * split bio has finished. */ struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, unsigned *segs, struct bio_set *bs, unsigned max_bytes) { … } EXPORT_SYMBOL_GPL(…); /** * __bio_split_to_limits - split a bio to fit the queue limits * @bio: bio to be split * @lim: queue limits to split based on * @nr_segs: returns the number of segments in the returned bio * * Check if @bio needs splitting based on the queue limits, and if so split off * a bio fitting the limits from the beginning of @bio and return it. @bio is * shortened to the remainder and re-submitted. * * The split bio is allocated from @q->bio_split, which is provided by the * block layer. */ struct bio *__bio_split_to_limits(struct bio *bio, const struct queue_limits *lim, unsigned int *nr_segs) { … } /** * bio_split_to_limits - split a bio to fit the queue limits * @bio: bio to be split * * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and * if so split off a bio fitting the limits from the beginning of @bio and * return it. @bio is shortened to the remainder and re-submitted. * * The split bio is allocated from @q->bio_split, which is provided by the * block layer. */ struct bio *bio_split_to_limits(struct bio *bio) { … } EXPORT_SYMBOL(…); unsigned int blk_recalc_rq_segments(struct request *rq) { … } static inline struct scatterlist *blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist) { … } static unsigned blk_bvec_map_sg(struct request_queue *q, struct bio_vec *bvec, struct scatterlist *sglist, struct scatterlist **sg) { … } static inline int __blk_bvec_map_sg(struct bio_vec bv, struct scatterlist *sglist, struct scatterlist **sg) { … } /* only try to merge bvecs into one sg if they are from two bios */ static inline bool __blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec, struct bio_vec *bvprv, struct scatterlist **sg) { … } static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, struct scatterlist *sglist, struct scatterlist **sg) { … } /* * map a request to scatterlist, return number of sg entries setup. Caller * must make sure sg can hold rq->nr_phys_segments entries */ int __blk_rq_map_sg(struct request_queue *q, struct request *rq, struct scatterlist *sglist, struct scatterlist **last_sg) { … } EXPORT_SYMBOL(…); static inline unsigned int blk_rq_get_max_sectors(struct request *rq, sector_t offset) { … } static inline int ll_new_hw_segment(struct request *req, struct bio *bio, unsigned int nr_phys_segs) { … } int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) { … } static int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) { … } static bool req_attempt_discard_merge(struct request_queue *q, struct request *req, struct request *next) { … } static int ll_merge_requests_fn(struct request_queue *q, struct request *req, struct request *next) { … } /** * blk_rq_set_mixed_merge - mark a request as mixed merge * @rq: request to mark as mixed merge * * Description: * @rq is about to be mixed merged. Make sure the attributes * which can be mixed are set in each bio and mark @rq as mixed * merged. */ static void blk_rq_set_mixed_merge(struct request *rq) { … } static inline blk_opf_t bio_failfast(const struct bio *bio) { … } /* * After we are marked as MIXED_MERGE, any new RA bio has to be updated * as failfast, and request's failfast has to be updated in case of * front merge. */ static inline void blk_update_mixed_merge(struct request *req, struct bio *bio, bool front_merge) { … } static void blk_account_io_merge_request(struct request *req) { … } static enum elv_merge blk_try_req_merge(struct request *req, struct request *next) { … } static bool blk_atomic_write_mergeable_rq_bio(struct request *rq, struct bio *bio) { … } static bool blk_atomic_write_mergeable_rqs(struct request *rq, struct request *next) { … } /* * For non-mq, this has to be called with the request spinlock acquired. * For mq with scheduling, the appropriate queue wide lock should be held. */ static struct request *attempt_merge(struct request_queue *q, struct request *req, struct request *next) { … } static struct request *attempt_back_merge(struct request_queue *q, struct request *rq) { … } static struct request *attempt_front_merge(struct request_queue *q, struct request *rq) { … } /* * Try to merge 'next' into 'rq'. Return true if the merge happened, false * otherwise. The caller is responsible for freeing 'next' if the merge * happened. */ bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next) { … } bool blk_rq_merge_ok(struct request *rq, struct bio *bio) { … } enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { … } static void blk_account_io_merge_bio(struct request *req) { … } enum bio_merge_status bio_attempt_back_merge(struct request *req, struct bio *bio, unsigned int nr_segs) { … } static enum bio_merge_status bio_attempt_front_merge(struct request *req, struct bio *bio, unsigned int nr_segs) { … } static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q, struct request *req, struct bio *bio) { … } static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q, struct request *rq, struct bio *bio, unsigned int nr_segs, bool sched_allow_merge) { … } /** * blk_attempt_plug_merge - try to merge with %current's plugged list * @q: request_queue new bio is being queued at * @bio: new bio being queued * @nr_segs: number of segments in @bio * from the passed in @q already in the plug list * * Determine whether @bio being queued on @q can be merged with the previous * request on %current's plugged list. Returns %true if merge was successful, * otherwise %false. * * Plugging coalesces IOs from the same issuer for the same purpose without * going through @q->queue_lock. As such it's more of an issuing mechanism * than scheduling, and the request, while may have elvpriv data, is not * added on the elevator at this point. In addition, we don't have * reliable access to the elevator outside queue lock. Only check basic * merging parameters without querying the elevator. * * Caller must ensure !blk_queue_nomerges(q) beforehand. */ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { … } /* * Iterate list of requests and see if we can merge this bio with any * of them. */ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, struct bio *bio, unsigned int nr_segs) { … } EXPORT_SYMBOL_GPL(…); bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs, struct request **merged_request) { … } EXPORT_SYMBOL_GPL(…);
Codebase Browser
linux