/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Header file for the BFQ I/O scheduler: data structures and * prototypes of interface functions among BFQ components. */ #ifndef _BFQ_H #define _BFQ_H #include <linux/blktrace_api.h> #include <linux/hrtimer.h> #include "blk-cgroup-rwstat.h" #define BFQ_IOPRIO_CLASSES … #define BFQ_CL_IDLE_TIMEOUT … #define BFQ_MIN_WEIGHT … #define BFQ_MAX_WEIGHT … #define BFQ_WEIGHT_CONVERSION_COEFF … #define BFQ_DEFAULT_QUEUE_IOPRIO … #define BFQ_DEFAULT_GRP_IOPRIO … #define BFQ_DEFAULT_GRP_CLASS … #define MAX_BFQQ_NAME_LENGTH … /* * Soft real-time applications are extremely more latency sensitive * than interactive ones. Over-raise the weight of the former to * privilege them against the latter. */ #define BFQ_SOFTRT_WEIGHT_FACTOR … /* * Maximum number of actuators supported. This constant is used simply * to define the size of the static array that will contain * per-actuator data. The current value is hopefully a good upper * bound to the possible number of actuators of any actual drive. */ #define BFQ_MAX_ACTUATORS … struct bfq_entity; /** * struct bfq_service_tree - per ioprio_class service tree. * * Each service tree represents a B-WF2Q+ scheduler on its own. Each * ioprio_class has its own independent scheduler, and so its own * bfq_service_tree. All the fields are protected by the queue lock * of the containing bfqd. */ struct bfq_service_tree { … }; /** * struct bfq_sched_data - multi-class scheduler. * * bfq_sched_data is the basic scheduler queue. It supports three * ioprio_classes, and can be used either as a toplevel queue or as an * intermediate queue in a hierarchical setup. * * The supported ioprio_classes are the same as in CFQ, in descending * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE. * Requests from higher priority queues are served before all the * requests from lower priority queues; among requests of the same * queue requests are served according to B-WF2Q+. * * The schedule is implemented by the service trees, plus the field * @next_in_service, which points to the entity on the active trees * that will be served next, if 1) no changes in the schedule occurs * before the current in-service entity is expired, 2) the in-service * queue becomes idle when it expires, and 3) if the entity pointed by * in_service_entity is not a queue, then the in-service child entity * of the entity pointed by in_service_entity becomes idle on * expiration. This peculiar definition allows for the following * optimization, not yet exploited: while a given entity is still in * service, we already know which is the best candidate for next * service among the other active entities in the same parent * entity. We can then quickly compare the timestamps of the * in-service entity with those of such best candidate. * * All fields are protected by the lock of the containing bfqd. */ struct bfq_sched_data { … }; /** * struct bfq_weight_counter - counter of the number of all active queues * with a given weight. */ struct bfq_weight_counter { … }; /** * struct bfq_entity - schedulable entity. * * A bfq_entity is used to represent either a bfq_queue (leaf node in the * cgroup hierarchy) or a bfq_group into the upper level scheduler. Each * entity belongs to the sched_data of the parent group in the cgroup * hierarchy. Non-leaf entities have also their own sched_data, stored * in @my_sched_data. * * Each entity stores independently its priority values; this would * allow different weights on different devices, but this * functionality is not exported to userspace by now. Priorities and * weights are updated lazily, first storing the new values into the * new_* fields, then setting the @prio_changed flag. As soon as * there is a transition in the entity state that allows the priority * update to take place the effective and the requested priority * values are synchronized. * * Unless cgroups are used, the weight value is calculated from the * ioprio to export the same interface as CFQ. When dealing with * "well-behaved" queues (i.e., queues that do not spend too much * time to consume their budget and have true sequential behavior, and * when there are no external factors breaking anticipation) the * relative weights at each level of the cgroups hierarchy should be * guaranteed. All the fields are protected by the queue lock of the * containing bfqd. */ struct bfq_entity { … }; struct bfq_group; /** * struct bfq_ttime - per process thinktime stats. */ struct bfq_ttime { … }; /** * struct bfq_queue - leaf schedulable entity. * * A bfq_queue is a leaf request queue; it can be associated with an * io_context or more, if it is async or shared between cooperating * processes. Besides, it contains I/O requests for only one actuator * (an io_context is associated with a different bfq_queue for each * actuator it generates I/O for). @cgroup holds a reference to the * cgroup, to be sure that it does not disappear while a bfqq still * references it (mostly to avoid races between request issuing and * task migration followed by cgroup destruction). All the fields are * protected by the queue lock of the containing bfqd. */ struct bfq_queue { … }; /** * struct bfq_data - bfqq data unique and persistent for associated bfq_io_cq */ struct bfq_iocq_bfqq_data { … }; /** * struct bfq_io_cq - per (request_queue, io_context) structure. */ struct bfq_io_cq { … }; /** * struct bfq_data - per-device data structure. * * All the fields are protected by @lock. */ struct bfq_data { … }; enum bfqq_state_flags { … }; #define BFQ_BFQQ_FNS … BFQ_BFQQ_FNS(just_created); BFQ_BFQQ_FNS(busy); BFQ_BFQQ_FNS(wait_request); BFQ_BFQQ_FNS(non_blocking_wait_rq); BFQ_BFQQ_FNS(fifo_expire); BFQ_BFQQ_FNS(has_short_ttime); BFQ_BFQQ_FNS(sync); BFQ_BFQQ_FNS(IO_bound); BFQ_BFQQ_FNS(in_large_burst); BFQ_BFQQ_FNS(coop); BFQ_BFQQ_FNS(split_coop); BFQ_BFQQ_FNS(softrt_update); #undef BFQ_BFQQ_FNS /* Expiration reasons. */ enum bfqq_expiration { … }; struct bfq_stat { … }; struct bfqg_stats { … }; #ifdef CONFIG_BFQ_GROUP_IOSCHED /* * struct bfq_group_data - per-blkcg storage for the blkio subsystem. * * @ps: @blkcg_policy_storage that this structure inherits * @weight: weight of the bfq_group */ struct bfq_group_data { … }; /** * struct bfq_group - per (device, cgroup) data structure. * @entity: schedulable entity to insert into the parent group sched_data. * @sched_data: own sched_data, to contain child entities (they may be * both bfq_queues and bfq_groups). * @bfqd: the bfq_data for the device this group acts upon. * @async_bfqq: array of async queues for all the tasks belonging to * the group, one queue per ioprio value per ioprio_class, * except for the idle class that has only one queue. * @async_idle_bfqq: async queue for the idle class (ioprio is ignored). * @my_entity: pointer to @entity, %NULL for the toplevel group; used * to avoid too many special cases during group creation/ * migration. * @stats: stats for this bfqg. * @active_entities: number of active entities belonging to the group; * unused for the root group. Used to know whether there * are groups with more than one active @bfq_entity * (see the comments to the function * bfq_bfqq_may_idle()). * @rq_pos_tree: rbtree sorted by next_request position, used when * determining if two or more queues have interleaving * requests (see bfq_find_close_cooperator()). * * Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup * there is a set of bfq_groups, each one collecting the lower-level * entities belonging to the group that are acting on the same device. * * Locking works as follows: * o @bfqd is protected by the queue lock, RCU is used to access it * from the readers. * o All the other fields are protected by the @bfqd queue lock. */ struct bfq_group { … }; #else struct bfq_group { struct bfq_entity entity; struct bfq_sched_data sched_data; struct bfq_queue *async_bfqq[2][IOPRIO_NR_LEVELS][BFQ_MAX_ACTUATORS]; struct bfq_queue *async_idle_bfqq[BFQ_MAX_ACTUATORS]; struct rb_root rq_pos_tree; }; #endif /* --------------- main algorithm interface ----------------- */ #define BFQ_SERVICE_TREE_INIT … extern const int bfq_timeout; struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync, unsigned int actuator_idx); void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync, unsigned int actuator_idx); struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic); void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_weights_tree_add(struct bfq_queue *bfqq); void bfq_weights_tree_remove(struct bfq_queue *bfqq); void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, bool compensate, enum bfqq_expiration reason); void bfq_put_queue(struct bfq_queue *bfqq); void bfq_put_cooperator(struct bfq_queue *bfqq); void bfq_end_wr_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_schedule_dispatch(struct bfq_data *bfqd); void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); /* ------------ end of main algorithm interface -------------- */ /* ---------------- cgroups-support interface ---------------- */ void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, u64 io_start_time_ns, blk_opf_t opf); void bfqg_stats_update_dequeue(struct bfq_group *bfqg); void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct bfq_group *bfqg); #ifdef CONFIG_BFQ_CGROUP_DEBUG void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, blk_opf_t opf); void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); void bfqg_stats_update_idle_time(struct bfq_group *bfqg); void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); #endif void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); void bfq_end_wr_async(struct bfq_data *bfqd); struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio); struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); struct bfq_group *bfqq_group(struct bfq_queue *bfqq); struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node); void bfqg_and_blkg_put(struct bfq_group *bfqg); #ifdef CONFIG_BFQ_GROUP_IOSCHED extern struct cftype bfq_blkcg_legacy_files[]; extern struct cftype bfq_blkg_files[]; extern struct blkcg_policy blkcg_policy_bfq; #endif /* ------------- end of cgroups-support interface ------------- */ /* - interface of the internal hierarchical B-WF2Q+ scheduler - */ #ifdef CONFIG_BFQ_GROUP_IOSCHED /* both next loops stop at one of the child entities of the root group */ #define for_each_entity(entity) … /* * For each iteration, compute parent in advance, so as to be safe if * entity is deallocated during the iteration. Such a deallocation may * happen as a consequence of a bfq_put_queue that frees the bfq_queue * containing entity. */ #define for_each_entity_safe(entity, parent) … #else /* CONFIG_BFQ_GROUP_IOSCHED */ /* * Next two macros are fake loops when cgroups support is not * enabled. I fact, in such a case, there is only one level to go up * (to reach the root group). */ #define for_each_entity … #define for_each_entity_safe … #endif /* CONFIG_BFQ_GROUP_IOSCHED */ struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity); unsigned int bfq_tot_busy_queues(struct bfq_data *bfqd); struct bfq_service_tree *bfq_entity_service_tree(struct bfq_entity *entity); struct bfq_entity *bfq_entity_of(struct rb_node *node); unsigned short bfq_ioprio_to_weight(int ioprio); void bfq_put_idle_entity(struct bfq_service_tree *st, struct bfq_entity *entity); struct bfq_service_tree * __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, struct bfq_entity *entity, bool update_class_too); void bfq_bfqq_served(struct bfq_queue *bfqq, int served); void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, unsigned long time_ms); bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree); bool next_queue_may_preempt(struct bfq_data *bfqd); struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd); bool __bfq_bfqd_reset_in_service(struct bfq_data *bfqd); void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bool ins_into_idle_tree, bool expiration); void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bool expiration); void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); void bfq_add_bfqq_busy(struct bfq_queue *bfqq); void bfq_add_bfqq_in_groups_with_pending_reqs(struct bfq_queue *bfqq); void bfq_del_bfqq_in_groups_with_pending_reqs(struct bfq_queue *bfqq); /* --------------- end of interface of B-WF2Q+ ---------------- */ /* Logging facilities. */ static inline void bfq_bfqq_name(struct bfq_queue *bfqq, char *str, int len) { … } #ifdef CONFIG_BFQ_GROUP_IOSCHED struct bfq_group *bfqq_group(struct bfq_queue *bfqq); #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) … #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) … #else /* CONFIG_BFQ_GROUP_IOSCHED */ #define bfq_log_bfqq … #define bfq_log_bfqg … #endif /* CONFIG_BFQ_GROUP_IOSCHED */ #define bfq_log(bfqd, fmt, args...) … #endif /* _BFQ_H */
Codebase Browser
linux