/* SPDX-License-Identifier: GPL-2.0 */ /* * Scheduler internal types and methods: */ #ifndef _KERNEL_SCHED_SCHED_H #define _KERNEL_SCHED_SCHED_H #include <linux/sched/affinity.h> #include <linux/sched/autogroup.h> #include <linux/sched/cpufreq.h> #include <linux/sched/deadline.h> #include <linux/sched.h> #include <linux/sched/loadavg.h> #include <linux/sched/mm.h> #include <linux/sched/rseq_api.h> #include <linux/sched/signal.h> #include <linux/sched/smt.h> #include <linux/sched/stat.h> #include <linux/sched/sysctl.h> #include <linux/sched/task_flags.h> #include <linux/sched/task.h> #include <linux/sched/topology.h> #include <linux/atomic.h> #include <linux/bitmap.h> #include <linux/bug.h> #include <linux/capability.h> #include <linux/cgroup_api.h> #include <linux/cgroup.h> #include <linux/context_tracking.h> #include <linux/cpufreq.h> #include <linux/cpumask_api.h> #include <linux/ctype.h> #include <linux/file.h> #include <linux/fs_api.h> #include <linux/hrtimer_api.h> #include <linux/interrupt.h> #include <linux/irq_work.h> #include <linux/jiffies.h> #include <linux/kref_api.h> #include <linux/kthread.h> #include <linux/ktime_api.h> #include <linux/lockdep_api.h> #include <linux/lockdep.h> #include <linux/minmax.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/mutex_api.h> #include <linux/plist.h> #include <linux/poll.h> #include <linux/proc_fs.h> #include <linux/profile.h> #include <linux/psi.h> #include <linux/rcupdate.h> #include <linux/seq_file.h> #include <linux/seqlock.h> #include <linux/softirq.h> #include <linux/spinlock_api.h> #include <linux/static_key.h> #include <linux/stop_machine.h> #include <linux/syscalls_api.h> #include <linux/syscalls.h> #include <linux/tick.h> #include <linux/topology.h> #include <linux/types.h> #include <linux/u64_stats_sync_api.h> #include <linux/uaccess.h> #include <linux/wait_api.h> #include <linux/wait_bit.h> #include <linux/workqueue_api.h> #include <trace/events/power.h> #include <trace/events/sched.h> #include "../workqueue_internal.h" struct rq; struct cfs_rq; struct rt_rq; struct sched_group; struct cpuidle_state; #ifdef CONFIG_PARAVIRT # include <asm/paravirt.h> # include <asm/paravirt_api_clock.h> #endif #include <asm/barrier.h> #include "cpupri.h" #include "cpudeadline.h" #ifdef CONFIG_SCHED_DEBUG #define SCHED_WARN_ON(x) … #else #define SCHED_WARN_ON … #endif /* task_struct::on_rq states: */ #define TASK_ON_RQ_QUEUED … #define TASK_ON_RQ_MIGRATING … extern __read_mostly int scheduler_running; extern unsigned long calc_load_update; extern atomic_long_t calc_load_tasks; extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); extern void call_trace_sched_update_nr_running(struct rq *rq, int count); extern int sysctl_sched_rt_period; extern int sysctl_sched_rt_runtime; extern int sched_rr_timeslice; /* * Asymmetric CPU capacity bits */ struct asym_cap_data { … }; extern struct list_head asym_cap_list; #define cpu_capacity_span(asym_data) … /* * Helpers for converting nanosecond timing to jiffy resolution */ #define NS_TO_JIFFIES(time) … /* * Increase resolution of nice-level calculations for 64-bit architectures. * The extra resolution improves shares distribution and load balancing of * low-weight task groups (eg. nice +19 on an autogroup), deeper task-group * hierarchies, especially on larger systems. This is not a user-visible change * and does not change the user-interface for setting shares/weights. * * We increase resolution only if we have enough bits to allow this increased * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit * are pretty high and the returns do not justify the increased costs. * * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to * increase coverage and consistency always enable it on 64-bit platforms. */ #ifdef CONFIG_64BIT #define NICE_0_LOAD_SHIFT … #define scale_load(w) … #define scale_load_down(w) … #else #define NICE_0_LOAD_SHIFT … #define scale_load … #define scale_load_down … #endif /* * Task weight (visible to users) and its load (invisible to users) have * independent resolution, but they should be well calibrated. We use * scale_load() and scale_load_down(w) to convert between them. The * following must be true: * * scale_load(sched_prio_to_weight[NICE_TO_PRIO(0)-MAX_RT_PRIO]) == NICE_0_LOAD * */ #define NICE_0_LOAD … /* * Single value that decides SCHED_DEADLINE internal math precision. * 10 -> just above 1us * 9 -> just above 0.5us */ #define DL_SCALE … /* * Single value that denotes runtime == period, ie unlimited time. */ #define RUNTIME_INF … static inline int idle_policy(int policy) { … } static inline int fair_policy(int policy) { … } static inline int rt_policy(int policy) { … } static inline int dl_policy(int policy) { … } static inline bool valid_policy(int policy) { … } static inline int task_has_idle_policy(struct task_struct *p) { … } static inline int task_has_rt_policy(struct task_struct *p) { … } static inline int task_has_dl_policy(struct task_struct *p) { … } #define cap_scale(v, s) … static inline void update_avg(u64 *avg, u64 sample) { … } /* * Shifting a value by an exponent greater *or equal* to the size of said value * is UB; cap at size-1. */ #define shr_bound(val, shift) … /* * !! For sched_setattr_nocheck() (kernel) only !! * * This is actually gross. :( * * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE * tasks, but still be able to sleep. We need this on platforms that cannot * atomically change clock frequency. Remove once fast switching will be * available on such platforms. * * SUGOV stands for SchedUtil GOVernor. */ #define SCHED_FLAG_SUGOV … #define SCHED_DL_FLAGS … static inline bool dl_entity_is_special(const struct sched_dl_entity *dl_se) { … } /* * Tells if entity @a should preempt entity @b. */ static inline bool dl_entity_preempt(const struct sched_dl_entity *a, const struct sched_dl_entity *b) { … } /* * This is the priority-queue data structure of the RT scheduling class: */ struct rt_prio_array { … }; struct rt_bandwidth { … }; static inline int dl_bandwidth_enabled(void) { … } /* * To keep the bandwidth of -deadline tasks under control * we need some place where: * - store the maximum -deadline bandwidth of each cpu; * - cache the fraction of bandwidth that is currently allocated in * each root domain; * * This is all done in the data structure below. It is similar to the * one used for RT-throttling (rt_bandwidth), with the main difference * that, since here we are only interested in admission control, we * do not decrease any runtime while the group "executes", neither we * need a timer to replenish it. * * With respect to SMP, bandwidth is given on a per root domain basis, * meaning that: * - bw (< 100%) is the deadline bandwidth of each CPU; * - total_bw is the currently allocated bandwidth in each root domain; */ struct dl_bw { … }; extern void init_dl_bw(struct dl_bw *dl_b); extern int sched_dl_global_validate(void); extern void sched_dl_do_global(void); extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr); extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr); extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr); extern bool __checkparam_dl(const struct sched_attr *attr); extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr); extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); extern int dl_bw_check_overflow(int cpu); /* * SCHED_DEADLINE supports servers (nested scheduling) with the following * interface: * * dl_se::rq -- runqueue we belong to. * * dl_se::server_has_tasks() -- used on bandwidth enforcement; we 'stop' the * server when it runs out of tasks to run. * * dl_se::server_pick() -- nested pick_next_task(); we yield the period if this * returns NULL. * * dl_server_update() -- called from update_curr_common(), propagates runtime * to the server. * * dl_server_start() * dl_server_stop() -- start/stop the server when it has (no) tasks. * * dl_server_init() -- initializes the server. */ extern void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec); extern void dl_server_start(struct sched_dl_entity *dl_se); extern void dl_server_stop(struct sched_dl_entity *dl_se); extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, dl_server_has_tasks_f has_tasks, dl_server_pick_f pick); #ifdef CONFIG_CGROUP_SCHED extern struct list_head task_groups; struct cfs_bandwidth { … }; /* Task group related information */ struct task_group { … }; #ifdef CONFIG_FAIR_GROUP_SCHED #define ROOT_TASK_GROUP_LOAD … /* * A weight of 0 or 1 can cause arithmetics problems. * A weight of a cfs_rq is the sum of weights of which entities * are queued on this cfs_rq, so a weight of a entity should not be * too large, so as the shares value of a task group. * (The default weight is 1024 - so there's no practical * limitation from this.) */ #define MIN_SHARES … #define MAX_SHARES … #endif tg_visitor; extern int walk_tg_tree_from(struct task_group *from, tg_visitor down, tg_visitor up, void *data); /* * Iterate the full tree, calling @down when first entering a node and @up when * leaving it for the final time. * * Caller must hold rcu_lock or sufficient equivalent. */ static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) { … } extern int tg_nop(struct task_group *tg, void *data); #ifdef CONFIG_FAIR_GROUP_SCHED extern void free_fair_sched_group(struct task_group *tg); extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); #else static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { return 1; } static inline void online_fair_sched_group(struct task_group *tg) { } static inline void unregister_fair_sched_group(struct task_group *tg) { } #endif extern void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, struct sched_entity *se, int cpu, struct sched_entity *parent); extern void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent); extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b); extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b); extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq); extern bool cfs_task_bw_constrained(struct task_struct *p); extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int cpu, struct sched_rt_entity *parent); extern int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us); extern int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us); extern long sched_group_rt_runtime(struct task_group *tg); extern long sched_group_rt_period(struct task_group *tg); extern int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk); extern struct task_group *sched_create_group(struct task_group *parent); extern void sched_online_group(struct task_group *tg, struct task_group *parent); extern void sched_destroy_group(struct task_group *tg); extern void sched_release_group(struct task_group *tg); extern void sched_move_task(struct task_struct *tsk); #ifdef CONFIG_FAIR_GROUP_SCHED extern int sched_group_set_shares(struct task_group *tg, unsigned long shares); extern int sched_group_set_idle(struct task_group *tg, long idle); #ifdef CONFIG_SMP extern void set_task_rq_fair(struct sched_entity *se, struct cfs_rq *prev, struct cfs_rq *next); #else /* !CONFIG_SMP */ static inline void set_task_rq_fair(struct sched_entity *se, struct cfs_rq *prev, struct cfs_rq *next) { } #endif /* CONFIG_SMP */ #endif /* CONFIG_FAIR_GROUP_SCHED */ #else /* CONFIG_CGROUP_SCHED */ struct cfs_bandwidth { }; static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false; } #endif /* CONFIG_CGROUP_SCHED */ extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); /* * u64_u32_load/u64_u32_store * * Use a copy of a u64 value to protect against data race. This is only * applicable for 32-bits architectures. */ #ifdef CONFIG_64BIT #define u64_u32_load_copy(var, copy) … #define u64_u32_store_copy(var, copy, val) … #else #define u64_u32_load_copy … #define u64_u32_store_copy … #endif #define u64_u32_load(var) … #define u64_u32_store(var, val) … /* CFS-related fields in a runqueue */ struct cfs_rq { … }; static inline int rt_bandwidth_enabled(void) { … } /* RT IPI pull logic requires IRQ_WORK */ #if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP) #define HAVE_RT_PUSH_IPI #endif /* Real-Time classes' related field in a runqueue: */ struct rt_rq { … }; static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq) { … } /* Deadline class' related fields in a runqueue */ struct dl_rq { … }; #ifdef CONFIG_FAIR_GROUP_SCHED /* An entity is a task if it doesn't "own" a runqueue */ #define entity_is_task(se) … static inline void se_update_runnable(struct sched_entity *se) { … } static inline long se_runnable(struct sched_entity *se) { … } #else /* !CONFIG_FAIR_GROUP_SCHED: */ #define entity_is_task … static inline void se_update_runnable(struct sched_entity *se) { } static inline long se_runnable(struct sched_entity *se) { return !!se->on_rq; } #endif /* !CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_SMP /* * XXX we want to get rid of these helpers and use the full load resolution. */ static inline long se_weight(struct sched_entity *se) { … } static inline bool sched_asym_prefer(int a, int b) { … } struct perf_domain { … }; /* * We add the notion of a root-domain which will be used to define per-domain * variables. Each exclusive cpuset essentially defines an island domain by * fully partitioning the member CPUs from any other cpuset. Whenever a new * exclusive cpuset is created, we also create and attach a new root-domain * object. * */ struct root_domain { … }; extern void init_defrootdomain(void); extern int sched_init_domains(const struct cpumask *cpu_map); extern void rq_attach_root(struct rq *rq, struct root_domain *rd); extern void sched_get_rd(struct root_domain *rd); extern void sched_put_rd(struct root_domain *rd); static inline int get_rd_overloaded(struct root_domain *rd) { … } static inline void set_rd_overloaded(struct root_domain *rd, int status) { … } #ifdef HAVE_RT_PUSH_IPI extern void rto_push_irq_work_func(struct irq_work *work); #endif #endif /* CONFIG_SMP */ #ifdef CONFIG_UCLAMP_TASK /* * struct uclamp_bucket - Utilization clamp bucket * @value: utilization clamp value for tasks on this clamp bucket * @tasks: number of RUNNABLE tasks on this clamp bucket * * Keep track of how many tasks are RUNNABLE for a given utilization * clamp value. */ struct uclamp_bucket { … }; /* * struct uclamp_rq - rq's utilization clamp * @value: currently active clamp values for a rq * @bucket: utilization clamp buckets affecting a rq * * Keep track of RUNNABLE tasks on a rq to aggregate their clamp values. * A clamp value is affecting a rq when there is at least one task RUNNABLE * (or actually running) with that value. * * There are up to UCLAMP_CNT possible different clamp values, currently there * are only two: minimum utilization and maximum utilization. * * All utilization clamping values are MAX aggregated, since: * - for util_min: we want to run the CPU at least at the max of the minimum * utilization required by its currently RUNNABLE tasks. * - for util_max: we want to allow the CPU to run up to the max of the * maximum utilization allowed by its currently RUNNABLE tasks. * * Since on each system we expect only a limited number of different * utilization clamp values (UCLAMP_BUCKETS), use a simple array to track * the metrics required to compute all the per-rq utilization clamp values. */ struct uclamp_rq { … }; DECLARE_STATIC_KEY_FALSE(sched_uclamp_used); #endif /* CONFIG_UCLAMP_TASK */ struct balance_callback { … }; /* * This is the main, per-CPU runqueue data structure. * * Locking rule: those places that want to lock multiple runqueues * (such as the load balancing or the thread migration code), lock * acquire operations must be ordered by ascending &runqueue. */ struct rq { … }; #ifdef CONFIG_FAIR_GROUP_SCHED /* CPU runqueue to which this cfs_rq is attached */ static inline struct rq *rq_of(struct cfs_rq *cfs_rq) { … } #else static inline struct rq *rq_of(struct cfs_rq *cfs_rq) { return container_of(cfs_rq, struct rq, cfs); } #endif static inline int cpu_of(struct rq *rq) { … } #define MDF_PUSH … static inline bool is_migration_disabled(struct task_struct *p) { … } DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #define cpu_rq(cpu) … #define this_rq() … #define task_rq(p) … #define cpu_curr(cpu) … #define raw_rq() … #ifdef CONFIG_SCHED_CORE static inline struct cpumask *sched_group_span(struct sched_group *sg); DECLARE_STATIC_KEY_FALSE(__sched_core_enabled); static inline bool sched_core_enabled(struct rq *rq) { … } static inline bool sched_core_disabled(void) { … } /* * Be careful with this function; not for general use. The return value isn't * stable unless you actually hold a relevant rq->__lock. */ static inline raw_spinlock_t *rq_lockp(struct rq *rq) { … } static inline raw_spinlock_t *__rq_lockp(struct rq *rq) { … } extern bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, bool fi); extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); /* * Helpers to check if the CPU's core cookie matches with the task's cookie * when core scheduling is enabled. * A special case is that the task's cookie always matches with CPU's core * cookie if the CPU is in an idle core. */ static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p) { … } static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p) { … } static inline bool sched_group_cookie_match(struct rq *rq, struct task_struct *p, struct sched_group *group) { … } static inline bool sched_core_enqueued(struct task_struct *p) { … } extern void sched_core_enqueue(struct rq *rq, struct task_struct *p); extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags); extern void sched_core_get(void); extern void sched_core_put(void); #else /* !CONFIG_SCHED_CORE: */ static inline bool sched_core_enabled(struct rq *rq) { return false; } static inline bool sched_core_disabled(void) { return true; } static inline raw_spinlock_t *rq_lockp(struct rq *rq) { return &rq->__lock; } static inline raw_spinlock_t *__rq_lockp(struct rq *rq) { return &rq->__lock; } static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p) { return true; } static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p) { return true; } static inline bool sched_group_cookie_match(struct rq *rq, struct task_struct *p, struct sched_group *group) { return true; } #endif /* !CONFIG_SCHED_CORE */ static inline void lockdep_assert_rq_held(struct rq *rq) { … } extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass); extern bool raw_spin_rq_trylock(struct rq *rq); extern void raw_spin_rq_unlock(struct rq *rq); static inline void raw_spin_rq_lock(struct rq *rq) { … } static inline void raw_spin_rq_lock_irq(struct rq *rq) { … } static inline void raw_spin_rq_unlock_irq(struct rq *rq) { … } static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) { … } static inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags) { … } #define raw_spin_rq_lock_irqsave(rq, flags) … #ifdef CONFIG_SCHED_SMT extern void __update_idle_core(struct rq *rq); static inline void update_idle_core(struct rq *rq) { … } #else static inline void update_idle_core(struct rq *rq) { } #endif #ifdef CONFIG_FAIR_GROUP_SCHED static inline struct task_struct *task_of(struct sched_entity *se) { … } static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) { … } /* runqueue on which this entity is (to be) queued */ static inline struct cfs_rq *cfs_rq_of(const struct sched_entity *se) { … } /* runqueue "owned" by this group */ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) { … } #else /* !CONFIG_FAIR_GROUP_SCHED: */ #define task_of … static inline struct cfs_rq *task_cfs_rq(const struct task_struct *p) { return &task_rq(p)->cfs; } static inline struct cfs_rq *cfs_rq_of(const struct sched_entity *se) { const struct task_struct *p = task_of(se); struct rq *rq = task_rq(p); return &rq->cfs; } /* runqueue "owned" by this group */ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) { return NULL; } #endif /* !CONFIG_FAIR_GROUP_SCHED */ extern void update_rq_clock(struct rq *rq); /* * rq::clock_update_flags bits * * %RQCF_REQ_SKIP - will request skipping of clock update on the next * call to __schedule(). This is an optimisation to avoid * neighbouring rq clock updates. * * %RQCF_ACT_SKIP - is set from inside of __schedule() when skipping is * in effect and calls to update_rq_clock() are being ignored. * * %RQCF_UPDATED - is a debug flag that indicates whether a call has been * made to update_rq_clock() since the last time rq::lock was pinned. * * If inside of __schedule(), clock_update_flags will have been * shifted left (a left shift is a cheap operation for the fast path * to promote %RQCF_REQ_SKIP to %RQCF_ACT_SKIP), so you must use, * * if (rq-clock_update_flags >= RQCF_UPDATED) * * to check if %RQCF_UPDATED is set. It'll never be shifted more than * one position though, because the next rq_unpin_lock() will shift it * back. */ #define RQCF_REQ_SKIP … #define RQCF_ACT_SKIP … #define RQCF_UPDATED … static inline void assert_clock_updated(struct rq *rq) { … } static inline u64 rq_clock(struct rq *rq) { … } static inline u64 rq_clock_task(struct rq *rq) { … } static inline void rq_clock_skip_update(struct rq *rq) { … } /* * See rt task throttling, which is the only time a skip * request is canceled. */ static inline void rq_clock_cancel_skipupdate(struct rq *rq) { … } /* * During cpu offlining and rq wide unthrottling, we can trigger * an update_rq_clock() for several cfs and rt runqueues (Typically * when using list_for_each_entry_*) * rq_clock_start_loop_update() can be called after updating the clock * once and before iterating over the list to prevent multiple update. * After the iterative traversal, we need to call rq_clock_stop_loop_update() * to clear RQCF_ACT_SKIP of rq->clock_update_flags. */ static inline void rq_clock_start_loop_update(struct rq *rq) { … } static inline void rq_clock_stop_loop_update(struct rq *rq) { … } struct rq_flags { … }; extern struct balance_callback balance_push_callback; /* * Lockdep annotation that avoids accidental unlocks; it's like a * sticky/continuous lockdep_assert_held(). * * This avoids code that has access to 'struct rq *rq' (basically everything in * the scheduler) from accidentally unlocking the rq if they do not also have a * copy of the (on-stack) 'struct rq_flags rf'. * * Also see Documentation/locking/lockdep-design.rst. */ static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf) { … } static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf) { … } static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf) { … } extern struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) __acquires(…); extern struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) __acquires(…) __acquires(…); static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf) __releases(rq->lock) { … } static inline void task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) __releases(rq->lock) __releases(p->pi_lock) { … } DEFINE_LOCK_GUARD_1(task_rq_lock, struct task_struct, _T->rq = … static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) __acquires(rq->lock) { … } static inline void rq_lock_irq(struct rq *rq, struct rq_flags *rf) __acquires(rq->lock) { … } static inline void rq_lock(struct rq *rq, struct rq_flags *rf) __acquires(rq->lock) { … } static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) __releases(rq->lock) { … } static inline void rq_unlock_irq(struct rq *rq, struct rq_flags *rf) __releases(rq->lock) { … } static inline void rq_unlock(struct rq *rq, struct rq_flags *rf) __releases(rq->lock) { … } DEFINE_LOCK_GUARD_1(rq_lock, struct rq, rq_lock(_T->lock, &_T->rf), rq_unlock(_T->lock, &_T->rf), … } DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, rq_lock_irq(_T->lock, &_T->rf), rq_unlock_irq(_T->lock, &_T->rf), … } DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, rq_lock_irqsave(_T->lock, &_T->rf), rq_unlock_irqrestore(_T->lock, &_T->rf), … } static inline struct rq *this_rq_lock_irq(struct rq_flags *rf) __acquires(rq->lock) { … } #ifdef CONFIG_NUMA enum numa_topology_type { … }; extern enum numa_topology_type sched_numa_topology_type; extern int sched_max_numa_distance; extern bool find_numa_distance(int distance); extern void sched_init_numa(int offline_node); extern void sched_update_numa(int cpu, bool online); extern void sched_domains_numa_masks_set(unsigned int cpu); extern void sched_domains_numa_masks_clear(unsigned int cpu); extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu); #else /* !CONFIG_NUMA: */ static inline void sched_init_numa(int offline_node) { } static inline void sched_update_numa(int cpu, bool online) { } static inline void sched_domains_numa_masks_set(unsigned int cpu) { } static inline void sched_domains_numa_masks_clear(unsigned int cpu) { } static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu) { return nr_cpu_ids; } #endif /* !CONFIG_NUMA */ #ifdef CONFIG_NUMA_BALANCING /* The regions in numa_faults array from task_struct */ enum numa_faults_stats { … }; extern void sched_setnuma(struct task_struct *p, int node); extern int migrate_task_to(struct task_struct *p, int cpu); extern int migrate_swap(struct task_struct *p, struct task_struct *t, int cpu, int scpu); extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p); #else /* !CONFIG_NUMA_BALANCING: */ static inline void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) { } #endif /* !CONFIG_NUMA_BALANCING */ #ifdef CONFIG_SMP static inline void queue_balance_callback(struct rq *rq, struct balance_callback *head, void (*func)(struct rq *rq)) { … } #define rcu_dereference_check_sched_domain(p) … /* * The domain tree (rq->sd) is protected by RCU's quiescent state transition. * See destroy_sched_domains: call_rcu for details. * * The domain tree of any CPU may only be accessed from within * preempt-disabled sections. */ #define for_each_domain(cpu, __sd) … /* A mask of all the SD flags that have the SDF_SHARED_CHILD metaflag */ #define SD_FLAG … static const unsigned int SD_SHARED_CHILD_MASK = …; #undef SD_FLAG /** * highest_flag_domain - Return highest sched_domain containing flag. * @cpu: The CPU whose highest level of sched domain is to * be returned. * @flag: The flag to check for the highest sched_domain * for the given CPU. * * Returns the highest sched_domain of a CPU which contains @flag. If @flag has * the SDF_SHARED_CHILD metaflag, all the children domains also have @flag. */ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) { … } static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) { … } DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); DECLARE_PER_CPU(int, sd_share_id); DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); extern struct static_key_false sched_asym_cpucapacity; extern struct static_key_false sched_cluster_active; static __always_inline bool sched_asym_cpucap_active(void) { … } struct sched_group_capacity { … }; struct sched_group { … }; static inline struct cpumask *sched_group_span(struct sched_group *sg) { … } /* * See build_balance_mask(). */ static inline struct cpumask *group_balance_mask(struct sched_group *sg) { … } extern int group_balance_cpu(struct sched_group *sg); #ifdef CONFIG_SCHED_DEBUG extern void update_sched_domain_debugfs(void); extern void dirty_sched_domain_sysctl(int cpu); #else static inline void update_sched_domain_debugfs(void) { } static inline void dirty_sched_domain_sysctl(int cpu) { } #endif extern int sched_update_scaling(void); static inline const struct cpumask *task_user_cpus(struct task_struct *p) { … } #endif /* CONFIG_SMP */ #include "stats.h" #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) extern void __sched_core_account_forceidle(struct rq *rq); static inline void sched_core_account_forceidle(struct rq *rq) { … } extern void __sched_core_tick(struct rq *rq); static inline void sched_core_tick(struct rq *rq) { … } #else /* !(CONFIG_SCHED_CORE && CONFIG_SCHEDSTATS): */ static inline void sched_core_account_forceidle(struct rq *rq) { } static inline void sched_core_tick(struct rq *rq) { } #endif /* !(CONFIG_SCHED_CORE && CONFIG_SCHEDSTATS) */ #ifdef CONFIG_CGROUP_SCHED /* * Return the group to which this tasks belongs. * * We cannot use task_css() and friends because the cgroup subsystem * changes that value before the cgroup_subsys::attach() method is called, * therefore we cannot pin it and might observe the wrong value. * * The same is true for autogroup's p->signal->autogroup->tg, the autogroup * core changes this before calling sched_move_task(). * * Instead we use a 'copy' which is updated from sched_move_task() while * holding both task_struct::pi_lock and rq::lock. */ static inline struct task_group *task_group(struct task_struct *p) { … } /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { … } #else /* !CONFIG_CGROUP_SCHED: */ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } static inline struct task_group *task_group(struct task_struct *p) { return NULL; } #endif /* !CONFIG_CGROUP_SCHED */ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { … } /* * Tunables that become constants when CONFIG_SCHED_DEBUG is off: */ #ifdef CONFIG_SCHED_DEBUG #define const_debug … #else #define const_debug … #endif #define SCHED_FEAT … enum { … }; #undef SCHED_FEAT #ifdef CONFIG_SCHED_DEBUG /* * To support run-time toggling of sched features, all the translation units * (but core.c) reference the sysctl_sched_features defined in core.c. */ extern const_debug unsigned int sysctl_sched_features; #ifdef CONFIG_JUMP_LABEL #define SCHED_FEAT … #include "features.h" #undef SCHED_FEAT extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; #define sched_feat(x) … #else /* !CONFIG_JUMP_LABEL: */ #define sched_feat … #endif /* !CONFIG_JUMP_LABEL */ #else /* !SCHED_DEBUG: */ /* * Each translation unit has its own copy of sysctl_sched_features to allow * constants propagation at compile time and compiler optimization based on * features default. */ #define SCHED_FEAT … static const_debug __maybe_unused unsigned int sysctl_sched_features = #include "features.h" 0; #undef SCHED_FEAT #define sched_feat … #endif /* !SCHED_DEBUG */ extern struct static_key_false sched_numa_balancing; extern struct static_key_false sched_schedstats; static inline u64 global_rt_period(void) { … } static inline u64 global_rt_runtime(void) { … } static inline int task_current(struct rq *rq, struct task_struct *p) { … } static inline int task_on_cpu(struct rq *rq, struct task_struct *p) { … } static inline int task_on_rq_queued(struct task_struct *p) { … } static inline int task_on_rq_migrating(struct task_struct *p) { … } /* Wake flags. The first three directly map to some SD flag value */ #define WF_EXEC … #define WF_FORK … #define WF_TTWU … #define WF_SYNC … #define WF_MIGRATED … #define WF_CURRENT_CPU … #ifdef CONFIG_SMP static_assert(…); static_assert(…); static_assert(…); #endif /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that * each task makes to its run queue's load is weighted according to its * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a * scaled version of the new time slice allocation that they receive on time * slice expiry etc. */ #define WEIGHT_IDLEPRIO … #define WMULT_IDLEPRIO … extern const int sched_prio_to_weight[40]; extern const u32 sched_prio_to_wmult[40]; /* * {de,en}queue flags: * * DEQUEUE_SLEEP - task is no longer runnable * ENQUEUE_WAKEUP - task just became runnable * * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks * are in a known state which allows modification. Such pairs * should preserve as much state as possible. * * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location * in the runqueue. * * NOCLOCK - skip the update_rq_clock() (avoids double updates) * * MIGRATION - p->on_rq == TASK_ON_RQ_MIGRATING (used for DEADLINE) * * ENQUEUE_HEAD - place at front of runqueue (tail if not specified) * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline) * ENQUEUE_MIGRATED - the task was migrated during wakeup * */ #define DEQUEUE_SLEEP … #define DEQUEUE_SAVE … #define DEQUEUE_MOVE … #define DEQUEUE_NOCLOCK … #define DEQUEUE_MIGRATING … #define ENQUEUE_WAKEUP … #define ENQUEUE_RESTORE … #define ENQUEUE_MOVE … #define ENQUEUE_NOCLOCK … #define ENQUEUE_HEAD … #define ENQUEUE_REPLENISH … #ifdef CONFIG_SMP #define ENQUEUE_MIGRATED … #else #define ENQUEUE_MIGRATED … #endif #define ENQUEUE_INITIAL … #define ENQUEUE_MIGRATING … #define RETRY_TASK … struct affinity_context { … }; extern s64 update_curr_common(struct rq *rq); struct sched_class { … }; static inline void put_prev_task(struct rq *rq, struct task_struct *prev) { … } static inline void set_next_task(struct rq *rq, struct task_struct *next) { … } /* * Helper to define a sched_class instance; each one is placed in a separate * section which is ordered by the linker script: * * include/asm-generic/vmlinux.lds.h * * *CAREFUL* they are laid out in *REVERSE* order!!! * * Also enforce alignment on the instance, not the type, to guarantee layout. */ #define DEFINE_SCHED_CLASS(name) … /* Defined in include/asm-generic/vmlinux.lds.h */ extern struct sched_class __sched_class_highest[]; extern struct sched_class __sched_class_lowest[]; #define for_class_range(class, _from, _to) … #define for_each_class(class) … #define sched_class_above(_a, _b) … extern const struct sched_class stop_sched_class; extern const struct sched_class dl_sched_class; extern const struct sched_class rt_sched_class; extern const struct sched_class fair_sched_class; extern const struct sched_class idle_sched_class; static inline bool sched_stop_runnable(struct rq *rq) { … } static inline bool sched_dl_runnable(struct rq *rq) { … } static inline bool sched_rt_runnable(struct rq *rq) { … } static inline bool sched_fair_runnable(struct rq *rq) { … } extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); extern struct task_struct *pick_next_task_idle(struct rq *rq); #define SCA_CHECK … #define SCA_MIGRATE_DISABLE … #define SCA_MIGRATE_ENABLE … #define SCA_USER … #ifdef CONFIG_SMP extern void update_group_capacity(struct sched_domain *sd, int cpu); extern void sched_balance_trigger(struct rq *rq); extern int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx); extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx); static inline cpumask_t *alloc_user_cpus_ptr(int node) { … } static inline struct task_struct *get_push_task(struct rq *rq) { … } extern int push_cpu_stop(void *arg); #else /* !CONFIG_SMP: */ static inline int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx) { return set_cpus_allowed_ptr(p, ctx->new_mask); } static inline cpumask_t *alloc_user_cpus_ptr(int node) { return NULL; } #endif /* !CONFIG_SMP */ #ifdef CONFIG_CPU_IDLE static inline void idle_set_state(struct rq *rq, struct cpuidle_state *idle_state) { … } static inline struct cpuidle_state *idle_get_state(struct rq *rq) { … } #else /* !CONFIG_CPU_IDLE: */ static inline void idle_set_state(struct rq *rq, struct cpuidle_state *idle_state) { } static inline struct cpuidle_state *idle_get_state(struct rq *rq) { return NULL; } #endif /* !CONFIG_CPU_IDLE */ extern void schedule_idle(void); asmlinkage void schedule_user(void); extern void sysrq_sched_debug_show(void); extern void sched_init_granularity(void); extern void update_max_interval(void); extern void init_sched_dl_class(void); extern void init_sched_rt_class(void); extern void init_sched_fair_class(void); extern void reweight_task(struct task_struct *p, const struct load_weight *lw); extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); extern struct rt_bandwidth def_rt_bandwidth; extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq); extern void init_dl_entity(struct sched_dl_entity *dl_se); #define BW_SHIFT … #define BW_UNIT … #define RATIO_SHIFT … #define MAX_BW_BITS … #define MAX_BW … extern unsigned long to_ratio(u64 period, u64 runtime); extern void init_entity_runnable_average(struct sched_entity *se); extern void post_init_entity_util_avg(struct task_struct *p); #ifdef CONFIG_NO_HZ_FULL extern bool sched_can_stop_tick(struct rq *rq); extern int __init sched_tick_offload_init(void); /* * Tick may be needed by tasks in the runqueue depending on their policy and * requirements. If tick is needed, lets send the target an IPI to kick it out of * nohz mode if necessary. */ static inline void sched_update_tick_dependency(struct rq *rq) { int cpu = cpu_of(rq); if (!tick_nohz_full_cpu(cpu)) return; if (sched_can_stop_tick(rq)) tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED); else tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); } #else /* !CONFIG_NO_HZ_FULL: */ static inline int sched_tick_offload_init(void) { … } static inline void sched_update_tick_dependency(struct rq *rq) { … } #endif /* !CONFIG_NO_HZ_FULL */ static inline void add_nr_running(struct rq *rq, unsigned count) { … } static inline void sub_nr_running(struct rq *rq, unsigned count) { … } extern void activate_task(struct rq *rq, struct task_struct *p, int flags); extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); extern void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags); #ifdef CONFIG_PREEMPT_RT #define SCHED_NR_MIGRATE_BREAK … #else #define SCHED_NR_MIGRATE_BREAK … #endif extern const_debug unsigned int sysctl_sched_nr_migrate; extern const_debug unsigned int sysctl_sched_migration_cost; extern unsigned int sysctl_sched_base_slice; #ifdef CONFIG_SCHED_DEBUG extern int sysctl_resched_latency_warn_ms; extern int sysctl_resched_latency_warn_once; extern unsigned int sysctl_sched_tunable_scaling; extern unsigned int sysctl_numa_balancing_scan_delay; extern unsigned int sysctl_numa_balancing_scan_period_min; extern unsigned int sysctl_numa_balancing_scan_period_max; extern unsigned int sysctl_numa_balancing_scan_size; extern unsigned int sysctl_numa_balancing_hot_threshold; #endif #ifdef CONFIG_SCHED_HRTICK /* * Use hrtick when: * - enabled by features * - hrtimer is actually high res */ static inline int hrtick_enabled(struct rq *rq) { … } static inline int hrtick_enabled_fair(struct rq *rq) { … } static inline int hrtick_enabled_dl(struct rq *rq) { … } extern void hrtick_start(struct rq *rq, u64 delay); #else /* !CONFIG_SCHED_HRTICK: */ static inline int hrtick_enabled_fair(struct rq *rq) { return 0; } static inline int hrtick_enabled_dl(struct rq *rq) { return 0; } static inline int hrtick_enabled(struct rq *rq) { return 0; } #endif /* !CONFIG_SCHED_HRTICK */ #ifndef arch_scale_freq_tick static __always_inline void arch_scale_freq_tick(void) { } #endif #ifndef arch_scale_freq_capacity /** * arch_scale_freq_capacity - get the frequency scale factor of a given CPU. * @cpu: the CPU in question. * * Return: the frequency scale factor normalized against SCHED_CAPACITY_SCALE, i.e. * * f_curr * ------ * SCHED_CAPACITY_SCALE * f_max */ static __always_inline unsigned long arch_scale_freq_capacity(int cpu) { return SCHED_CAPACITY_SCALE; } #endif #ifdef CONFIG_SCHED_DEBUG /* * In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to * acquire rq lock instead of rq_lock(). So at the end of these two functions * we need to call double_rq_clock_clear_update() to clear RQCF_UPDATED of * rq->clock_update_flags to avoid the WARN_DOUBLE_CLOCK warning. */ static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) { … } #else static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) { } #endif #define DEFINE_LOCK_GUARD_2(name, type, _lock, _unlock, ...) … #ifdef CONFIG_SMP static inline bool rq_order_less(struct rq *rq1, struct rq *rq2) { … } extern void double_rq_lock(struct rq *rq1, struct rq *rq2); #ifdef CONFIG_PREEMPTION /* * fair double_lock_balance: Safely acquires both rq->locks in a fair * way at the expense of forcing extra atomic operations in all * invocations. This assures that the double_lock is acquired using the * same underlying policy as the spinlock_t on this architecture, which * reduces latency compared to the unfair variant below. However, it * also adds more overhead and therefore may reduce throughput. */ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) __releases(this_rq->lock) __acquires(busiest->lock) __acquires(this_rq->lock) { … } #else /* !CONFIG_PREEMPTION: */ /* * Unfair double_lock_balance: Optimizes throughput at the expense of * latency by eliminating extra atomic operations when the locks are * already in proper order on entry. This favors lower CPU-ids and will * grant the double lock to lower CPUs over higher ids under contention, * regardless of entry order into the function. */ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) __releases(this_rq->lock) __acquires(busiest->lock) __acquires(this_rq->lock) { if (__rq_lockp(this_rq) == __rq_lockp(busiest) || likely(raw_spin_rq_trylock(busiest))) { double_rq_clock_clear_update(this_rq, busiest); return 0; } if (rq_order_less(this_rq, busiest)) { raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING); double_rq_clock_clear_update(this_rq, busiest); return 0; } raw_spin_rq_unlock(this_rq); double_rq_lock(this_rq, busiest); return 1; } #endif /* !CONFIG_PREEMPTION */ /* * double_lock_balance - lock the busiest runqueue, this_rq is locked already. */ static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest) { … } static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) __releases(busiest->lock) { … } static inline void double_lock(spinlock_t *l1, spinlock_t *l2) { … } static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) { … } static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) { … } static inline void double_raw_unlock(raw_spinlock_t *l1, raw_spinlock_t *l2) { … } DEFINE_LOCK_GUARD_2(double_raw_spinlock, raw_spinlock_t, double_raw_lock(_T->lock, _T->lock2), double_raw_unlock(_T->lock, _T->lock2)) /* * double_rq_unlock - safely unlock two runqueues * * Note this does not restore interrupts like task_rq_unlock, * you need to do so manually after calling. */ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) __releases(rq1->lock) __releases(rq2->lock) { … } extern void set_rq_online (struct rq *rq); extern void set_rq_offline(struct rq *rq); extern bool sched_smp_initialized; #else /* !CONFIG_SMP: */ /* * double_rq_lock - safely lock two runqueues * * Note this does not disable interrupts like task_rq_lock, * you need to do so manually before calling. */ static inline void double_rq_lock(struct rq *rq1, struct rq *rq2) __acquires(rq1->lock) __acquires(rq2->lock) { WARN_ON_ONCE(!irqs_disabled()); WARN_ON_ONCE(rq1 != rq2); raw_spin_rq_lock(rq1); __acquire(rq2->lock); /* Fake it out ;) */ double_rq_clock_clear_update(rq1, rq2); } /* * double_rq_unlock - safely unlock two runqueues * * Note this does not restore interrupts like task_rq_unlock, * you need to do so manually after calling. */ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) __releases(rq1->lock) __releases(rq2->lock) { WARN_ON_ONCE(rq1 != rq2); raw_spin_rq_unlock(rq1); __release(rq2->lock); } #endif /* !CONFIG_SMP */ DEFINE_LOCK_GUARD_2(double_rq_lock, … } extern struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq); extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq); extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq); #ifdef CONFIG_SCHED_DEBUG extern bool sched_debug_verbose; extern void print_cfs_stats(struct seq_file *m, int cpu); extern void print_rt_stats(struct seq_file *m, int cpu); extern void print_dl_stats(struct seq_file *m, int cpu); extern void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq); extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq); extern void resched_latency_warn(int cpu, u64 latency); # ifdef CONFIG_NUMA_BALANCING extern void show_numa_stats(struct task_struct *p, struct seq_file *m); extern void print_numa_stats(struct seq_file *m, int node, unsigned long tsf, unsigned long tpf, unsigned long gsf, unsigned long gpf); # endif /* CONFIG_NUMA_BALANCING */ #else /* !CONFIG_SCHED_DEBUG: */ static inline void resched_latency_warn(int cpu, u64 latency) { } #endif /* !CONFIG_SCHED_DEBUG */ extern void init_cfs_rq(struct cfs_rq *cfs_rq); extern void init_rt_rq(struct rt_rq *rt_rq); extern void init_dl_rq(struct dl_rq *dl_rq); extern void cfs_bandwidth_usage_inc(void); extern void cfs_bandwidth_usage_dec(void); #ifdef CONFIG_NO_HZ_COMMON #define NOHZ_BALANCE_KICK_BIT … #define NOHZ_STATS_KICK_BIT … #define NOHZ_NEWILB_KICK_BIT … #define NOHZ_NEXT_KICK_BIT … /* Run sched_balance_domains() */ #define NOHZ_BALANCE_KICK … /* Update blocked load */ #define NOHZ_STATS_KICK … /* Update blocked load when entering idle */ #define NOHZ_NEWILB_KICK … /* Update nohz.next_balance */ #define NOHZ_NEXT_KICK … #define NOHZ_KICK_MASK … #define nohz_flags(cpu) … extern void nohz_balance_exit_idle(struct rq *rq); #else /* !CONFIG_NO_HZ_COMMON: */ static inline void nohz_balance_exit_idle(struct rq *rq) { } #endif /* !CONFIG_NO_HZ_COMMON */ #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void nohz_run_idle_balance(int cpu); #else static inline void nohz_run_idle_balance(int cpu) { } #endif #ifdef CONFIG_IRQ_TIME_ACCOUNTING struct irqtime { … }; DECLARE_PER_CPU(struct irqtime, cpu_irqtime); /* * Returns the irqtime minus the softirq time computed by ksoftirqd. * Otherwise ksoftirqd's sum_exec_runtime is subtracted its own runtime * and never move forward. */ static inline u64 irq_time_read(int cpu) { … } #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ #ifdef CONFIG_CPU_FREQ DECLARE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data); /** * cpufreq_update_util - Take a note about CPU utilization changes. * @rq: Runqueue to carry out the update for. * @flags: Update reason flags. * * This function is called by the scheduler on the CPU whose utilization is * being updated. * * It can only be called from RCU-sched read-side critical sections. * * The way cpufreq is currently arranged requires it to evaluate the CPU * performance state (frequency/voltage) on a regular basis to prevent it from * being stuck in a completely inadequate performance level for too long. * That is not guaranteed to happen if the updates are only triggered from CFS * and DL, though, because they may not be coming in if only RT tasks are * active all the time (or there are RT tasks only). * * As a workaround for that issue, this function is called periodically by the * RT sched class to trigger extra cpufreq updates to prevent it from stalling, * but that really is a band-aid. Going forward it should be replaced with * solutions targeted more specifically at RT tasks. */ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) { … } #else /* !CONFIG_CPU_FREQ: */ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) { } #endif /* !CONFIG_CPU_FREQ */ #ifdef arch_scale_freq_capacity # ifndef arch_scale_freq_invariant #define arch_scale_freq_invariant … # endif #else #define arch_scale_freq_invariant … #endif #ifdef CONFIG_SMP unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, unsigned long *min, unsigned long *max); unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual, unsigned long min, unsigned long max); /* * Verify the fitness of task @p to run on @cpu taking into account the * CPU original capacity and the runtime/deadline ratio of the task. * * The function will return true if the original capacity of @cpu is * greater than or equal to task's deadline density right shifted by * (BW_SHIFT - SCHED_CAPACITY_SHIFT) and false otherwise. */ static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) { … } static inline unsigned long cpu_bw_dl(struct rq *rq) { … } static inline unsigned long cpu_util_dl(struct rq *rq) { … } extern unsigned long cpu_util_cfs(int cpu); extern unsigned long cpu_util_cfs_boost(int cpu); static inline unsigned long cpu_util_rt(struct rq *rq) { … } #endif /* CONFIG_SMP */ #ifdef CONFIG_UCLAMP_TASK unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id); static inline unsigned long uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id) { … } static inline void uclamp_rq_set(struct rq *rq, enum uclamp_id clamp_id, unsigned int value) { … } static inline bool uclamp_rq_is_idle(struct rq *rq) { … } /* Is the rq being capped/throttled by uclamp_max? */ static inline bool uclamp_rq_is_capped(struct rq *rq) { … } /* * When uclamp is compiled in, the aggregation at rq level is 'turned off' * by default in the fast path and only gets turned on once userspace performs * an operation that requires it. * * Returns true if userspace opted-in to use uclamp and aggregation at rq level * hence is active. */ static inline bool uclamp_is_used(void) { … } #define for_each_clamp_id(clamp_id) … extern unsigned int sysctl_sched_uclamp_util_min_rt_default; static inline unsigned int uclamp_none(enum uclamp_id clamp_id) { … } /* Integer rounded range for each bucket */ #define UCLAMP_BUCKET_DELTA … static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) { … } static inline void uclamp_se_set(struct uclamp_se *uc_se, unsigned int value, bool user_defined) { … } #else /* !CONFIG_UCLAMP_TASK: */ static inline unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) { if (clamp_id == UCLAMP_MIN) return 0; return SCHED_CAPACITY_SCALE; } static inline bool uclamp_rq_is_capped(struct rq *rq) { return false; } static inline bool uclamp_is_used(void) { return false; } static inline unsigned long uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id) { if (clamp_id == UCLAMP_MIN) return 0; return SCHED_CAPACITY_SCALE; } static inline void uclamp_rq_set(struct rq *rq, enum uclamp_id clamp_id, unsigned int value) { } static inline bool uclamp_rq_is_idle(struct rq *rq) { return false; } #endif /* !CONFIG_UCLAMP_TASK */ #ifdef CONFIG_HAVE_SCHED_AVG_IRQ static inline unsigned long cpu_util_irq(struct rq *rq) { … } static inline unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) { … } #else /* !CONFIG_HAVE_SCHED_AVG_IRQ: */ static inline unsigned long cpu_util_irq(struct rq *rq) { return 0; } static inline unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) { return util; } #endif /* !CONFIG_HAVE_SCHED_AVG_IRQ */ #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) #define perf_domain_span(pd) … DECLARE_STATIC_KEY_FALSE(sched_energy_present); static inline bool sched_energy_enabled(void) { … } extern struct cpufreq_governor schedutil_gov; #else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */ #define perf_domain_span … static inline bool sched_energy_enabled(void) { return false; } #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */ #ifdef CONFIG_MEMBARRIER /* * The scheduler provides memory barriers required by membarrier between: * - prior user-space memory accesses and store to rq->membarrier_state, * - store to rq->membarrier_state and following user-space memory accesses. * In the same way it provides those guarantees around store to rq->curr. */ static inline void membarrier_switch_mm(struct rq *rq, struct mm_struct *prev_mm, struct mm_struct *next_mm) { … } #else /* !CONFIG_MEMBARRIER :*/ static inline void membarrier_switch_mm(struct rq *rq, struct mm_struct *prev_mm, struct mm_struct *next_mm) { } #endif /* !CONFIG_MEMBARRIER */ #ifdef CONFIG_SMP static inline bool is_per_cpu_kthread(struct task_struct *p) { … } #endif extern void swake_up_all_locked(struct swait_queue_head *q); extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); extern int try_to_wake_up(struct task_struct *tsk, unsigned int state, int wake_flags); #ifdef CONFIG_PREEMPT_DYNAMIC extern int preempt_dynamic_mode; extern int sched_dynamic_mode(const char *str); extern void sched_dynamic_update(int mode); #endif #ifdef CONFIG_SCHED_MM_CID #define SCHED_MM_CID_PERIOD_NS … #define MM_CID_SCAN_DELAY … extern raw_spinlock_t cid_lock; extern int use_cid_lock; extern void sched_mm_cid_migrate_from(struct task_struct *t); extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t); extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr); extern void init_sched_mm_cid(struct task_struct *t); static inline void __mm_cid_put(struct mm_struct *mm, int cid) { … } /* * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to * be held to transition to other states. * * State transitions synchronized with cmpxchg or try_cmpxchg need to be * consistent across CPUs, which prevents use of this_cpu_cmpxchg. */ static inline void mm_cid_put_lazy(struct task_struct *t) { … } static inline int mm_cid_pcpu_unset(struct mm_struct *mm) { … } static inline void mm_cid_put(struct mm_struct *mm) { … } static inline int __mm_cid_try_get(struct mm_struct *mm) { … } /* * Save a snapshot of the current runqueue time of this cpu * with the per-cpu cid value, allowing to estimate how recently it was used. */ static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm) { … } static inline int __mm_cid_get(struct rq *rq, struct mm_struct *mm) { … } static inline int mm_cid_get(struct rq *rq, struct mm_struct *mm) { … } static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { … } #else /* !CONFIG_SCHED_MM_CID: */ static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { } static inline void sched_mm_cid_migrate_from(struct task_struct *t) { } static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { } static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { } static inline void init_sched_mm_cid(struct task_struct *t) { } #endif /* !CONFIG_SCHED_MM_CID */ extern u64 avg_vruntime(struct cfs_rq *cfs_rq); extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se); #ifdef CONFIG_RT_MUTEXES static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) { … } static inline int rt_effective_prio(struct task_struct *p, int prio) { … } #else /* !CONFIG_RT_MUTEXES: */ static inline int rt_effective_prio(struct task_struct *p, int prio) { return prio; } #endif /* !CONFIG_RT_MUTEXES */ extern int __sched_setscheduler(struct task_struct *p, const struct sched_attr *attr, bool user, bool pi); extern int __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx); extern void __setscheduler_prio(struct task_struct *p, int prio); extern void set_load_weight(struct task_struct *p, bool update_load); extern void enqueue_task(struct rq *rq, struct task_struct *p, int flags); extern void dequeue_task(struct rq *rq, struct task_struct *p, int flags); extern void check_class_changed(struct rq *rq, struct task_struct *p, const struct sched_class *prev_class, int oldprio); #ifdef CONFIG_SMP extern struct balance_callback *splice_balance_callbacks(struct rq *rq); extern void balance_callbacks(struct rq *rq, struct balance_callback *head); #else static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) { return NULL; } static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) { } #endif #endif /* _KERNEL_SCHED_SCHED_H */
Codebase Browser
linux