sched: Make separate sched*.c translation units
Since once needs to do something at conferences and fixing compile warnings doesn't actually require much if any attention I decided to break up the sched.c #include "*.c" fest. This further modularizes the scheduler code. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/n/tip-x0fcd3mnp8f9c99grcpewmhi@git.kernel.org Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Peter Zijlstra
Ingo Molnar
parent
60686317da
commit
029632fbb7
|
|
@ -10,6 +10,8 @@
|
|||
#define _INCLUDE_GUARD_LATENCYTOP_H_
|
||||
|
||||
#include <linux/compiler.h>
|
||||
struct task_struct;
|
||||
|
||||
#ifdef CONFIG_LATENCYTOP
|
||||
|
||||
#define LT_SAVECOUNT 32
|
||||
|
|
@ -23,7 +25,6 @@ struct latency_record {
|
|||
};
|
||||
|
||||
|
||||
struct task_struct;
|
||||
|
||||
extern int latencytop_enabled;
|
||||
void __account_scheduler_latency(struct task_struct *task, int usecs, int inter);
|
||||
|
|
|
|||
|
|
@ -925,6 +925,15 @@ static inline struct cpumask *sched_group_cpus(struct sched_group *sg)
|
|||
return to_cpumask(sg->cpumask);
|
||||
}
|
||||
|
||||
/**
|
||||
* group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
|
||||
* @group: The group whose first cpu is to be returned.
|
||||
*/
|
||||
static inline unsigned int group_first_cpu(struct sched_group *group)
|
||||
{
|
||||
return cpumask_first(sched_group_cpus(group));
|
||||
}
|
||||
|
||||
struct sched_domain_attr {
|
||||
int relax_domain_level;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Makefile for the linux kernel.
|
||||
#
|
||||
|
||||
obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
|
||||
obj-y = fork.o exec_domain.o panic.o printk.o \
|
||||
cpu.o exit.o itimer.o time.o softirq.o resource.o \
|
||||
sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \
|
||||
signal.o sys.o kmod.o workqueue.o pid.o \
|
||||
|
|
@ -10,8 +10,12 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
|
|||
kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
|
||||
hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
|
||||
notifier.o ksysfs.o sched_clock.o cred.o \
|
||||
async.o range.o
|
||||
obj-y += groups.o
|
||||
async.o range.o groups.o
|
||||
|
||||
obj-y += sched.o sched_idletask.o sched_fair.o sched_rt.o sched_stoptask.o
|
||||
obj-$(CONFIG_SCHED_AUTOGROUP) += sched_autogroup.o
|
||||
obj-$(CONFIG_SCHEDSTATS) += sched_stats.o
|
||||
obj-$(CONFIG_SCHED_DEBUG) += sched_debug.o
|
||||
|
||||
ifdef CONFIG_FUNCTION_TRACER
|
||||
# Do not trace debug files and internal ftrace files
|
||||
|
|
|
|||
1828
kernel/sched.c
1828
kernel/sched.c
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,15 +1,19 @@
|
|||
#ifdef CONFIG_SCHED_AUTOGROUP
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
#include <linux/proc_fs.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <linux/kallsyms.h>
|
||||
#include <linux/utsname.h>
|
||||
#include <linux/security.h>
|
||||
#include <linux/export.h>
|
||||
|
||||
unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
|
||||
static struct autogroup autogroup_default;
|
||||
static atomic_t autogroup_seq_nr;
|
||||
|
||||
static void __init autogroup_init(struct task_struct *init_task)
|
||||
void __init autogroup_init(struct task_struct *init_task)
|
||||
{
|
||||
autogroup_default.tg = &root_task_group;
|
||||
kref_init(&autogroup_default.kref);
|
||||
|
|
@ -17,7 +21,7 @@ static void __init autogroup_init(struct task_struct *init_task)
|
|||
init_task->signal->autogroup = &autogroup_default;
|
||||
}
|
||||
|
||||
static inline void autogroup_free(struct task_group *tg)
|
||||
void autogroup_free(struct task_group *tg)
|
||||
{
|
||||
kfree(tg->autogroup);
|
||||
}
|
||||
|
|
@ -59,10 +63,6 @@ static inline struct autogroup *autogroup_task_get(struct task_struct *p)
|
|||
return ag;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_RT_GROUP_SCHED
|
||||
static void free_rt_sched_group(struct task_group *tg);
|
||||
#endif
|
||||
|
||||
static inline struct autogroup *autogroup_create(void)
|
||||
{
|
||||
struct autogroup *ag = kzalloc(sizeof(*ag), GFP_KERNEL);
|
||||
|
|
@ -108,8 +108,7 @@ out_fail:
|
|||
return autogroup_kref_get(&autogroup_default);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
task_wants_autogroup(struct task_struct *p, struct task_group *tg)
|
||||
bool task_wants_autogroup(struct task_struct *p, struct task_group *tg)
|
||||
{
|
||||
if (tg != &root_task_group)
|
||||
return false;
|
||||
|
|
@ -127,22 +126,6 @@ task_wants_autogroup(struct task_struct *p, struct task_group *tg)
|
|||
return true;
|
||||
}
|
||||
|
||||
static inline bool task_group_is_autogroup(struct task_group *tg)
|
||||
{
|
||||
return !!tg->autogroup;
|
||||
}
|
||||
|
||||
static inline struct task_group *
|
||||
autogroup_task_group(struct task_struct *p, struct task_group *tg)
|
||||
{
|
||||
int enabled = ACCESS_ONCE(sysctl_sched_autogroup_enabled);
|
||||
|
||||
if (enabled && task_wants_autogroup(p, tg))
|
||||
return p->signal->autogroup->tg;
|
||||
|
||||
return tg;
|
||||
}
|
||||
|
||||
static void
|
||||
autogroup_move_group(struct task_struct *p, struct autogroup *ag)
|
||||
{
|
||||
|
|
@ -263,7 +246,7 @@ out:
|
|||
#endif /* CONFIG_PROC_FS */
|
||||
|
||||
#ifdef CONFIG_SCHED_DEBUG
|
||||
static inline int autogroup_path(struct task_group *tg, char *buf, int buflen)
|
||||
int autogroup_path(struct task_group *tg, char *buf, int buflen)
|
||||
{
|
||||
if (!task_group_is_autogroup(tg))
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#ifdef CONFIG_SCHED_AUTOGROUP
|
||||
|
||||
#include <linux/kref.h>
|
||||
#include <linux/rwsem.h>
|
||||
|
||||
struct autogroup {
|
||||
/*
|
||||
* reference doesn't mean how many thread attach to this
|
||||
|
|
@ -13,9 +16,28 @@ struct autogroup {
|
|||
int nice;
|
||||
};
|
||||
|
||||
static inline bool task_group_is_autogroup(struct task_group *tg);
|
||||
extern void autogroup_init(struct task_struct *init_task);
|
||||
extern void autogroup_free(struct task_group *tg);
|
||||
|
||||
static inline bool task_group_is_autogroup(struct task_group *tg)
|
||||
{
|
||||
return !!tg->autogroup;
|
||||
}
|
||||
|
||||
extern bool task_wants_autogroup(struct task_struct *p, struct task_group *tg);
|
||||
|
||||
static inline struct task_group *
|
||||
autogroup_task_group(struct task_struct *p, struct task_group *tg);
|
||||
autogroup_task_group(struct task_struct *p, struct task_group *tg)
|
||||
{
|
||||
int enabled = ACCESS_ONCE(sysctl_sched_autogroup_enabled);
|
||||
|
||||
if (enabled && task_wants_autogroup(p, tg))
|
||||
return p->signal->autogroup->tg;
|
||||
|
||||
return tg;
|
||||
}
|
||||
|
||||
extern int autogroup_path(struct task_group *tg, char *buf, int buflen);
|
||||
|
||||
#else /* !CONFIG_SCHED_AUTOGROUP */
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
#include <linux/kallsyms.h>
|
||||
#include <linux/utsname.h>
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
static DEFINE_SPINLOCK(sched_debug_lock);
|
||||
|
||||
/*
|
||||
|
|
@ -373,7 +375,7 @@ static int sched_debug_show(struct seq_file *m, void *v)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void sysrq_sched_debug_show(void)
|
||||
void sysrq_sched_debug_show(void)
|
||||
{
|
||||
sched_debug_show(NULL, NULL);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,13 @@
|
|||
#include <linux/latencytop.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/profile.h>
|
||||
#include <linux/interrupt.h>
|
||||
|
||||
#include <trace/events/sched.h>
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
/*
|
||||
* Targeted preemption latency for CPU-bound tasks:
|
||||
|
|
@ -103,7 +110,110 @@ unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL;
|
|||
unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
|
||||
#endif
|
||||
|
||||
static const struct sched_class fair_sched_class;
|
||||
/*
|
||||
* Increase the granularity value when there are more CPUs,
|
||||
* because with more CPUs the 'effective latency' as visible
|
||||
* to users decreases. But the relationship is not linear,
|
||||
* so pick a second-best guess by going with the log2 of the
|
||||
* number of CPUs.
|
||||
*
|
||||
* This idea comes from the SD scheduler of Con Kolivas:
|
||||
*/
|
||||
static int get_update_sysctl_factor(void)
|
||||
{
|
||||
unsigned int cpus = min_t(int, num_online_cpus(), 8);
|
||||
unsigned int factor;
|
||||
|
||||
switch (sysctl_sched_tunable_scaling) {
|
||||
case SCHED_TUNABLESCALING_NONE:
|
||||
factor = 1;
|
||||
break;
|
||||
case SCHED_TUNABLESCALING_LINEAR:
|
||||
factor = cpus;
|
||||
break;
|
||||
case SCHED_TUNABLESCALING_LOG:
|
||||
default:
|
||||
factor = 1 + ilog2(cpus);
|
||||
break;
|
||||
}
|
||||
|
||||
return factor;
|
||||
}
|
||||
|
||||
static void update_sysctl(void)
|
||||
{
|
||||
unsigned int factor = get_update_sysctl_factor();
|
||||
|
||||
#define SET_SYSCTL(name) \
|
||||
(sysctl_##name = (factor) * normalized_sysctl_##name)
|
||||
SET_SYSCTL(sched_min_granularity);
|
||||
SET_SYSCTL(sched_latency);
|
||||
SET_SYSCTL(sched_wakeup_granularity);
|
||||
#undef SET_SYSCTL
|
||||
}
|
||||
|
||||
void sched_init_granularity(void)
|
||||
{
|
||||
update_sysctl();
|
||||
}
|
||||
|
||||
#if BITS_PER_LONG == 32
|
||||
# define WMULT_CONST (~0UL)
|
||||
#else
|
||||
# define WMULT_CONST (1UL << 32)
|
||||
#endif
|
||||
|
||||
#define WMULT_SHIFT 32
|
||||
|
||||
/*
|
||||
* Shift right and round:
|
||||
*/
|
||||
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
|
||||
|
||||
/*
|
||||
* delta *= weight / lw
|
||||
*/
|
||||
static unsigned long
|
||||
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
|
||||
struct load_weight *lw)
|
||||
{
|
||||
u64 tmp;
|
||||
|
||||
/*
|
||||
* weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched
|
||||
* entities since MIN_SHARES = 2. Treat weight as 1 if less than
|
||||
* 2^SCHED_LOAD_RESOLUTION.
|
||||
*/
|
||||
if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION)))
|
||||
tmp = (u64)delta_exec * scale_load_down(weight);
|
||||
else
|
||||
tmp = (u64)delta_exec;
|
||||
|
||||
if (!lw->inv_weight) {
|
||||
unsigned long w = scale_load_down(lw->weight);
|
||||
|
||||
if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
|
||||
lw->inv_weight = 1;
|
||||
else if (unlikely(!w))
|
||||
lw->inv_weight = WMULT_CONST;
|
||||
else
|
||||
lw->inv_weight = WMULT_CONST / w;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check whether we'd overflow the 64-bit multiplication:
|
||||
*/
|
||||
if (unlikely(tmp > WMULT_CONST))
|
||||
tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
|
||||
WMULT_SHIFT/2);
|
||||
else
|
||||
tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
|
||||
|
||||
return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
|
||||
}
|
||||
|
||||
|
||||
const struct sched_class fair_sched_class;
|
||||
|
||||
/**************************************************************
|
||||
* CFS operations on generic schedulable entities:
|
||||
|
|
@ -413,7 +523,7 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|||
rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
|
||||
}
|
||||
|
||||
static struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
|
||||
struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
struct rb_node *left = cfs_rq->rb_leftmost;
|
||||
|
||||
|
|
@ -434,7 +544,7 @@ static struct sched_entity *__pick_next_entity(struct sched_entity *se)
|
|||
}
|
||||
|
||||
#ifdef CONFIG_SCHED_DEBUG
|
||||
static struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
|
||||
struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
struct rb_node *last = rb_last(&cfs_rq->tasks_timeline);
|
||||
|
||||
|
|
@ -684,7 +794,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|||
{
|
||||
update_load_add(&cfs_rq->load, se->load.weight);
|
||||
if (!parent_entity(se))
|
||||
inc_cpu_load(rq_of(cfs_rq), se->load.weight);
|
||||
update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
|
||||
if (entity_is_task(se)) {
|
||||
add_cfs_task_weight(cfs_rq, se->load.weight);
|
||||
list_add(&se->group_node, &cfs_rq->tasks);
|
||||
|
|
@ -697,7 +807,7 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|||
{
|
||||
update_load_sub(&cfs_rq->load, se->load.weight);
|
||||
if (!parent_entity(se))
|
||||
dec_cpu_load(rq_of(cfs_rq), se->load.weight);
|
||||
update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
|
||||
if (entity_is_task(se)) {
|
||||
add_cfs_task_weight(cfs_rq, -se->load.weight);
|
||||
list_del_init(&se->group_node);
|
||||
|
|
@ -1287,6 +1397,32 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
|
|||
*/
|
||||
|
||||
#ifdef CONFIG_CFS_BANDWIDTH
|
||||
|
||||
#ifdef HAVE_JUMP_LABEL
|
||||
static struct jump_label_key __cfs_bandwidth_used;
|
||||
|
||||
static inline bool cfs_bandwidth_used(void)
|
||||
{
|
||||
return static_branch(&__cfs_bandwidth_used);
|
||||
}
|
||||
|
||||
void account_cfs_bandwidth_used(int enabled, int was_enabled)
|
||||
{
|
||||
/* only need to count groups transitioning between enabled/!enabled */
|
||||
if (enabled && !was_enabled)
|
||||
jump_label_inc(&__cfs_bandwidth_used);
|
||||
else if (!enabled && was_enabled)
|
||||
jump_label_dec(&__cfs_bandwidth_used);
|
||||
}
|
||||
#else /* HAVE_JUMP_LABEL */
|
||||
static bool cfs_bandwidth_used(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
void account_cfs_bandwidth_used(int enabled, int was_enabled) {}
|
||||
#endif /* HAVE_JUMP_LABEL */
|
||||
|
||||
/*
|
||||
* default period for cfs group bandwidth.
|
||||
* default: 0.1s, units: nanoseconds
|
||||
|
|
@ -1308,7 +1444,7 @@ static inline u64 sched_cfs_bandwidth_slice(void)
|
|||
*
|
||||
* requires cfs_b->lock
|
||||
*/
|
||||
static void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
|
||||
void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
|
||||
{
|
||||
u64 now;
|
||||
|
||||
|
|
@ -1320,6 +1456,11 @@ static void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
|
|||
cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
|
||||
}
|
||||
|
||||
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
|
||||
{
|
||||
return &tg->cfs_bandwidth;
|
||||
}
|
||||
|
||||
/* returns 0 on failure to allocate runtime */
|
||||
static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
|
|
@ -1530,7 +1671,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
|
|||
raw_spin_unlock(&cfs_b->lock);
|
||||
}
|
||||
|
||||
static void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
|
||||
void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
struct rq *rq = rq_of(cfs_rq);
|
||||
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
||||
|
|
@ -1839,7 +1980,112 @@ static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
|||
|
||||
throttle_cfs_rq(cfs_rq);
|
||||
}
|
||||
#else
|
||||
|
||||
static inline u64 default_cfs_period(void);
|
||||
static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
|
||||
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b);
|
||||
|
||||
static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
|
||||
{
|
||||
struct cfs_bandwidth *cfs_b =
|
||||
container_of(timer, struct cfs_bandwidth, slack_timer);
|
||||
do_sched_cfs_slack_timer(cfs_b);
|
||||
|
||||
return HRTIMER_NORESTART;
|
||||
}
|
||||
|
||||
static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
|
||||
{
|
||||
struct cfs_bandwidth *cfs_b =
|
||||
container_of(timer, struct cfs_bandwidth, period_timer);
|
||||
ktime_t now;
|
||||
int overrun;
|
||||
int idle = 0;
|
||||
|
||||
for (;;) {
|
||||
now = hrtimer_cb_get_time(timer);
|
||||
overrun = hrtimer_forward(timer, now, cfs_b->period);
|
||||
|
||||
if (!overrun)
|
||||
break;
|
||||
|
||||
idle = do_sched_cfs_period_timer(cfs_b, overrun);
|
||||
}
|
||||
|
||||
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
|
||||
}
|
||||
|
||||
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
||||
{
|
||||
raw_spin_lock_init(&cfs_b->lock);
|
||||
cfs_b->runtime = 0;
|
||||
cfs_b->quota = RUNTIME_INF;
|
||||
cfs_b->period = ns_to_ktime(default_cfs_period());
|
||||
|
||||
INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
|
||||
hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
||||
cfs_b->period_timer.function = sched_cfs_period_timer;
|
||||
hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
||||
cfs_b->slack_timer.function = sched_cfs_slack_timer;
|
||||
}
|
||||
|
||||
static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
cfs_rq->runtime_enabled = 0;
|
||||
INIT_LIST_HEAD(&cfs_rq->throttled_list);
|
||||
}
|
||||
|
||||
/* requires cfs_b->lock, may release to reprogram timer */
|
||||
void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
||||
{
|
||||
/*
|
||||
* The timer may be active because we're trying to set a new bandwidth
|
||||
* period or because we're racing with the tear-down path
|
||||
* (timer_active==0 becomes visible before the hrtimer call-back
|
||||
* terminates). In either case we ensure that it's re-programmed
|
||||
*/
|
||||
while (unlikely(hrtimer_active(&cfs_b->period_timer))) {
|
||||
raw_spin_unlock(&cfs_b->lock);
|
||||
/* ensure cfs_b->lock is available while we wait */
|
||||
hrtimer_cancel(&cfs_b->period_timer);
|
||||
|
||||
raw_spin_lock(&cfs_b->lock);
|
||||
/* if someone else restarted the timer then we're done */
|
||||
if (cfs_b->timer_active)
|
||||
return;
|
||||
}
|
||||
|
||||
cfs_b->timer_active = 1;
|
||||
start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
|
||||
}
|
||||
|
||||
static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
||||
{
|
||||
hrtimer_cancel(&cfs_b->period_timer);
|
||||
hrtimer_cancel(&cfs_b->slack_timer);
|
||||
}
|
||||
|
||||
void unthrottle_offline_cfs_rqs(struct rq *rq)
|
||||
{
|
||||
struct cfs_rq *cfs_rq;
|
||||
|
||||
for_each_leaf_cfs_rq(rq, cfs_rq) {
|
||||
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
||||
|
||||
if (!cfs_rq->runtime_enabled)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* clock_task is not advancing so we just need to make sure
|
||||
* there's some valid quota amount
|
||||
*/
|
||||
cfs_rq->runtime_remaining = cfs_b->quota;
|
||||
if (cfs_rq_throttled(cfs_rq))
|
||||
unthrottle_cfs_rq(cfs_rq);
|
||||
}
|
||||
}
|
||||
|
||||
#else /* CONFIG_CFS_BANDWIDTH */
|
||||
static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
|
||||
unsigned long delta_exec) {}
|
||||
static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
|
||||
|
|
@ -1861,8 +2107,22 @@ static inline int throttled_lb_pair(struct task_group *tg,
|
|||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
|
||||
#endif
|
||||
|
||||
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
|
||||
void unthrottle_offline_cfs_rqs(struct rq *rq) {}
|
||||
|
||||
#endif /* CONFIG_CFS_BANDWIDTH */
|
||||
|
||||
/**************************************************
|
||||
* CFS operations on tasks:
|
||||
*/
|
||||
|
|
@ -2029,6 +2289,61 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
|||
}
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/* Used instead of source_load when we know the type == 0 */
|
||||
static unsigned long weighted_cpuload(const int cpu)
|
||||
{
|
||||
return cpu_rq(cpu)->load.weight;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return a low guess at the load of a migration-source cpu weighted
|
||||
* according to the scheduling class and "nice" value.
|
||||
*
|
||||
* We want to under-estimate the load of migration sources, to
|
||||
* balance conservatively.
|
||||
*/
|
||||
static unsigned long source_load(int cpu, int type)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long total = weighted_cpuload(cpu);
|
||||
|
||||
if (type == 0 || !sched_feat(LB_BIAS))
|
||||
return total;
|
||||
|
||||
return min(rq->cpu_load[type-1], total);
|
||||
}
|
||||
|
||||
/*
|
||||
* Return a high guess at the load of a migration-target cpu weighted
|
||||
* according to the scheduling class and "nice" value.
|
||||
*/
|
||||
static unsigned long target_load(int cpu, int type)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long total = weighted_cpuload(cpu);
|
||||
|
||||
if (type == 0 || !sched_feat(LB_BIAS))
|
||||
return total;
|
||||
|
||||
return max(rq->cpu_load[type-1], total);
|
||||
}
|
||||
|
||||
static unsigned long power_of(int cpu)
|
||||
{
|
||||
return cpu_rq(cpu)->cpu_power;
|
||||
}
|
||||
|
||||
static unsigned long cpu_avg_load_per_task(int cpu)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
|
||||
|
||||
if (nr_running)
|
||||
return rq->load.weight / nr_running;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static void task_waking_fair(struct task_struct *p)
|
||||
{
|
||||
|
|
@ -2782,6 +3097,38 @@ static void pull_task(struct rq *src_rq, struct task_struct *p,
|
|||
check_preempt_curr(this_rq, p, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Is this task likely cache-hot:
|
||||
*/
|
||||
static int
|
||||
task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
|
||||
{
|
||||
s64 delta;
|
||||
|
||||
if (p->sched_class != &fair_sched_class)
|
||||
return 0;
|
||||
|
||||
if (unlikely(p->policy == SCHED_IDLE))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* Buddy candidates are cache hot:
|
||||
*/
|
||||
if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
|
||||
(&p->se == cfs_rq_of(&p->se)->next ||
|
||||
&p->se == cfs_rq_of(&p->se)->last))
|
||||
return 1;
|
||||
|
||||
if (sysctl_sched_migration_cost == -1)
|
||||
return 1;
|
||||
if (sysctl_sched_migration_cost == 0)
|
||||
return 0;
|
||||
|
||||
delta = now - p->se.exec_start;
|
||||
|
||||
return delta < (s64)sysctl_sched_migration_cost;
|
||||
}
|
||||
|
||||
/*
|
||||
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
|
||||
*/
|
||||
|
|
@ -3161,15 +3508,6 @@ struct sg_lb_stats {
|
|||
int group_has_capacity; /* Is there extra capacity in the group? */
|
||||
};
|
||||
|
||||
/**
|
||||
* group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
|
||||
* @group: The group whose first cpu is to be returned.
|
||||
*/
|
||||
static inline unsigned int group_first_cpu(struct sched_group *group)
|
||||
{
|
||||
return cpumask_first(sched_group_cpus(group));
|
||||
}
|
||||
|
||||
/**
|
||||
* get_sd_load_idx - Obtain the load index for a given sched domain.
|
||||
* @sd: The sched_domain whose load_idx is to be obtained.
|
||||
|
|
@ -3419,7 +3757,7 @@ static void update_cpu_power(struct sched_domain *sd, int cpu)
|
|||
sdg->sgp->power = power;
|
||||
}
|
||||
|
||||
static void update_group_power(struct sched_domain *sd, int cpu)
|
||||
void update_group_power(struct sched_domain *sd, int cpu)
|
||||
{
|
||||
struct sched_domain *child = sd->child;
|
||||
struct sched_group *group, *sdg = sd->groups;
|
||||
|
|
@ -3685,11 +4023,6 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
|
|||
} while (sg != sd->groups);
|
||||
}
|
||||
|
||||
int __weak arch_sd_sibling_asym_packing(void)
|
||||
{
|
||||
return 0*SD_ASYM_PACKING;
|
||||
}
|
||||
|
||||
/**
|
||||
* check_asym_packing - Check to see if the group is packed into the
|
||||
* sched doman.
|
||||
|
|
@ -4053,7 +4386,7 @@ find_busiest_queue(struct sched_domain *sd, struct sched_group *group,
|
|||
#define MAX_PINNED_INTERVAL 512
|
||||
|
||||
/* Working cpumask for load_balance and load_balance_newidle. */
|
||||
static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
|
||||
DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
|
||||
|
||||
static int need_active_balance(struct sched_domain *sd, int idle,
|
||||
int busiest_cpu, int this_cpu)
|
||||
|
|
@ -4256,7 +4589,7 @@ out:
|
|||
* idle_balance is called by schedule() if this_cpu is about to become
|
||||
* idle. Attempts to pull tasks from other CPUs.
|
||||
*/
|
||||
static void idle_balance(int this_cpu, struct rq *this_rq)
|
||||
void idle_balance(int this_cpu, struct rq *this_rq)
|
||||
{
|
||||
struct sched_domain *sd;
|
||||
int pulled_task = 0;
|
||||
|
|
@ -4631,7 +4964,7 @@ static unsigned long __read_mostly max_load_balance_interval = HZ/10;
|
|||
* Scale the max load_balance interval with the number of CPUs in the system.
|
||||
* This trades load-balance latency on larger machines for less cross talk.
|
||||
*/
|
||||
static void update_max_interval(void)
|
||||
void update_max_interval(void)
|
||||
{
|
||||
max_load_balance_interval = HZ*num_online_cpus()/10;
|
||||
}
|
||||
|
|
@ -4833,7 +5166,7 @@ static inline int on_null_domain(int cpu)
|
|||
/*
|
||||
* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
|
||||
*/
|
||||
static inline void trigger_load_balance(struct rq *rq, int cpu)
|
||||
void trigger_load_balance(struct rq *rq, int cpu)
|
||||
{
|
||||
/* Don't need to rebalance while attached to NULL domain */
|
||||
if (time_after_eq(jiffies, rq->next_balance) &&
|
||||
|
|
@ -4855,15 +5188,6 @@ static void rq_offline_fair(struct rq *rq)
|
|||
update_sysctl();
|
||||
}
|
||||
|
||||
#else /* CONFIG_SMP */
|
||||
|
||||
/*
|
||||
* on UP we do not need to balance between CPUs:
|
||||
*/
|
||||
static inline void idle_balance(int cpu, struct rq *rq)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
/*
|
||||
|
|
@ -5006,6 +5330,16 @@ static void set_curr_task_fair(struct rq *rq)
|
|||
}
|
||||
}
|
||||
|
||||
void init_cfs_rq(struct cfs_rq *cfs_rq)
|
||||
{
|
||||
cfs_rq->tasks_timeline = RB_ROOT;
|
||||
INIT_LIST_HEAD(&cfs_rq->tasks);
|
||||
cfs_rq->min_vruntime = (u64)(-(1LL << 20));
|
||||
#ifndef CONFIG_64BIT
|
||||
cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
static void task_move_group_fair(struct task_struct *p, int on_rq)
|
||||
{
|
||||
|
|
@ -5028,7 +5362,161 @@ static void task_move_group_fair(struct task_struct *p, int on_rq)
|
|||
if (!on_rq)
|
||||
p->se.vruntime += cfs_rq_of(&p->se)->min_vruntime;
|
||||
}
|
||||
|
||||
void free_fair_sched_group(struct task_group *tg)
|
||||
{
|
||||
int i;
|
||||
|
||||
destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
if (tg->cfs_rq)
|
||||
kfree(tg->cfs_rq[i]);
|
||||
if (tg->se)
|
||||
kfree(tg->se[i]);
|
||||
}
|
||||
|
||||
kfree(tg->cfs_rq);
|
||||
kfree(tg->se);
|
||||
}
|
||||
|
||||
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
|
||||
{
|
||||
struct cfs_rq *cfs_rq;
|
||||
struct sched_entity *se;
|
||||
int i;
|
||||
|
||||
tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
|
||||
if (!tg->cfs_rq)
|
||||
goto err;
|
||||
tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
|
||||
if (!tg->se)
|
||||
goto err;
|
||||
|
||||
tg->shares = NICE_0_LOAD;
|
||||
|
||||
init_cfs_bandwidth(tg_cfs_bandwidth(tg));
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
if (!cfs_rq)
|
||||
goto err;
|
||||
|
||||
se = kzalloc_node(sizeof(struct sched_entity),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
if (!se)
|
||||
goto err_free_rq;
|
||||
|
||||
init_cfs_rq(cfs_rq);
|
||||
init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
|
||||
}
|
||||
|
||||
return 1;
|
||||
|
||||
err_free_rq:
|
||||
kfree(cfs_rq);
|
||||
err:
|
||||
return 0;
|
||||
}
|
||||
|
||||
void unregister_fair_sched_group(struct task_group *tg, int cpu)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
unsigned long flags;
|
||||
|
||||
/*
|
||||
* Only empty task groups can be destroyed; so we can speculatively
|
||||
* check on_list without danger of it being re-added.
|
||||
*/
|
||||
if (!tg->cfs_rq[cpu]->on_list)
|
||||
return;
|
||||
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
|
||||
struct sched_entity *se, int cpu,
|
||||
struct sched_entity *parent)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
cfs_rq->tg = tg;
|
||||
cfs_rq->rq = rq;
|
||||
#ifdef CONFIG_SMP
|
||||
/* allow initial update_cfs_load() to truncate */
|
||||
cfs_rq->load_stamp = 1;
|
||||
#endif
|
||||
init_cfs_rq_runtime(cfs_rq);
|
||||
|
||||
tg->cfs_rq[cpu] = cfs_rq;
|
||||
tg->se[cpu] = se;
|
||||
|
||||
/* se could be NULL for root_task_group */
|
||||
if (!se)
|
||||
return;
|
||||
|
||||
if (!parent)
|
||||
se->cfs_rq = &rq->cfs;
|
||||
else
|
||||
se->cfs_rq = parent->my_q;
|
||||
|
||||
se->my_q = cfs_rq;
|
||||
update_load_set(&se->load, 0);
|
||||
se->parent = parent;
|
||||
}
|
||||
|
||||
static DEFINE_MUTEX(shares_mutex);
|
||||
|
||||
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
|
||||
{
|
||||
int i;
|
||||
unsigned long flags;
|
||||
|
||||
/*
|
||||
* We can't change the weight of the root cgroup.
|
||||
*/
|
||||
if (!tg->se[0])
|
||||
return -EINVAL;
|
||||
|
||||
shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
|
||||
|
||||
mutex_lock(&shares_mutex);
|
||||
if (tg->shares == shares)
|
||||
goto done;
|
||||
|
||||
tg->shares = shares;
|
||||
for_each_possible_cpu(i) {
|
||||
struct rq *rq = cpu_rq(i);
|
||||
struct sched_entity *se;
|
||||
|
||||
se = tg->se[i];
|
||||
/* Propagate contribution to hierarchy */
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
for_each_sched_entity(se)
|
||||
update_cfs_shares(group_cfs_rq(se));
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
done:
|
||||
mutex_unlock(&shares_mutex);
|
||||
return 0;
|
||||
}
|
||||
#else /* CONFIG_FAIR_GROUP_SCHED */
|
||||
|
||||
void free_fair_sched_group(struct task_group *tg) { }
|
||||
|
||||
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
void unregister_fair_sched_group(struct task_group *tg, int cpu) { }
|
||||
|
||||
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
||||
|
||||
|
||||
static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
|
||||
{
|
||||
|
|
@ -5048,7 +5536,7 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
|
|||
/*
|
||||
* All the scheduling class methods:
|
||||
*/
|
||||
static const struct sched_class fair_sched_class = {
|
||||
const struct sched_class fair_sched_class = {
|
||||
.next = &idle_sched_class,
|
||||
.enqueue_task = enqueue_task_fair,
|
||||
.dequeue_task = dequeue_task_fair,
|
||||
|
|
@ -5085,7 +5573,7 @@ static const struct sched_class fair_sched_class = {
|
|||
};
|
||||
|
||||
#ifdef CONFIG_SCHED_DEBUG
|
||||
static void print_cfs_stats(struct seq_file *m, int cpu)
|
||||
void print_cfs_stats(struct seq_file *m, int cpu)
|
||||
{
|
||||
struct cfs_rq *cfs_rq;
|
||||
|
||||
|
|
@ -5095,3 +5583,19 @@ static void print_cfs_stats(struct seq_file *m, int cpu)
|
|||
rcu_read_unlock();
|
||||
}
|
||||
#endif
|
||||
|
||||
__init void init_sched_fair_class(void)
|
||||
{
|
||||
#ifdef CONFIG_SMP
|
||||
open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
|
||||
|
||||
#ifdef CONFIG_NO_HZ
|
||||
zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
|
||||
alloc_cpumask_var(&nohz.grp_idle_mask, GFP_NOWAIT);
|
||||
atomic_set(&nohz.load_balancer, nr_cpu_ids);
|
||||
atomic_set(&nohz.first_pick_cpu, nr_cpu_ids);
|
||||
atomic_set(&nohz.second_pick_cpu, nr_cpu_ids);
|
||||
#endif
|
||||
#endif /* SMP */
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#include "sched.h"
|
||||
|
||||
/*
|
||||
* idle-task scheduling class.
|
||||
*
|
||||
|
|
@ -71,7 +73,7 @@ static unsigned int get_rr_interval_idle(struct rq *rq, struct task_struct *task
|
|||
/*
|
||||
* Simple, special scheduling class for the per-CPU idle tasks:
|
||||
*/
|
||||
static const struct sched_class idle_sched_class = {
|
||||
const struct sched_class idle_sched_class = {
|
||||
/* .next is NULL */
|
||||
/* no enqueue/yield_task for idle tasks */
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,92 @@
|
|||
* policies)
|
||||
*/
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
#include <linux/slab.h>
|
||||
|
||||
static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
|
||||
|
||||
struct rt_bandwidth def_rt_bandwidth;
|
||||
|
||||
static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
|
||||
{
|
||||
struct rt_bandwidth *rt_b =
|
||||
container_of(timer, struct rt_bandwidth, rt_period_timer);
|
||||
ktime_t now;
|
||||
int overrun;
|
||||
int idle = 0;
|
||||
|
||||
for (;;) {
|
||||
now = hrtimer_cb_get_time(timer);
|
||||
overrun = hrtimer_forward(timer, now, rt_b->rt_period);
|
||||
|
||||
if (!overrun)
|
||||
break;
|
||||
|
||||
idle = do_sched_rt_period_timer(rt_b, overrun);
|
||||
}
|
||||
|
||||
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
|
||||
}
|
||||
|
||||
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
|
||||
{
|
||||
rt_b->rt_period = ns_to_ktime(period);
|
||||
rt_b->rt_runtime = runtime;
|
||||
|
||||
raw_spin_lock_init(&rt_b->rt_runtime_lock);
|
||||
|
||||
hrtimer_init(&rt_b->rt_period_timer,
|
||||
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
||||
rt_b->rt_period_timer.function = sched_rt_period_timer;
|
||||
}
|
||||
|
||||
static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
|
||||
{
|
||||
if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
|
||||
return;
|
||||
|
||||
if (hrtimer_active(&rt_b->rt_period_timer))
|
||||
return;
|
||||
|
||||
raw_spin_lock(&rt_b->rt_runtime_lock);
|
||||
start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);
|
||||
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
||||
}
|
||||
|
||||
void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
|
||||
{
|
||||
struct rt_prio_array *array;
|
||||
int i;
|
||||
|
||||
array = &rt_rq->active;
|
||||
for (i = 0; i < MAX_RT_PRIO; i++) {
|
||||
INIT_LIST_HEAD(array->queue + i);
|
||||
__clear_bit(i, array->bitmap);
|
||||
}
|
||||
/* delimiter for bitsearch: */
|
||||
__set_bit(MAX_RT_PRIO, array->bitmap);
|
||||
|
||||
#if defined CONFIG_SMP
|
||||
rt_rq->highest_prio.curr = MAX_RT_PRIO;
|
||||
rt_rq->highest_prio.next = MAX_RT_PRIO;
|
||||
rt_rq->rt_nr_migratory = 0;
|
||||
rt_rq->overloaded = 0;
|
||||
plist_head_init(&rt_rq->pushable_tasks);
|
||||
#endif
|
||||
|
||||
rt_rq->rt_time = 0;
|
||||
rt_rq->rt_throttled = 0;
|
||||
rt_rq->rt_runtime = 0;
|
||||
raw_spin_lock_init(&rt_rq->rt_runtime_lock);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_RT_GROUP_SCHED
|
||||
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
|
||||
{
|
||||
hrtimer_cancel(&rt_b->rt_period_timer);
|
||||
}
|
||||
|
||||
#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
|
||||
|
||||
|
|
@ -25,6 +110,91 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
|
|||
return rt_se->rt_rq;
|
||||
}
|
||||
|
||||
void free_rt_sched_group(struct task_group *tg)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (tg->rt_se)
|
||||
destroy_rt_bandwidth(&tg->rt_bandwidth);
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
if (tg->rt_rq)
|
||||
kfree(tg->rt_rq[i]);
|
||||
if (tg->rt_se)
|
||||
kfree(tg->rt_se[i]);
|
||||
}
|
||||
|
||||
kfree(tg->rt_rq);
|
||||
kfree(tg->rt_se);
|
||||
}
|
||||
|
||||
void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
|
||||
struct sched_rt_entity *rt_se, int cpu,
|
||||
struct sched_rt_entity *parent)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
rt_rq->highest_prio.curr = MAX_RT_PRIO;
|
||||
rt_rq->rt_nr_boosted = 0;
|
||||
rt_rq->rq = rq;
|
||||
rt_rq->tg = tg;
|
||||
|
||||
tg->rt_rq[cpu] = rt_rq;
|
||||
tg->rt_se[cpu] = rt_se;
|
||||
|
||||
if (!rt_se)
|
||||
return;
|
||||
|
||||
if (!parent)
|
||||
rt_se->rt_rq = &rq->rt;
|
||||
else
|
||||
rt_se->rt_rq = parent->my_q;
|
||||
|
||||
rt_se->my_q = rt_rq;
|
||||
rt_se->parent = parent;
|
||||
INIT_LIST_HEAD(&rt_se->run_list);
|
||||
}
|
||||
|
||||
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
|
||||
{
|
||||
struct rt_rq *rt_rq;
|
||||
struct sched_rt_entity *rt_se;
|
||||
int i;
|
||||
|
||||
tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
|
||||
if (!tg->rt_rq)
|
||||
goto err;
|
||||
tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
|
||||
if (!tg->rt_se)
|
||||
goto err;
|
||||
|
||||
init_rt_bandwidth(&tg->rt_bandwidth,
|
||||
ktime_to_ns(def_rt_bandwidth.rt_period), 0);
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
rt_rq = kzalloc_node(sizeof(struct rt_rq),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
if (!rt_rq)
|
||||
goto err;
|
||||
|
||||
rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
if (!rt_se)
|
||||
goto err_free_rq;
|
||||
|
||||
init_rt_rq(rt_rq, cpu_rq(i));
|
||||
rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
|
||||
init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
|
||||
}
|
||||
|
||||
return 1;
|
||||
|
||||
err_free_rq:
|
||||
kfree(rt_rq);
|
||||
err:
|
||||
return 0;
|
||||
}
|
||||
|
||||
#else /* CONFIG_RT_GROUP_SCHED */
|
||||
|
||||
#define rt_entity_is_task(rt_se) (1)
|
||||
|
|
@ -47,6 +217,12 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
|
|||
return &rq->rt;
|
||||
}
|
||||
|
||||
void free_rt_sched_group(struct task_group *tg) { }
|
||||
|
||||
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
#endif /* CONFIG_RT_GROUP_SCHED */
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
|
|
@ -556,6 +732,28 @@ static void enable_runtime(struct rq *rq)
|
|||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
int update_runtime(struct notifier_block *nfb, unsigned long action, void *hcpu)
|
||||
{
|
||||
int cpu = (int)(long)hcpu;
|
||||
|
||||
switch (action) {
|
||||
case CPU_DOWN_PREPARE:
|
||||
case CPU_DOWN_PREPARE_FROZEN:
|
||||
disable_runtime(cpu_rq(cpu));
|
||||
return NOTIFY_OK;
|
||||
|
||||
case CPU_DOWN_FAILED:
|
||||
case CPU_DOWN_FAILED_FROZEN:
|
||||
case CPU_ONLINE:
|
||||
case CPU_ONLINE_FROZEN:
|
||||
enable_runtime(cpu_rq(cpu));
|
||||
return NOTIFY_OK;
|
||||
|
||||
default:
|
||||
return NOTIFY_DONE;
|
||||
}
|
||||
}
|
||||
|
||||
static int balance_runtime(struct rt_rq *rt_rq)
|
||||
{
|
||||
int more = 0;
|
||||
|
|
@ -1178,8 +1376,6 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
|
|||
/* Only try algorithms three times */
|
||||
#define RT_MAX_TRIES 3
|
||||
|
||||
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep);
|
||||
|
||||
static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
|
||||
{
|
||||
if (!task_running(rq, p) &&
|
||||
|
|
@ -1653,13 +1849,14 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p)
|
|||
pull_rt_task(rq);
|
||||
}
|
||||
|
||||
static inline void init_sched_rt_class(void)
|
||||
void init_sched_rt_class(void)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
for_each_possible_cpu(i)
|
||||
for_each_possible_cpu(i) {
|
||||
zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
}
|
||||
}
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
|
|
@ -1800,7 +1997,7 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static const struct sched_class rt_sched_class = {
|
||||
const struct sched_class rt_sched_class = {
|
||||
.next = &fair_sched_class,
|
||||
.enqueue_task = enqueue_task_rt,
|
||||
.dequeue_task = dequeue_task_rt,
|
||||
|
|
@ -1835,7 +2032,7 @@ static const struct sched_class rt_sched_class = {
|
|||
#ifdef CONFIG_SCHED_DEBUG
|
||||
extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
|
||||
|
||||
static void print_rt_stats(struct seq_file *m, int cpu)
|
||||
void print_rt_stats(struct seq_file *m, int cpu)
|
||||
{
|
||||
rt_rq_iter_t iter;
|
||||
struct rt_rq *rt_rq;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
|
||||
#include <linux/slab.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <linux/proc_fs.h>
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
/*
|
||||
* bump this up when changing the output format or the meaning of an existing
|
||||
* format, so that tools can adapt (or abort)
|
||||
*/
|
||||
#define SCHEDSTAT_VERSION 15
|
||||
|
||||
static int show_schedstat(struct seq_file *seq, void *v)
|
||||
{
|
||||
int cpu;
|
||||
int mask_len = DIV_ROUND_UP(NR_CPUS, 32) * 9;
|
||||
char *mask_str = kmalloc(mask_len, GFP_KERNEL);
|
||||
|
||||
if (mask_str == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION);
|
||||
seq_printf(seq, "timestamp %lu\n", jiffies);
|
||||
for_each_online_cpu(cpu) {
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
#ifdef CONFIG_SMP
|
||||
struct sched_domain *sd;
|
||||
int dcount = 0;
|
||||
#endif
|
||||
|
||||
/* runqueue-specific stats */
|
||||
seq_printf(seq,
|
||||
"cpu%d %u %u %u %u %u %u %llu %llu %lu",
|
||||
cpu, rq->yld_count,
|
||||
rq->sched_switch, rq->sched_count, rq->sched_goidle,
|
||||
rq->ttwu_count, rq->ttwu_local,
|
||||
rq->rq_cpu_time,
|
||||
rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount);
|
||||
|
||||
seq_printf(seq, "\n");
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/* domain-specific stats */
|
||||
rcu_read_lock();
|
||||
for_each_domain(cpu, sd) {
|
||||
enum cpu_idle_type itype;
|
||||
|
||||
cpumask_scnprintf(mask_str, mask_len,
|
||||
sched_domain_span(sd));
|
||||
seq_printf(seq, "domain%d %s", dcount++, mask_str);
|
||||
for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES;
|
||||
itype++) {
|
||||
seq_printf(seq, " %u %u %u %u %u %u %u %u",
|
||||
sd->lb_count[itype],
|
||||
sd->lb_balanced[itype],
|
||||
sd->lb_failed[itype],
|
||||
sd->lb_imbalance[itype],
|
||||
sd->lb_gained[itype],
|
||||
sd->lb_hot_gained[itype],
|
||||
sd->lb_nobusyq[itype],
|
||||
sd->lb_nobusyg[itype]);
|
||||
}
|
||||
seq_printf(seq,
|
||||
" %u %u %u %u %u %u %u %u %u %u %u %u\n",
|
||||
sd->alb_count, sd->alb_failed, sd->alb_pushed,
|
||||
sd->sbe_count, sd->sbe_balanced, sd->sbe_pushed,
|
||||
sd->sbf_count, sd->sbf_balanced, sd->sbf_pushed,
|
||||
sd->ttwu_wake_remote, sd->ttwu_move_affine,
|
||||
sd->ttwu_move_balance);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
#endif
|
||||
}
|
||||
kfree(mask_str);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int schedstat_open(struct inode *inode, struct file *file)
|
||||
{
|
||||
unsigned int size = PAGE_SIZE * (1 + num_online_cpus() / 32);
|
||||
char *buf = kmalloc(size, GFP_KERNEL);
|
||||
struct seq_file *m;
|
||||
int res;
|
||||
|
||||
if (!buf)
|
||||
return -ENOMEM;
|
||||
res = single_open(file, show_schedstat, NULL);
|
||||
if (!res) {
|
||||
m = file->private_data;
|
||||
m->buf = buf;
|
||||
m->size = size;
|
||||
} else
|
||||
kfree(buf);
|
||||
return res;
|
||||
}
|
||||
|
||||
static const struct file_operations proc_schedstat_operations = {
|
||||
.open = schedstat_open,
|
||||
.read = seq_read,
|
||||
.llseek = seq_lseek,
|
||||
.release = single_release,
|
||||
};
|
||||
|
||||
static int __init proc_schedstat_init(void)
|
||||
{
|
||||
proc_create("schedstat", 0, NULL, &proc_schedstat_operations);
|
||||
return 0;
|
||||
}
|
||||
module_init(proc_schedstat_init);
|
||||
|
|
@ -1,108 +1,5 @@
|
|||
|
||||
#ifdef CONFIG_SCHEDSTATS
|
||||
/*
|
||||
* bump this up when changing the output format or the meaning of an existing
|
||||
* format, so that tools can adapt (or abort)
|
||||
*/
|
||||
#define SCHEDSTAT_VERSION 15
|
||||
|
||||
static int show_schedstat(struct seq_file *seq, void *v)
|
||||
{
|
||||
int cpu;
|
||||
int mask_len = DIV_ROUND_UP(NR_CPUS, 32) * 9;
|
||||
char *mask_str = kmalloc(mask_len, GFP_KERNEL);
|
||||
|
||||
if (mask_str == NULL)
|
||||
return -ENOMEM;
|
||||
|
||||
seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION);
|
||||
seq_printf(seq, "timestamp %lu\n", jiffies);
|
||||
for_each_online_cpu(cpu) {
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
#ifdef CONFIG_SMP
|
||||
struct sched_domain *sd;
|
||||
int dcount = 0;
|
||||
#endif
|
||||
|
||||
/* runqueue-specific stats */
|
||||
seq_printf(seq,
|
||||
"cpu%d %u %u %u %u %u %u %llu %llu %lu",
|
||||
cpu, rq->yld_count,
|
||||
rq->sched_switch, rq->sched_count, rq->sched_goidle,
|
||||
rq->ttwu_count, rq->ttwu_local,
|
||||
rq->rq_cpu_time,
|
||||
rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount);
|
||||
|
||||
seq_printf(seq, "\n");
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/* domain-specific stats */
|
||||
rcu_read_lock();
|
||||
for_each_domain(cpu, sd) {
|
||||
enum cpu_idle_type itype;
|
||||
|
||||
cpumask_scnprintf(mask_str, mask_len,
|
||||
sched_domain_span(sd));
|
||||
seq_printf(seq, "domain%d %s", dcount++, mask_str);
|
||||
for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES;
|
||||
itype++) {
|
||||
seq_printf(seq, " %u %u %u %u %u %u %u %u",
|
||||
sd->lb_count[itype],
|
||||
sd->lb_balanced[itype],
|
||||
sd->lb_failed[itype],
|
||||
sd->lb_imbalance[itype],
|
||||
sd->lb_gained[itype],
|
||||
sd->lb_hot_gained[itype],
|
||||
sd->lb_nobusyq[itype],
|
||||
sd->lb_nobusyg[itype]);
|
||||
}
|
||||
seq_printf(seq,
|
||||
" %u %u %u %u %u %u %u %u %u %u %u %u\n",
|
||||
sd->alb_count, sd->alb_failed, sd->alb_pushed,
|
||||
sd->sbe_count, sd->sbe_balanced, sd->sbe_pushed,
|
||||
sd->sbf_count, sd->sbf_balanced, sd->sbf_pushed,
|
||||
sd->ttwu_wake_remote, sd->ttwu_move_affine,
|
||||
sd->ttwu_move_balance);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
#endif
|
||||
}
|
||||
kfree(mask_str);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int schedstat_open(struct inode *inode, struct file *file)
|
||||
{
|
||||
unsigned int size = PAGE_SIZE * (1 + num_online_cpus() / 32);
|
||||
char *buf = kmalloc(size, GFP_KERNEL);
|
||||
struct seq_file *m;
|
||||
int res;
|
||||
|
||||
if (!buf)
|
||||
return -ENOMEM;
|
||||
res = single_open(file, show_schedstat, NULL);
|
||||
if (!res) {
|
||||
m = file->private_data;
|
||||
m->buf = buf;
|
||||
m->size = size;
|
||||
} else
|
||||
kfree(buf);
|
||||
return res;
|
||||
}
|
||||
|
||||
static const struct file_operations proc_schedstat_operations = {
|
||||
.open = schedstat_open,
|
||||
.read = seq_read,
|
||||
.llseek = seq_lseek,
|
||||
.release = single_release,
|
||||
};
|
||||
|
||||
static int __init proc_schedstat_init(void)
|
||||
{
|
||||
proc_create("schedstat", 0, NULL, &proc_schedstat_operations);
|
||||
return 0;
|
||||
}
|
||||
module_init(proc_schedstat_init);
|
||||
|
||||
/*
|
||||
* Expects runqueue lock to be held for atomicity of update
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#include "sched.h"
|
||||
|
||||
/*
|
||||
* stop-task scheduling class.
|
||||
*
|
||||
|
|
@ -80,7 +82,7 @@ get_rr_interval_stop(struct rq *rq, struct task_struct *task)
|
|||
/*
|
||||
* Simple, special scheduling class for the per-CPU stop tasks:
|
||||
*/
|
||||
static const struct sched_class stop_sched_class = {
|
||||
const struct sched_class stop_sched_class = {
|
||||
.next = &rt_sched_class,
|
||||
|
||||
.enqueue_task = enqueue_task_stop,
|
||||
|
|
|
|||
Reference in New Issue