blkcg: move blkio_group_stats_cpu and friends to blk-throttle.c
blkio_group_stats_cpu is used only by blk-throtl and has no reason to be defined in blkcg core. * Move blkio_group_stats_cpu to blk-throttle.c and rename it to tg_stats_cpu. * blkg_policy_data->stats_cpu is replaced with throtl_grp->stats_cpu. prfill functions updated accordingly. * All related macros / functions are renamed so that they have tg_ prefix and the unnecessary @pol arguments are dropped. * Per-cpu stats allocation code is also moved from blk-cgroup.c to blk-throttle.c and gets simplified to only deal with BLKIO_POLICY_THROTL. percpu stat free is performed by the exit method throtl_exit_blkio_group(). * throtl_reset_group_stats() implemented for blkio_reset_group_stats_fn method so that tg->stats_cpu can be reset. Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
parent
155fead9b6
commit
8a3d26151f
|
@ -30,13 +30,6 @@ static LIST_HEAD(blkio_list);
|
||||||
static DEFINE_MUTEX(all_q_mutex);
|
static DEFINE_MUTEX(all_q_mutex);
|
||||||
static LIST_HEAD(all_q_list);
|
static LIST_HEAD(all_q_list);
|
||||||
|
|
||||||
/* List of groups pending per cpu stats allocation */
|
|
||||||
static DEFINE_SPINLOCK(alloc_list_lock);
|
|
||||||
static LIST_HEAD(alloc_list);
|
|
||||||
|
|
||||||
static void blkio_stat_alloc_fn(struct work_struct *);
|
|
||||||
static DECLARE_DELAYED_WORK(blkio_stat_alloc_work, blkio_stat_alloc_fn);
|
|
||||||
|
|
||||||
struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
|
struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
|
||||||
EXPORT_SYMBOL_GPL(blkio_root_cgroup);
|
EXPORT_SYMBOL_GPL(blkio_root_cgroup);
|
||||||
|
|
||||||
|
@ -63,60 +56,6 @@ struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(bio_blkio_cgroup);
|
EXPORT_SYMBOL_GPL(bio_blkio_cgroup);
|
||||||
|
|
||||||
/*
|
|
||||||
* Worker for allocating per cpu stat for blk groups. This is scheduled on
|
|
||||||
* the system_nrt_wq once there are some groups on the alloc_list waiting
|
|
||||||
* for allocation.
|
|
||||||
*/
|
|
||||||
static void blkio_stat_alloc_fn(struct work_struct *work)
|
|
||||||
{
|
|
||||||
static void *pcpu_stats[BLKIO_NR_POLICIES];
|
|
||||||
struct delayed_work *dwork = to_delayed_work(work);
|
|
||||||
struct blkio_group *blkg;
|
|
||||||
int i;
|
|
||||||
bool empty = false;
|
|
||||||
|
|
||||||
alloc_stats:
|
|
||||||
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
||||||
if (pcpu_stats[i] != NULL)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
pcpu_stats[i] = alloc_percpu(struct blkio_group_stats_cpu);
|
|
||||||
|
|
||||||
/* Allocation failed. Try again after some time. */
|
|
||||||
if (pcpu_stats[i] == NULL) {
|
|
||||||
queue_delayed_work(system_nrt_wq, dwork,
|
|
||||||
msecs_to_jiffies(10));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
spin_lock_irq(&blkio_list_lock);
|
|
||||||
spin_lock(&alloc_list_lock);
|
|
||||||
|
|
||||||
/* cgroup got deleted or queue exited. */
|
|
||||||
if (!list_empty(&alloc_list)) {
|
|
||||||
blkg = list_first_entry(&alloc_list, struct blkio_group,
|
|
||||||
alloc_node);
|
|
||||||
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
||||||
struct blkg_policy_data *pd = blkg->pd[i];
|
|
||||||
|
|
||||||
if (blkio_policy[i] && pd && !pd->stats_cpu)
|
|
||||||
swap(pd->stats_cpu, pcpu_stats[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
list_del_init(&blkg->alloc_node);
|
|
||||||
}
|
|
||||||
|
|
||||||
empty = list_empty(&alloc_list);
|
|
||||||
|
|
||||||
spin_unlock(&alloc_list_lock);
|
|
||||||
spin_unlock_irq(&blkio_list_lock);
|
|
||||||
|
|
||||||
if (!empty)
|
|
||||||
goto alloc_stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* blkg_free - free a blkg
|
* blkg_free - free a blkg
|
||||||
* @blkg: blkg to free
|
* @blkg: blkg to free
|
||||||
|
@ -140,7 +79,6 @@ static void blkg_free(struct blkio_group *blkg)
|
||||||
if (pol && pol->ops.blkio_exit_group_fn)
|
if (pol && pol->ops.blkio_exit_group_fn)
|
||||||
pol->ops.blkio_exit_group_fn(blkg);
|
pol->ops.blkio_exit_group_fn(blkg);
|
||||||
|
|
||||||
free_percpu(pd->stats_cpu);
|
|
||||||
kfree(pd);
|
kfree(pd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -167,7 +105,6 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
|
||||||
|
|
||||||
blkg->q = q;
|
blkg->q = q;
|
||||||
INIT_LIST_HEAD(&blkg->q_node);
|
INIT_LIST_HEAD(&blkg->q_node);
|
||||||
INIT_LIST_HEAD(&blkg->alloc_node);
|
|
||||||
blkg->blkcg = blkcg;
|
blkg->blkcg = blkcg;
|
||||||
blkg->refcnt = 1;
|
blkg->refcnt = 1;
|
||||||
cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
|
cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
|
||||||
|
@ -245,12 +182,6 @@ struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
|
||||||
hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
|
hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
|
||||||
list_add(&blkg->q_node, &q->blkg_list);
|
list_add(&blkg->q_node, &q->blkg_list);
|
||||||
spin_unlock(&blkcg->lock);
|
spin_unlock(&blkcg->lock);
|
||||||
|
|
||||||
spin_lock(&alloc_list_lock);
|
|
||||||
list_add(&blkg->alloc_node, &alloc_list);
|
|
||||||
/* Queue per cpu stat allocation from worker thread. */
|
|
||||||
queue_delayed_work(system_nrt_wq, &blkio_stat_alloc_work, 0);
|
|
||||||
spin_unlock(&alloc_list_lock);
|
|
||||||
out:
|
out:
|
||||||
return blkg;
|
return blkg;
|
||||||
}
|
}
|
||||||
|
@ -284,10 +215,6 @@ static void blkg_destroy(struct blkio_group *blkg)
|
||||||
list_del_init(&blkg->q_node);
|
list_del_init(&blkg->q_node);
|
||||||
hlist_del_init_rcu(&blkg->blkcg_node);
|
hlist_del_init_rcu(&blkg->blkcg_node);
|
||||||
|
|
||||||
spin_lock(&alloc_list_lock);
|
|
||||||
list_del_init(&blkg->alloc_node);
|
|
||||||
spin_unlock(&alloc_list_lock);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Put the reference taken at the time of creation so that when all
|
* Put the reference taken at the time of creation so that when all
|
||||||
* queues are gone, group can be destroyed.
|
* queues are gone, group can be destroyed.
|
||||||
|
@ -319,9 +246,6 @@ void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid)
|
||||||
pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
|
pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
|
||||||
WARN_ON_ONCE(!pd);
|
WARN_ON_ONCE(!pd);
|
||||||
|
|
||||||
pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
|
|
||||||
WARN_ON_ONCE(!pd->stats_cpu);
|
|
||||||
|
|
||||||
blkg->pd[plid] = pd;
|
blkg->pd[plid] = pd;
|
||||||
pd->blkg = blkg;
|
pd->blkg = blkg;
|
||||||
pol->ops.blkio_init_group_fn(blkg);
|
pol->ops.blkio_init_group_fn(blkg);
|
||||||
|
@ -381,23 +305,6 @@ void __blkg_release(struct blkio_group *blkg)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(__blkg_release);
|
EXPORT_SYMBOL_GPL(__blkg_release);
|
||||||
|
|
||||||
static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
|
|
||||||
{
|
|
||||||
struct blkg_policy_data *pd = blkg->pd[plid];
|
|
||||||
int cpu;
|
|
||||||
|
|
||||||
if (pd->stats_cpu == NULL)
|
|
||||||
return;
|
|
||||||
|
|
||||||
for_each_possible_cpu(cpu) {
|
|
||||||
struct blkio_group_stats_cpu *sc =
|
|
||||||
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
||||||
|
|
||||||
blkg_rwstat_reset(&sc->service_bytes);
|
|
||||||
blkg_rwstat_reset(&sc->serviced);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
static int
|
||||||
blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
|
blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
|
||||||
{
|
{
|
||||||
|
@ -416,12 +323,9 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
|
||||||
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
|
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
|
||||||
struct blkio_policy_type *pol;
|
struct blkio_policy_type *pol;
|
||||||
|
|
||||||
list_for_each_entry(pol, &blkio_list, list) {
|
list_for_each_entry(pol, &blkio_list, list)
|
||||||
blkio_reset_stats_cpu(blkg, pol->plid);
|
|
||||||
|
|
||||||
if (pol->ops.blkio_reset_group_stats_fn)
|
if (pol->ops.blkio_reset_group_stats_fn)
|
||||||
pol->ops.blkio_reset_group_stats_fn(blkg);
|
pol->ops.blkio_reset_group_stats_fn(blkg);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_unlock_irq(&blkcg->lock);
|
spin_unlock_irq(&blkcg->lock);
|
||||||
|
|
|
@ -64,14 +64,6 @@ struct blkg_rwstat {
|
||||||
uint64_t cnt[BLKG_RWSTAT_NR];
|
uint64_t cnt[BLKG_RWSTAT_NR];
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Per cpu blkio group stats */
|
|
||||||
struct blkio_group_stats_cpu {
|
|
||||||
/* total bytes transferred */
|
|
||||||
struct blkg_rwstat service_bytes;
|
|
||||||
/* total IOs serviced, post merge */
|
|
||||||
struct blkg_rwstat serviced;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct blkio_group_conf {
|
struct blkio_group_conf {
|
||||||
unsigned int weight;
|
unsigned int weight;
|
||||||
u64 iops[2];
|
u64 iops[2];
|
||||||
|
@ -86,9 +78,6 @@ struct blkg_policy_data {
|
||||||
/* Configuration */
|
/* Configuration */
|
||||||
struct blkio_group_conf conf;
|
struct blkio_group_conf conf;
|
||||||
|
|
||||||
/* Per cpu stats pointer */
|
|
||||||
struct blkio_group_stats_cpu __percpu *stats_cpu;
|
|
||||||
|
|
||||||
/* pol->pdata_size bytes of private data used by policy impl */
|
/* pol->pdata_size bytes of private data used by policy impl */
|
||||||
char pdata[] __aligned(__alignof__(unsigned long long));
|
char pdata[] __aligned(__alignof__(unsigned long long));
|
||||||
};
|
};
|
||||||
|
@ -106,8 +95,6 @@ struct blkio_group {
|
||||||
|
|
||||||
struct blkg_policy_data *pd[BLKIO_NR_POLICIES];
|
struct blkg_policy_data *pd[BLKIO_NR_POLICIES];
|
||||||
|
|
||||||
/* List of blkg waiting for per cpu stats memory to be allocated */
|
|
||||||
struct list_head alloc_node;
|
|
||||||
struct rcu_head rcu_head;
|
struct rcu_head rcu_head;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -40,6 +40,14 @@ struct throtl_rb_root {
|
||||||
|
|
||||||
#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
|
#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
|
||||||
|
|
||||||
|
/* Per-cpu group stats */
|
||||||
|
struct tg_stats_cpu {
|
||||||
|
/* total bytes transferred */
|
||||||
|
struct blkg_rwstat service_bytes;
|
||||||
|
/* total IOs serviced, post merge */
|
||||||
|
struct blkg_rwstat serviced;
|
||||||
|
};
|
||||||
|
|
||||||
struct throtl_grp {
|
struct throtl_grp {
|
||||||
/* active throtl group service_tree member */
|
/* active throtl group service_tree member */
|
||||||
struct rb_node rb_node;
|
struct rb_node rb_node;
|
||||||
|
@ -76,6 +84,12 @@ struct throtl_grp {
|
||||||
|
|
||||||
/* Some throttle limits got updated for the group */
|
/* Some throttle limits got updated for the group */
|
||||||
int limits_changed;
|
int limits_changed;
|
||||||
|
|
||||||
|
/* Per cpu stats pointer */
|
||||||
|
struct tg_stats_cpu __percpu *stats_cpu;
|
||||||
|
|
||||||
|
/* List of tgs waiting for per cpu stats memory to be allocated */
|
||||||
|
struct list_head stats_alloc_node;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct throtl_data
|
struct throtl_data
|
||||||
|
@ -100,6 +114,13 @@ struct throtl_data
|
||||||
int limits_changed;
|
int limits_changed;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* list and work item to allocate percpu group stats */
|
||||||
|
static DEFINE_SPINLOCK(tg_stats_alloc_lock);
|
||||||
|
static LIST_HEAD(tg_stats_alloc_list);
|
||||||
|
|
||||||
|
static void tg_stats_alloc_fn(struct work_struct *);
|
||||||
|
static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
|
||||||
|
|
||||||
static inline struct throtl_grp *blkg_to_tg(struct blkio_group *blkg)
|
static inline struct throtl_grp *blkg_to_tg(struct blkio_group *blkg)
|
||||||
{
|
{
|
||||||
return blkg_to_pdata(blkg, &blkio_policy_throtl);
|
return blkg_to_pdata(blkg, &blkio_policy_throtl);
|
||||||
|
@ -142,6 +163,44 @@ static inline unsigned int total_nr_queued(struct throtl_data *td)
|
||||||
return td->nr_queued[0] + td->nr_queued[1];
|
return td->nr_queued[0] + td->nr_queued[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Worker for allocating per cpu stat for tgs. This is scheduled on the
|
||||||
|
* system_nrt_wq once there are some groups on the alloc_list waiting for
|
||||||
|
* allocation.
|
||||||
|
*/
|
||||||
|
static void tg_stats_alloc_fn(struct work_struct *work)
|
||||||
|
{
|
||||||
|
static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */
|
||||||
|
struct delayed_work *dwork = to_delayed_work(work);
|
||||||
|
bool empty = false;
|
||||||
|
|
||||||
|
alloc_stats:
|
||||||
|
if (!stats_cpu) {
|
||||||
|
stats_cpu = alloc_percpu(struct tg_stats_cpu);
|
||||||
|
if (!stats_cpu) {
|
||||||
|
/* allocation failed, try again after some time */
|
||||||
|
queue_delayed_work(system_nrt_wq, dwork,
|
||||||
|
msecs_to_jiffies(10));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spin_lock_irq(&tg_stats_alloc_lock);
|
||||||
|
|
||||||
|
if (!list_empty(&tg_stats_alloc_list)) {
|
||||||
|
struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,
|
||||||
|
struct throtl_grp,
|
||||||
|
stats_alloc_node);
|
||||||
|
swap(tg->stats_cpu, stats_cpu);
|
||||||
|
list_del_init(&tg->stats_alloc_node);
|
||||||
|
}
|
||||||
|
|
||||||
|
empty = list_empty(&tg_stats_alloc_list);
|
||||||
|
spin_unlock_irq(&tg_stats_alloc_lock);
|
||||||
|
if (!empty)
|
||||||
|
goto alloc_stats;
|
||||||
|
}
|
||||||
|
|
||||||
static void throtl_init_blkio_group(struct blkio_group *blkg)
|
static void throtl_init_blkio_group(struct blkio_group *blkg)
|
||||||
{
|
{
|
||||||
struct throtl_grp *tg = blkg_to_tg(blkg);
|
struct throtl_grp *tg = blkg_to_tg(blkg);
|
||||||
|
@ -155,6 +214,43 @@ static void throtl_init_blkio_group(struct blkio_group *blkg)
|
||||||
tg->bps[WRITE] = -1;
|
tg->bps[WRITE] = -1;
|
||||||
tg->iops[READ] = -1;
|
tg->iops[READ] = -1;
|
||||||
tg->iops[WRITE] = -1;
|
tg->iops[WRITE] = -1;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Ugh... We need to perform per-cpu allocation for tg->stats_cpu
|
||||||
|
* but percpu allocator can't be called from IO path. Queue tg on
|
||||||
|
* tg_stats_alloc_list and allocate from work item.
|
||||||
|
*/
|
||||||
|
spin_lock(&tg_stats_alloc_lock);
|
||||||
|
list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);
|
||||||
|
queue_delayed_work(system_nrt_wq, &tg_stats_alloc_work, 0);
|
||||||
|
spin_unlock(&tg_stats_alloc_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void throtl_exit_blkio_group(struct blkio_group *blkg)
|
||||||
|
{
|
||||||
|
struct throtl_grp *tg = blkg_to_tg(blkg);
|
||||||
|
|
||||||
|
spin_lock(&tg_stats_alloc_lock);
|
||||||
|
list_del_init(&tg->stats_alloc_node);
|
||||||
|
spin_unlock(&tg_stats_alloc_lock);
|
||||||
|
|
||||||
|
free_percpu(tg->stats_cpu);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void throtl_reset_group_stats(struct blkio_group *blkg)
|
||||||
|
{
|
||||||
|
struct throtl_grp *tg = blkg_to_tg(blkg);
|
||||||
|
int cpu;
|
||||||
|
|
||||||
|
if (tg->stats_cpu == NULL)
|
||||||
|
return;
|
||||||
|
|
||||||
|
for_each_possible_cpu(cpu) {
|
||||||
|
struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
|
||||||
|
|
||||||
|
blkg_rwstat_reset(&sc->service_bytes);
|
||||||
|
blkg_rwstat_reset(&sc->serviced);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct
|
static struct
|
||||||
|
@ -565,12 +661,12 @@ static bool tg_may_dispatch(struct throtl_data *td, struct throtl_grp *tg,
|
||||||
static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,
|
static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,
|
||||||
int rw)
|
int rw)
|
||||||
{
|
{
|
||||||
struct blkg_policy_data *pd = blkg->pd[BLKIO_POLICY_THROTL];
|
struct throtl_grp *tg = blkg_to_tg(blkg);
|
||||||
struct blkio_group_stats_cpu *stats_cpu;
|
struct tg_stats_cpu *stats_cpu;
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
/* If per cpu stats are not allocated yet, don't do any accounting. */
|
/* If per cpu stats are not allocated yet, don't do any accounting. */
|
||||||
if (pd->stats_cpu == NULL)
|
if (tg->stats_cpu == NULL)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -580,7 +676,7 @@ static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,
|
||||||
*/
|
*/
|
||||||
local_irq_save(flags);
|
local_irq_save(flags);
|
||||||
|
|
||||||
stats_cpu = this_cpu_ptr(pd->stats_cpu);
|
stats_cpu = this_cpu_ptr(tg->stats_cpu);
|
||||||
|
|
||||||
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
||||||
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
||||||
|
@ -842,15 +938,15 @@ static void throtl_update_blkio_group_common(struct throtl_data *td,
|
||||||
throtl_schedule_delayed_work(td, 0);
|
throtl_schedule_delayed_work(td, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
|
static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
|
||||||
struct blkg_policy_data *pd, int off)
|
struct blkg_policy_data *pd, int off)
|
||||||
{
|
{
|
||||||
|
struct throtl_grp *tg = (void *)pd->pdata;
|
||||||
struct blkg_rwstat rwstat = { }, tmp;
|
struct blkg_rwstat rwstat = { }, tmp;
|
||||||
int i, cpu;
|
int i, cpu;
|
||||||
|
|
||||||
for_each_possible_cpu(cpu) {
|
for_each_possible_cpu(cpu) {
|
||||||
struct blkio_group_stats_cpu *sc =
|
struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
|
||||||
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
||||||
|
|
||||||
tmp = blkg_rwstat_read((void *)sc + off);
|
tmp = blkg_rwstat_read((void *)sc + off);
|
||||||
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
||||||
|
@ -861,12 +957,12 @@ static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
|
||||||
}
|
}
|
||||||
|
|
||||||
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
||||||
static int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
||||||
struct seq_file *sf)
|
struct seq_file *sf)
|
||||||
{
|
{
|
||||||
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
||||||
|
|
||||||
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,
|
blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat,
|
||||||
BLKCG_STAT_POL(cft->private),
|
BLKCG_STAT_POL(cft->private),
|
||||||
BLKCG_STAT_OFF(cft->private), true);
|
BLKCG_STAT_OFF(cft->private), true);
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -1012,14 +1108,14 @@ static struct cftype throtl_files[] = {
|
||||||
{
|
{
|
||||||
.name = "throttle.io_service_bytes",
|
.name = "throttle.io_service_bytes",
|
||||||
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
||||||
offsetof(struct blkio_group_stats_cpu, service_bytes)),
|
offsetof(struct tg_stats_cpu, service_bytes)),
|
||||||
.read_seq_string = blkcg_print_cpu_rwstat,
|
.read_seq_string = tg_print_cpu_rwstat,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.name = "throttle.io_serviced",
|
.name = "throttle.io_serviced",
|
||||||
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
||||||
offsetof(struct blkio_group_stats_cpu, serviced)),
|
offsetof(struct tg_stats_cpu, serviced)),
|
||||||
.read_seq_string = blkcg_print_cpu_rwstat,
|
.read_seq_string = tg_print_cpu_rwstat,
|
||||||
},
|
},
|
||||||
{ } /* terminate */
|
{ } /* terminate */
|
||||||
};
|
};
|
||||||
|
@ -1034,6 +1130,8 @@ static void throtl_shutdown_wq(struct request_queue *q)
|
||||||
static struct blkio_policy_type blkio_policy_throtl = {
|
static struct blkio_policy_type blkio_policy_throtl = {
|
||||||
.ops = {
|
.ops = {
|
||||||
.blkio_init_group_fn = throtl_init_blkio_group,
|
.blkio_init_group_fn = throtl_init_blkio_group,
|
||||||
|
.blkio_exit_group_fn = throtl_exit_blkio_group,
|
||||||
|
.blkio_reset_group_stats_fn = throtl_reset_group_stats,
|
||||||
},
|
},
|
||||||
.plid = BLKIO_POLICY_THROTL,
|
.plid = BLKIO_POLICY_THROTL,
|
||||||
.pdata_size = sizeof(struct throtl_grp),
|
.pdata_size = sizeof(struct throtl_grp),
|
||||||
|
|
Reference in New Issue