dect
/
linux-2.6
Archived
13
0
Fork 0
Code Releases Activity

rcu: Place pointer to call_rcu() in rcu_data structure 

This is a preparatory commit for increasing rcu_barrier()'s concurrency.
It adds a pointer in the rcu_data structure to the corresponding call_rcu()
function.  This allows a pointer to the rcu_data structure to imply the
function pointer, which allows _rcu_barrier() state to be placed in the
rcu_state structure.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
This commit is contained in:
Paul E. McKenney 2012-05-28 23:26:01 -07:00 committed by Paul E. McKenney
parent 6c90cc7bf0
commit 037b64ed0b
3 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
kernel/rcutree.c
View File

@ -62,8 +62,9 @@
static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
#define RCU_STATE_INITIALIZER(sname) { \
#define RCU_STATE_INITIALIZER(sname, cr) { \
.level = { &sname##_state.node[0] }, \
.call = cr, \
.fqs_state = RCU_GP_IDLE, \
.gpnum = -300, \
.completed = -300, \
@ -76,10 +77,11 @@ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
.name = #sname, \
}
struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched);
struct rcu_state rcu_sched_state =
RCU_STATE_INITIALIZER(rcu_sched, call_rcu_sched);
DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh);
struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh);
DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
static struct rcu_state *rcu_state;
@ -2282,21 +2284,17 @@ static void rcu_barrier_func(void *type)
{
int cpu = smp_processor_id();
struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu);
void (*call_rcu_func)(struct rcu_head *head,
void (*func)(struct rcu_head *head));
struct rcu_state *rsp = type;
atomic_inc(&rcu_barrier_cpu_count);
call_rcu_func = type;
call_rcu_func(head, rcu_barrier_callback);
rsp->call(head, rcu_barrier_callback);
}
/*
* Orchestrate the specified type of RCU barrier, waiting for all
* RCU callbacks of the specified type to complete.
*/
static void _rcu_barrier(struct rcu_state *rsp,
void (*call_rcu_func)(struct rcu_head *head,
void (*func)(struct rcu_head *head)))
static void _rcu_barrier(struct rcu_state *rsp)
{
int cpu;
unsigned long flags;
@ -2348,8 +2346,7 @@ static void _rcu_barrier(struct rcu_state *rsp,
while (cpu_is_offline(cpu) && ACCESS_ONCE(rdp->qlen))
schedule_timeout_interruptible(1);
} else if (ACCESS_ONCE(rdp->qlen)) {
smp_call_function_single(cpu, rcu_barrier_func,
(void *)call_rcu_func, 1);
smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
preempt_enable();
} else {
preempt_enable();
@ -2370,7 +2367,7 @@ static void _rcu_barrier(struct rcu_state *rsp,
raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
atomic_inc(&rcu_barrier_cpu_count);
smp_mb__after_atomic_inc(); /* Ensure atomic_inc() before callback. */
call_rcu_func(&rh, rcu_barrier_callback);
rsp->call(&rh, rcu_barrier_callback);
/*
* Now that we have an rcu_barrier_callback() callback on each
@ -2393,7 +2390,7 @@ static void _rcu_barrier(struct rcu_state *rsp,
*/
void rcu_barrier_bh(void)
{
_rcu_barrier(&rcu_bh_state, call_rcu_bh);
_rcu_barrier(&rcu_bh_state);
}
EXPORT_SYMBOL_GPL(rcu_barrier_bh);
@ -2402,7 +2399,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier_bh);
*/
void rcu_barrier_sched(void)
{
_rcu_barrier(&rcu_sched_state, call_rcu_sched);
_rcu_barrier(&rcu_sched_state);
}
EXPORT_SYMBOL_GPL(rcu_barrier_sched);

2
kernel/rcutree.h
View File

@ -364,6 +364,8 @@ struct rcu_state {
u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */
u8 levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */
struct rcu_data __percpu *rda; /* pointer of percu rcu_data. */
void (*call)(struct rcu_head *head, /* call_rcu() flavor. */
void (*func)(struct rcu_head *head));
/* The following fields are guarded by the root rcu_node's lock. */

5
kernel/rcutree_plugin.h
View File

@ -78,7 +78,8 @@ static void __init rcu_bootup_announce_oddness(void)
#ifdef CONFIG_TREE_PREEMPT_RCU
struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt);
struct rcu_state rcu_preempt_state =
RCU_STATE_INITIALIZER(rcu_preempt, call_rcu);
DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
static struct rcu_state *rcu_state = &rcu_preempt_state;
@ -944,7 +945,7 @@ static int rcu_preempt_cpu_has_callbacks(int cpu)
*/
void rcu_barrier(void)
{
_rcu_barrier(&rcu_preempt_state, call_rcu);
_rcu_barrier(&rcu_preempt_state);
}
EXPORT_SYMBOL_GPL(rcu_barrier);