Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 394f99a9 authored by Lai Jiangshan's avatar Lai Jiangshan Committed by Paul E. McKenney
Browse files

rcu: simplify the usage of percpu data 



&percpu_data is compatible with allocated percpu data.

And we use it and remove the "->rda[NR_CPUS]" array, saving significant
storage on systems with large numbers of CPUs.  This does add an additional
level of indirection and thus an additional cache line referenced, but
because ->rda is not used on the read side, this is OK.

Signed-off-by: default avatarLai Jiangshan <laijs@cn.fujitsu.com>
Reviewed-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent e546f485

kernel/rcutree.c

+15 −27
Original line number Original line Diff line number Diff line
@@ -712,7 +712,7 @@ static void 
rcu_start_gp(struct rcu_state *rsp, unsigned long flags)

rcu_start_gp(struct rcu_state *rsp, unsigned long flags)

	__releases(rcu_get_root(rsp)->lock)

	__releases(rcu_get_root(rsp)->lock)

{

{

	struct rcu_data *rdp = rsp->rda[smp_processor_id()];

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);

	struct rcu_node *rnp = rcu_get_root(rsp);

	struct rcu_node *rnp = rcu_get_root(rsp);





	if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) {

	if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) {
@@ -960,7 +960,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 
static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)

static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)

{

{

	int i;

	int i;

	struct rcu_data *rdp = rsp->rda[smp_processor_id()];

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);





	if (rdp->nxtlist == NULL)

	if (rdp->nxtlist == NULL)

		return;  /* irqs disabled, so comparison is stable. */

		return;  /* irqs disabled, so comparison is stable. */
@@ -984,7 +984,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp) 
	struct rcu_data *rdp;

	struct rcu_data *rdp;





	raw_spin_lock_irqsave(&rsp->onofflock, flags);

	raw_spin_lock_irqsave(&rsp->onofflock, flags);

	rdp = rsp->rda[smp_processor_id()];

	rdp = this_cpu_ptr(rsp->rda);

	if (rsp->orphan_cbs_list == NULL) {

	if (rsp->orphan_cbs_list == NULL) {

		raw_spin_unlock_irqrestore(&rsp->onofflock, flags);

		raw_spin_unlock_irqrestore(&rsp->onofflock, flags);

		return;

		return;
@@ -1007,7 +1007,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) 
	unsigned long flags;

	unsigned long flags;

	unsigned long mask;

	unsigned long mask;

	int need_report = 0;

	int need_report = 0;

	struct rcu_data *rdp = rsp->rda[cpu];

	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);

	struct rcu_node *rnp;

	struct rcu_node *rnp;





	/* Exclude any attempts to start a new grace period. */

	/* Exclude any attempts to start a new grace period. */
@@ -1226,7 +1226,8 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *)) 
		cpu = rnp->grplo;

		cpu = rnp->grplo;

		bit = 1;

		bit = 1;

		for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {

		for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {

			if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))

			if ((rnp->qsmask & bit) != 0 &&

			    f(per_cpu_ptr(rsp->rda, cpu)))

				mask |= bit;

				mask |= bit;

		}

		}

		if (mask != 0) {

		if (mask != 0) {
@@ -1402,7 +1403,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	 * a quiescent state betweentimes.

	 * a quiescent state betweentimes.

	 */

	 */

	local_irq_save(flags);

	local_irq_save(flags);

	rdp = rsp->rda[smp_processor_id()];

	rdp = this_cpu_ptr(rsp->rda);

	rcu_process_gp_end(rsp, rdp);

	rcu_process_gp_end(rsp, rdp);

	check_for_new_grace_period(rsp, rdp);

	check_for_new_grace_period(rsp, rdp);
@@ -1701,7 +1702,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) 
{

{

	unsigned long flags;

	unsigned long flags;

	int i;

	int i;

	struct rcu_data *rdp = rsp->rda[cpu];

	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);

	struct rcu_node *rnp = rcu_get_root(rsp);

	struct rcu_node *rnp = rcu_get_root(rsp);





	/* Set up local state, ensuring consistent view of global state. */

	/* Set up local state, ensuring consistent view of global state. */
@@ -1729,7 +1730,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable) 
{

{

	unsigned long flags;

	unsigned long flags;

	unsigned long mask;

	unsigned long mask;

	struct rcu_data *rdp = rsp->rda[cpu];

	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);

	struct rcu_node *rnp = rcu_get_root(rsp);

	struct rcu_node *rnp = rcu_get_root(rsp);





	/* Set up local state, ensuring consistent view of global state. */

	/* Set up local state, ensuring consistent view of global state. */
@@ -1865,7 +1866,8 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp) 
/*

/*

 * Helper function for rcu_init() that initializes one rcu_state structure.

 * Helper function for rcu_init() that initializes one rcu_state structure.

 */

 */

static void __init rcu_init_one(struct rcu_state *rsp)

static void __init rcu_init_one(struct rcu_state *rsp,

		struct rcu_data __percpu *rda)

{

{

	static char *buf[] = { "rcu_node_level_0",

	static char *buf[] = { "rcu_node_level_0",

			       "rcu_node_level_1",

			       "rcu_node_level_1",
@@ -1918,37 +1920,23 @@ static void __init rcu_init_one(struct rcu_state *rsp) 
		}

		}

	}

	}





	rsp->rda = rda;

	rnp = rsp->level[NUM_RCU_LVLS - 1];

	rnp = rsp->level[NUM_RCU_LVLS - 1];

	for_each_possible_cpu(i) {

	for_each_possible_cpu(i) {

		while (i > rnp->grphi)

		while (i > rnp->grphi)

			rnp++;

			rnp++;

		rsp->rda[i]->mynode = rnp;

		per_cpu_ptr(rsp->rda, i)->mynode = rnp;

		rcu_boot_init_percpu_data(i, rsp);

		rcu_boot_init_percpu_data(i, rsp);

	}

	}

}

}





/*

 * Helper macro for __rcu_init() and __rcu_init_preempt().  To be used

 * nowhere else!  Assigns leaf node pointers into each CPU's rcu_data

 * structure.

 */

#define RCU_INIT_FLAVOR(rsp, rcu_data) \

do { \

	int i; \

	\

	for_each_possible_cpu(i) { \

		(rsp)->rda[i] = &per_cpu(rcu_data, i); \

	} \

	rcu_init_one(rsp); \

} while (0)



void __init rcu_init(void)

void __init rcu_init(void)

{

{

	int cpu;

	int cpu;





	rcu_bootup_announce();

	rcu_bootup_announce();

	RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);

	rcu_init_one(&rcu_sched_state, &rcu_sched_data);

	RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);

	rcu_init_one(&rcu_bh_state, &rcu_bh_data);

	__rcu_init_preempt();

	__rcu_init_preempt();

	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);

	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);

kernel/rcutree.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -283,7 +283,7 @@ struct rcu_state { 
	struct rcu_node *level[NUM_RCU_LVLS];	/* Hierarchy levels. */

	struct rcu_node *level[NUM_RCU_LVLS];	/* Hierarchy levels. */

	u32 levelcnt[MAX_RCU_LVLS + 1];		/* # nodes in each level. */

	u32 levelcnt[MAX_RCU_LVLS + 1];		/* # nodes in each level. */

	u8 levelspread[NUM_RCU_LVLS];		/* kids/node in each level. */

	u8 levelspread[NUM_RCU_LVLS];		/* kids/node in each level. */

	struct rcu_data *rda[NR_CPUS];		/* array of rdp pointers. */

	struct rcu_data __percpu *rda;		/* pointer of percu rcu_data. */





	/* The following fields are guarded by the root rcu_node's lock. */

	/* The following fields are guarded by the root rcu_node's lock. */

kernel/rcutree_plugin.h

+2 −2
Original line number Original line Diff line number Diff line
@@ -154,7 +154,7 @@ static void rcu_preempt_note_context_switch(int cpu) 
	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {

	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {





		/* Possibly blocking in an RCU read-side critical section. */

		/* Possibly blocking in an RCU read-side critical section. */

		rdp = rcu_preempt_state.rda[cpu];

		rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);

		rnp = rdp->mynode;

		rnp = rdp->mynode;

		raw_spin_lock_irqsave(&rnp->lock, flags);

		raw_spin_lock_irqsave(&rnp->lock, flags);

		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;

		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
@@ -771,7 +771,7 @@ static void rcu_preempt_send_cbs_to_orphanage(void) 
 */

 */

static void __init __rcu_init_preempt(void)

static void __init __rcu_init_preempt(void)

{

{

	RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);

	rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);

}

}





/*

/*

kernel/rcutree_trace.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -262,7 +262,7 @@ static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp) 
	struct rcu_data *rdp;

	struct rcu_data *rdp;





	for_each_possible_cpu(cpu) {

	for_each_possible_cpu(cpu) {

		rdp = rsp->rda[cpu];

		rdp = per_cpu_ptr(rsp->rda, cpu);

		if (rdp->beenonline)

		if (rdp->beenonline)

			print_one_rcu_pending(m, rdp);

			print_one_rcu_pending(m, rdp);

	}

	}