Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5b74c458 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Make ->cpu_no_qs be a union for aggregate OR 



This commit converts the rcu_data structure's ->cpu_no_qs field
to a union.  The bytewise side of this union allows individual access
to indications as to whether this CPU needs to find a quiescent state
for a normal (.norm) and/or expedited (.exp) grace period.  The setwise
side of the union allows testing whether or not a quiescent state is
needed at all, for either type of grace period.

For now, only .norm is used.  A later commit will introduce the expedited
usage.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 0d43eb34

kernel/rcu/tree.c

+11 −11
Original line number Original line Diff line number Diff line
@@ -245,21 +245,21 @@ static int rcu_gp_in_progress(struct rcu_state *rsp) 
 */

 */

void rcu_sched_qs(void)

void rcu_sched_qs(void)

{

{

	if (__this_cpu_read(rcu_sched_data.cpu_no_qs)) {

	if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {

		trace_rcu_grace_period(TPS("rcu_sched"),

		trace_rcu_grace_period(TPS("rcu_sched"),

				       __this_cpu_read(rcu_sched_data.gpnum),

				       __this_cpu_read(rcu_sched_data.gpnum),

				       TPS("cpuqs"));

				       TPS("cpuqs"));

		__this_cpu_write(rcu_sched_data.cpu_no_qs, false);

		__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);

	}

	}

}

}





void rcu_bh_qs(void)

void rcu_bh_qs(void)

{

{

	if (__this_cpu_read(rcu_bh_data.cpu_no_qs)) {

	if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {

		trace_rcu_grace_period(TPS("rcu_bh"),

		trace_rcu_grace_period(TPS("rcu_bh"),

				       __this_cpu_read(rcu_bh_data.gpnum),

				       __this_cpu_read(rcu_bh_data.gpnum),

				       TPS("cpuqs"));

				       TPS("cpuqs"));

		__this_cpu_write(rcu_bh_data.cpu_no_qs, false);

		__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);

	}

	}

}

}
@@ -1744,7 +1744,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, 
		 */

		 */

		rdp->gpnum = rnp->gpnum;

		rdp->gpnum = rnp->gpnum;

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));

		rdp->cpu_no_qs = true;

		rdp->cpu_no_qs.b.norm = true;

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);

		rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);

		zero_cpu_stall_ticks(rdp);

		zero_cpu_stall_ticks(rdp);
@@ -2337,7 +2337,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) 
	rnp = rdp->mynode;

	rnp = rdp->mynode;

	raw_spin_lock_irqsave(&rnp->lock, flags);

	raw_spin_lock_irqsave(&rnp->lock, flags);

	smp_mb__after_unlock_lock();

	smp_mb__after_unlock_lock();

	if ((rdp->cpu_no_qs &&

	if ((rdp->cpu_no_qs.b.norm &&

	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||

	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||

	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||

	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||

	    rdp->gpwrap) {

	    rdp->gpwrap) {
@@ -2348,7 +2348,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) 
		 * We will instead need a new quiescent state that lies

		 * We will instead need a new quiescent state that lies

		 * within the current grace period.

		 * within the current grace period.

		 */

		 */

		rdp->cpu_no_qs = true;	/* need qs for new gp. */

		rdp->cpu_no_qs.b.norm = true;	/* need qs for new gp. */

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		raw_spin_unlock_irqrestore(&rnp->lock, flags);

		raw_spin_unlock_irqrestore(&rnp->lock, flags);

		return;

		return;
@@ -2395,7 +2395,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 
	 * Was there a quiescent state since the beginning of the grace

	 * Was there a quiescent state since the beginning of the grace

	 * period? If no, then exit and wait for the next call.

	 * period? If no, then exit and wait for the next call.

	 */

	 */

	if (rdp->cpu_no_qs &&

	if (rdp->cpu_no_qs.b.norm &&

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))

		return;

		return;
@@ -3828,11 +3828,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) 




	/* Is the RCU core waiting for a quiescent state from this CPU? */

	/* Is the RCU core waiting for a quiescent state from this CPU? */

	if (rcu_scheduler_fully_active &&

	if (rcu_scheduler_fully_active &&

	    rdp->core_needs_qs && rdp->cpu_no_qs &&

	    rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {

		rdp->n_rp_core_needs_qs++;

		rdp->n_rp_core_needs_qs++;

	} else if (rdp->core_needs_qs &&

	} else if (rdp->core_needs_qs &&

		   (!rdp->cpu_no_qs ||

		   (!rdp->cpu_no_qs.b.norm ||

		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {

		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {

		rdp->n_rp_report_qs++;

		rdp->n_rp_report_qs++;

		return 1;

		return 1;
@@ -4155,7 +4155,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) 
	rdp->beenonline = true;	 /* We have now been online. */

	rdp->beenonline = true;	 /* We have now been online. */

	rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */

	rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */

	rdp->completed = rnp->completed;

	rdp->completed = rnp->completed;

	rdp->cpu_no_qs = true;

	rdp->cpu_no_qs.b.norm = true;

	rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu);

	rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu);

	rdp->core_needs_qs = false;

	rdp->core_needs_qs = false;

	trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

	trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

kernel/rcu/tree.h

+13 −1
Original line number Original line Diff line number Diff line
@@ -286,6 +286,18 @@ struct rcu_node { 
	for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \

	for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \

	     (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)

	     (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)





/*

 * Union to allow "aggregate OR" operation on the need for a quiescent

 * state by the normal and expedited grace periods.

 */

union rcu_noqs {

	struct {

		u8 norm;

		u8 exp;

	} b; /* Bits. */

	u16 s; /* Set of bits, aggregate OR here. */

};



/* Index values for nxttail array in struct rcu_data. */

/* Index values for nxttail array in struct rcu_data. */

#define RCU_DONE_TAIL		0	/* Also RCU_WAIT head. */

#define RCU_DONE_TAIL		0	/* Also RCU_WAIT head. */

#define RCU_WAIT_TAIL		1	/* Also RCU_NEXT_READY head. */

#define RCU_WAIT_TAIL		1	/* Also RCU_NEXT_READY head. */
@@ -302,7 +314,7 @@ struct rcu_data { 
					/*  is aware of having started. */

					/*  is aware of having started. */

	unsigned long	rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */

	unsigned long	rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */

					/*  for rcu_all_qs() invocations. */

					/*  for rcu_all_qs() invocations. */

	bool		cpu_no_qs;	/* No QS yet for this CPU. */

	union rcu_noqs	cpu_no_qs;	/* No QSes yet for this CPU. */

	bool		core_needs_qs;	/* Core waits for quiesc state. */

	bool		core_needs_qs;	/* Core waits for quiesc state. */

	bool		beenonline;	/* CPU online at least once. */

	bool		beenonline;	/* CPU online at least once. */

	bool		gpwrap;		/* Possible gpnum/completed wrap. */

	bool		gpwrap;		/* Possible gpnum/completed wrap. */

kernel/rcu/tree_plugin.h

+3 −3
Original line number Original line Diff line number Diff line
@@ -265,11 +265,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp, 
 */

 */

static void rcu_preempt_qs(void)

static void rcu_preempt_qs(void)

{

{

	if (__this_cpu_read(rcu_data_p->cpu_no_qs)) {

	if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {

		trace_rcu_grace_period(TPS("rcu_preempt"),

		trace_rcu_grace_period(TPS("rcu_preempt"),

				       __this_cpu_read(rcu_data_p->gpnum),

				       __this_cpu_read(rcu_data_p->gpnum),

				       TPS("cpuqs"));

				       TPS("cpuqs"));

		__this_cpu_write(rcu_data_p->cpu_no_qs, false);

		__this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);

		barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */

		barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */

		current->rcu_read_unlock_special.b.need_qs = false;

		current->rcu_read_unlock_special.b.need_qs = false;

	}

	}
@@ -620,7 +620,7 @@ static void rcu_preempt_check_callbacks(void) 
	}

	}

	if (t->rcu_read_lock_nesting > 0 &&

	if (t->rcu_read_lock_nesting > 0 &&

	    __this_cpu_read(rcu_data_p->core_needs_qs) &&

	    __this_cpu_read(rcu_data_p->core_needs_qs) &&

	    __this_cpu_read(rcu_data_p->cpu_no_qs))

	    __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))

		t->rcu_read_unlock_special.b.need_qs = true;

		t->rcu_read_unlock_special.b.need_qs = true;

}

}

kernel/rcu/tree_trace.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -121,7 +121,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) 
		   rdp->cpu,

		   rdp->cpu,

		   cpu_is_offline(rdp->cpu) ? '!' : ' ',

		   cpu_is_offline(rdp->cpu) ? '!' : ' ',

		   ulong2long(rdp->completed), ulong2long(rdp->gpnum),

		   ulong2long(rdp->completed), ulong2long(rdp->gpnum),

		   rdp->cpu_no_qs,

		   rdp->cpu_no_qs.b.norm,

		   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),

		   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),

		   rdp->core_needs_qs);

		   rdp->core_needs_qs);

	seq_printf(m, " dt=%d/%llx/%d df=%lu",

	seq_printf(m, " dt=%d/%llx/%d df=%lu",