Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4ea3e85b authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Consolidate expedited GP code into rcu_exp_wait_wake() 



Currently, synchronize_rcu_expedited() and rcu_sched_expedited() have
significant duplicate code.  This commit therefore consolidates some of
this code into rcu_exp_wake(), which is now renamed to rcu_exp_wait_wake()
in recognition of its added responsibilities.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 356051e1

kernel/rcu/tree.c

+9 −9
Original line number Original line Diff line number Diff line
@@ -3839,14 +3839,18 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) 
}

}





/*

/*

 * Wake up everyone who piggybacked on the just-completed expedited

 * Wait for the current expedited grace period to complete, and then

 * wake up everyone who piggybacked on the just-completed expedited

 * grace period.  Also update all the ->exp_seq_rq counters as needed

 * grace period.  Also update all the ->exp_seq_rq counters as needed

 * in order to avoid counter-wrap problems.

 * in order to avoid counter-wrap problems.

 */

 */

static void rcu_exp_wake(struct rcu_state *rsp, unsigned long s)

static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)

{

{

	struct rcu_node *rnp;

	struct rcu_node *rnp;





	synchronize_sched_expedited_wait(rsp);

	rcu_exp_gp_seq_end(rsp);

	trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));

	rcu_for_each_node_breadth_first(rsp, rnp) {

	rcu_for_each_node_breadth_first(rsp, rnp) {

		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {

		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {

			spin_lock(&rnp->exp_lock);

			spin_lock(&rnp->exp_lock);
@@ -3857,6 +3861,8 @@ static void rcu_exp_wake(struct rcu_state *rsp, unsigned long s) 
		}

		}

		wake_up_all(&rnp->exp_wq[(rsp->expedited_sequence >> 1) & 0x1]);

		wake_up_all(&rnp->exp_wq[(rsp->expedited_sequence >> 1) & 0x1]);

	}

	}

	trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));

	mutex_unlock(&rsp->exp_mutex);

}

}





/**

/**
@@ -3904,13 +3910,7 @@ void synchronize_sched_expedited(void) 
	sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);

	sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);





	/* Wait and clean up, including waking everyone. */

	/* Wait and clean up, including waking everyone. */

	synchronize_sched_expedited_wait(rsp);

	rcu_exp_wait_wake(rsp, s);

	rcu_exp_gp_seq_end(rsp);

	trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));

	rcu_exp_wake(rsp, s);



	trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));

	mutex_unlock(&rsp->exp_mutex);

}

}

EXPORT_SYMBOL_GPL(synchronize_sched_expedited);

EXPORT_SYMBOL_GPL(synchronize_sched_expedited);

kernel/rcu/tree_plugin.h

+2 −8
Original line number Original line Diff line number Diff line
@@ -759,14 +759,8 @@ void synchronize_rcu_expedited(void) 
	/* Initialize the rcu_node tree in preparation for the wait. */

	/* Initialize the rcu_node tree in preparation for the wait. */

	sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);

	sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);





	/* Wait for snapshotted ->blkd_tasks lists to drain. */

	/* Wait for ->blkd_tasks lists to drain, then wake everyone up. */

	synchronize_sched_expedited_wait(rsp);

	rcu_exp_wait_wake(rsp, s);

	rcu_exp_gp_seq_end(rsp);

	trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));

	rcu_exp_wake(rsp, s);



	trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));

	mutex_unlock(&rsp->exp_mutex);

}

}

EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);

EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);