Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 543c6158 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Make synchronize_rcu_expedited() use sequence-counter scheme 



Although synchronize_rcu_expedited() uses a sequence-counter scheme, it
is based on a single increment per grace period, which means that tasks
piggybacking off of concurrent grace periods may be forced to wait longer
than necessary.  This commit therefore applies the new sequence-count
functions developed for synchronize_sched_expedited() to speed things
up a bit and to consolidate the sequence-counter implementation.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 28f00767

kernel/rcu/tree_plugin.h

+6 −10
Original line number Original line Diff line number Diff line
@@ -536,7 +536,6 @@ void synchronize_rcu(void) 
EXPORT_SYMBOL_GPL(synchronize_rcu);

EXPORT_SYMBOL_GPL(synchronize_rcu);





static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);

static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);

static unsigned long sync_rcu_preempt_exp_count;

static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);

static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);





/*

/*
@@ -704,12 +703,10 @@ void synchronize_rcu_expedited(void) 
{

{

	struct rcu_node *rnp;

	struct rcu_node *rnp;

	struct rcu_state *rsp = rcu_state_p;

	struct rcu_state *rsp = rcu_state_p;

	unsigned long snap;

	unsigned long s;

	int trycount = 0;

	int trycount = 0;





	smp_mb(); /* Caller's modifications seen first by other CPUs. */

	s = rcu_exp_gp_seq_snap(rsp);

	snap = READ_ONCE(sync_rcu_preempt_exp_count) + 1;

	smp_mb(); /* Above access cannot bleed into critical section. */





	/*

	/*

	 * Acquire lock, falling back to synchronize_rcu() if too many

	 * Acquire lock, falling back to synchronize_rcu() if too many
@@ -717,8 +714,7 @@ void synchronize_rcu_expedited(void) 
	 * expedited grace period for us, just leave.

	 * expedited grace period for us, just leave.

	 */

	 */

	while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {

	while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {

		if (ULONG_CMP_LT(snap,

		if (rcu_exp_gp_seq_done(rsp, s))

		    READ_ONCE(sync_rcu_preempt_exp_count)))

			goto mb_ret; /* Others did our work for us. */

			goto mb_ret; /* Others did our work for us. */

		if (trycount++ < 10) {

		if (trycount++ < 10) {

			udelay(trycount * num_online_cpus());

			udelay(trycount * num_online_cpus());
@@ -727,8 +723,9 @@ void synchronize_rcu_expedited(void) 
			return;

			return;

		}

		}

	}

	}

	if (ULONG_CMP_LT(snap, READ_ONCE(sync_rcu_preempt_exp_count)))

	if (rcu_exp_gp_seq_done(rsp, s))

		goto unlock_mb_ret; /* Others did our work for us. */

		goto unlock_mb_ret; /* Others did our work for us. */

	rcu_exp_gp_seq_start(rsp);





	/* force all RCU readers onto ->blkd_tasks lists. */

	/* force all RCU readers onto ->blkd_tasks lists. */

	synchronize_sched_expedited();

	synchronize_sched_expedited();
@@ -750,8 +747,7 @@ void synchronize_rcu_expedited(void) 
		   sync_rcu_preempt_exp_done(rnp));

		   sync_rcu_preempt_exp_done(rnp));





	/* Clean up and exit. */

	/* Clean up and exit. */

	smp_mb(); /* ensure expedited GP seen before counter increment. */

	rcu_exp_gp_seq_end(rsp);

	WRITE_ONCE(sync_rcu_preempt_exp_count, sync_rcu_preempt_exp_count + 1);

unlock_mb_ret:

unlock_mb_ret:

	mutex_unlock(&sync_rcu_preempt_exp_mutex);

	mutex_unlock(&sync_rcu_preempt_exp_mutex);

mb_ret:

mb_ret: