Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1ce46ee5 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Conditionally compile RCU's eqs warnings 



This commit applies some warning-omission micro-optimizations to RCU's
various extended-quiescent-state functions, which are on the kernel/user
hotpath for CONFIG_NO_HZ_FULL=y.

Reported-by: default avatarRik van Riel <riel@redhat.com>
Reported by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent e72aeafc

kernel/rcu/tree.c

+15 −8
Original line number Diff line number Diff line
@@ -620,7 +620,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user) 
	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);



	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);

	if (!user && !is_idle_task(current)) {

	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

	    !user && !is_idle_task(current)) {

		struct task_struct *idle __maybe_unused =

			idle_task(smp_processor_id());
@@ -639,7 +640,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user) 
	smp_mb__before_atomic();  /* See above. */

	atomic_inc(&rdtp->dynticks);

	smp_mb__after_atomic();  /* Force ordering with next sojourn. */

	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

		     atomic_read(&rdtp->dynticks) & 0x1);

	rcu_dynticks_task_enter();



	/*
@@ -665,7 +667,8 @@ static void rcu_eqs_enter(bool user) 


	rdtp = this_cpu_ptr(&rcu_dynticks);

	oldval = rdtp->dynticks_nesting;

	WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

		     (oldval & DYNTICK_TASK_NEST_MASK) == 0);

	if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {

		rdtp->dynticks_nesting = 0;

		rcu_eqs_enter_common(oldval, user);
@@ -738,7 +741,8 @@ void rcu_irq_exit(void) 
	rdtp = this_cpu_ptr(&rcu_dynticks);

	oldval = rdtp->dynticks_nesting;

	rdtp->dynticks_nesting--;

	WARN_ON_ONCE(rdtp->dynticks_nesting < 0);

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

		     rdtp->dynticks_nesting < 0);

	if (rdtp->dynticks_nesting)

		trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);

	else
@@ -763,10 +767,12 @@ static void rcu_eqs_exit_common(long long oldval, int user) 
	atomic_inc(&rdtp->dynticks);

	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */

	smp_mb__after_atomic();  /* See above. */

	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

		     !(atomic_read(&rdtp->dynticks) & 0x1));

	rcu_cleanup_after_idle();

	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);

	if (!user && !is_idle_task(current)) {

	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

	    !user && !is_idle_task(current)) {

		struct task_struct *idle __maybe_unused =

			idle_task(smp_processor_id());
@@ -790,7 +796,7 @@ static void rcu_eqs_exit(bool user) 


	rdtp = this_cpu_ptr(&rcu_dynticks);

	oldval = rdtp->dynticks_nesting;

	WARN_ON_ONCE(oldval < 0);

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);

	if (oldval & DYNTICK_TASK_NEST_MASK) {

		rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;

	} else {
@@ -863,7 +869,8 @@ void rcu_irq_enter(void) 
	rdtp = this_cpu_ptr(&rcu_dynticks);

	oldval = rdtp->dynticks_nesting;

	rdtp->dynticks_nesting++;

	WARN_ON_ONCE(rdtp->dynticks_nesting == 0);

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&

		     rdtp->dynticks_nesting == 0);

	if (oldval)

		trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);

	else

lib/Kconfig.debug

+11 −0
Original line number Diff line number Diff line
@@ -1373,6 +1373,17 @@ config RCU_TRACE 
	  Say Y here if you want to enable RCU tracing

	  Say N if you are unsure.



config RCU_EQS_DEBUG

	bool "Use this when adding any sort of NO_HZ support to your arch"

	depends on DEBUG_KERNEL

	help

	  This option provides consistency checks in RCU's handling of

	  NO_HZ.  These checks have proven quite helpful in detecting

	  bugs in arch-specific NO_HZ code.



	  Say N here if you need ultimate kernel/user switch latencies

	  Say Y if you are unsure



endmenu # "RCU Debugging"



config DEBUG_BLOCK_EXT_DEVT