Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c0f4dfd4 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Make RCU_FAST_NO_HZ take advantage of numbered callbacks 



Because RCU callbacks are now associated with the number of the grace
period that they must wait for, CPUs can now take advance callbacks
corresponding to grace periods that ended while a given CPU was in
dyntick-idle mode.  This eliminates the need to try forcing the RCU
state machine while entering idle, thus reducing the CPU intensiveness
of RCU_FAST_NO_HZ, which should increase its energy efficiency.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent b11cc576

Documentation/kernel-parameters.txt

+19 −9
Original line number Original line Diff line number Diff line
@@ -2490,6 +2490,17 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
			leaf rcu_node structure.  Useful for very large

			leaf rcu_node structure.  Useful for very large

			systems.

			systems.





	rcutree.jiffies_till_first_fqs= [KNL,BOOT]

			Set delay from grace-period initialization to

			first attempt to force quiescent states.

			Units are jiffies, minimum value is zero,

			and maximum value is HZ.



	rcutree.jiffies_till_next_fqs= [KNL,BOOT]

			Set delay between subsequent attempts to force

			quiescent states.  Units are jiffies, minimum

			value is one, and maximum value is HZ.



	rcutree.qhimark=	[KNL,BOOT]

	rcutree.qhimark=	[KNL,BOOT]

			Set threshold of queued

			Set threshold of queued

			RCU callbacks over which batch limiting is disabled.

			RCU callbacks over which batch limiting is disabled.
@@ -2504,16 +2515,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
	rcutree.rcu_cpu_stall_timeout= [KNL,BOOT]

	rcutree.rcu_cpu_stall_timeout= [KNL,BOOT]

			Set timeout for RCU CPU stall warning messages.

			Set timeout for RCU CPU stall warning messages.





	rcutree.jiffies_till_first_fqs= [KNL,BOOT]

	rcutree.rcu_idle_gp_delay=	[KNL,BOOT]

			Set delay from grace-period initialization to

			Set wakeup interval for idle CPUs that have

			first attempt to force quiescent states.

			RCU callbacks (RCU_FAST_NO_HZ=y).

			Units are jiffies, minimum value is zero,

			and maximum value is HZ.





	rcutree.jiffies_till_next_fqs= [KNL,BOOT]

	rcutree.rcu_idle_lazy_gp_delay=	[KNL,BOOT]

			Set delay between subsequent attempts to force

			Set wakeup interval for idle CPUs that have

			quiescent states.  Units are jiffies, minimum

			only "lazy" RCU callbacks (RCU_FAST_NO_HZ=y).

			value is one, and maximum value is HZ.

			Lazy RCU callbacks are those which RCU can

			prove do nothing more than free memory.





	rcutorture.fqs_duration= [KNL,BOOT]

	rcutorture.fqs_duration= [KNL,BOOT]

			Set duration of force_quiescent_state bursts.

			Set duration of force_quiescent_state bursts.

include/linux/rcupdate.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -80,6 +80,7 @@ extern void do_trace_rcu_torture_read(char *rcutorturename, 
#define UINT_CMP_LT(a, b)	(UINT_MAX / 2 < (a) - (b))

#define UINT_CMP_LT(a, b)	(UINT_MAX / 2 < (a) - (b))

#define ULONG_CMP_GE(a, b)	(ULONG_MAX / 2 >= (a) - (b))

#define ULONG_CMP_GE(a, b)	(ULONG_MAX / 2 >= (a) - (b))

#define ULONG_CMP_LT(a, b)	(ULONG_MAX / 2 < (a) - (b))

#define ULONG_CMP_LT(a, b)	(ULONG_MAX / 2 < (a) - (b))

#define ulong2long(a)		(*(long *)(&(a)))





/* Exported common interfaces */

/* Exported common interfaces */

init/Kconfig

+10 −7
Original line number Original line Diff line number Diff line
@@ -582,13 +582,16 @@ config RCU_FAST_NO_HZ 
	depends on NO_HZ && SMP

	depends on NO_HZ && SMP

	default n

	default n

	help

	help

	  This option causes RCU to attempt to accelerate grace periods in

	  This option permits CPUs to enter dynticks-idle state even if

	  order to allow CPUs to enter dynticks-idle state more quickly.

	  they have RCU callbacks queued, and prevents RCU from waking

	  On the other hand, this option increases the overhead of the

	  these CPUs up more than roughly once every four jiffies (by

	  dynticks-idle checking, thus degrading scheduling latency.

	  default, you can adjust this using the rcutree.rcu_idle_gp_delay



	  parameter), thus improving energy efficiency.  On the other

	  Say Y if energy efficiency is critically important, and you don't

	  hand, this option increases the duration of RCU grace periods,

	  	care about real-time response.

	  for example, slowing down synchronize_rcu().



	  Say Y if energy efficiency is critically important, and you

	  	don't care about increased grace-period durations.





	  Say N if you are unsure.

	  Say N if you are unsure.

kernel/rcutree.c

+17 −11
Original line number Original line Diff line number Diff line
@@ -2640,19 +2640,27 @@ static int rcu_pending(int cpu) 
}

}





/*

/*

 * Check to see if any future RCU-related work will need to be done

 * Return true if the specified CPU has any callback.  If all_lazy is

 * by the current CPU, even if none need be done immediately, returning

 * non-NULL, store an indication of whether all callbacks are lazy.

 * 1 if so.

 * (If there are no callbacks, all of them are deemed to be lazy.)

 */

 */

static int rcu_cpu_has_callbacks(int cpu)

static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)

{

{

	bool al = true;

	bool hc = false;

	struct rcu_data *rdp;

	struct rcu_state *rsp;

	struct rcu_state *rsp;





	/* RCU callbacks either ready or pending? */

	for_each_rcu_flavor(rsp) {

	for_each_rcu_flavor(rsp)

		rdp = per_cpu_ptr(rsp->rda, cpu);

		if (per_cpu_ptr(rsp->rda, cpu)->nxtlist)

		if (rdp->qlen != rdp->qlen_lazy)

			return 1;

			al = false;

	return 0;

		if (rdp->nxtlist)

			hc = true;

	}

	if (all_lazy)

		*all_lazy = al;

	return hc;

}

}





/*

/*
@@ -2871,7 +2879,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) 
	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;

	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;

	atomic_set(&rdp->dynticks->dynticks,

	atomic_set(&rdp->dynticks->dynticks,

		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);

		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);

	rcu_prepare_for_idle_init(cpu);

	raw_spin_unlock(&rnp->lock);		/* irqs remain disabled. */

	raw_spin_unlock(&rnp->lock);		/* irqs remain disabled. */





	/* Add CPU to rcu_node bitmasks. */

	/* Add CPU to rcu_node bitmasks. */
@@ -2945,7 +2952,6 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self, 
		 */

		 */

		for_each_rcu_flavor(rsp)

		for_each_rcu_flavor(rsp)

			rcu_cleanup_dying_cpu(rsp);

			rcu_cleanup_dying_cpu(rsp);

		rcu_cleanup_after_idle(cpu);

		break;

		break;

	case CPU_DEAD:

	case CPU_DEAD:

	case CPU_DEAD_FROZEN:

	case CPU_DEAD_FROZEN:

kernel/rcutree.h

+3 −9
Original line number Original line Diff line number Diff line
@@ -88,18 +88,13 @@ struct rcu_dynticks { 
	int dynticks_nmi_nesting;   /* Track NMI nesting level. */

	int dynticks_nmi_nesting;   /* Track NMI nesting level. */

	atomic_t dynticks;	    /* Even value for idle, else odd. */

	atomic_t dynticks;	    /* Even value for idle, else odd. */

#ifdef CONFIG_RCU_FAST_NO_HZ

#ifdef CONFIG_RCU_FAST_NO_HZ

	int dyntick_drain;	    /* Prepare-for-idle state variable. */

	bool all_lazy;		    /* Are all CPU's CBs lazy? */

	unsigned long dyntick_holdoff;

				    /* No retries for the jiffy of failure. */

	struct timer_list idle_gp_timer;

				    /* Wake up CPU sleeping with callbacks. */

	unsigned long idle_gp_timer_expires;

				    /* When to wake up CPU (for repost). */

	bool idle_first_pass;	    /* First pass of attempt to go idle? */

	unsigned long nonlazy_posted;

	unsigned long nonlazy_posted;

				    /* # times non-lazy CBs posted to CPU. */

				    /* # times non-lazy CBs posted to CPU. */

	unsigned long nonlazy_posted_snap;

	unsigned long nonlazy_posted_snap;

				    /* idle-period nonlazy_posted snapshot. */

				    /* idle-period nonlazy_posted snapshot. */

	unsigned long last_accelerate;

				    /* Last jiffy CBs were accelerated. */

	int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */

	int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */

#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */

#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */

};

};
@@ -521,7 +516,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, 
						 struct rcu_node *rnp);

						 struct rcu_node *rnp);

#endif /* #ifdef CONFIG_RCU_BOOST */

#endif /* #ifdef CONFIG_RCU_BOOST */

static void __cpuinit rcu_prepare_kthreads(int cpu);

static void __cpuinit rcu_prepare_kthreads(int cpu);

static void rcu_prepare_for_idle_init(int cpu);

static void rcu_cleanup_after_idle(int cpu);

static void rcu_cleanup_after_idle(int cpu);

static void rcu_prepare_for_idle(int cpu);

static void rcu_prepare_for_idle(int cpu);

static void rcu_idle_count_callbacks_posted(void);

static void rcu_idle_count_callbacks_posted(void);