Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fae4b54f authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Introduce rcutorture testing for rcu_barrier() 



Although rcutorture does invoke rcu_barrier() and friends, it cannot
really be called a torture test given that it invokes them only once
at the end of the test.  This commit therefore introduces heavy-duty
rcutorture testing for rcu_barrier(), which may be carried out
concurrently with normal rcutorture testing.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 37e377d2

Documentation/RCU/torture.txt

+14 −1
Original line number Diff line number Diff line
@@ -47,6 +47,16 @@ irqreader Says to invoke RCU readers from irq level. This is currently 
		permit this.  (Or, more accurately, variants of RCU that do

		-not- permit this know to ignore this variable.)



n_barrier_cbs	If this is nonzero, RCU barrier testing will be conducted,

		in which case n_barrier_cbs specifies the number of

		RCU callbacks (and corresponding kthreads) to use for

		this testing.  The value cannot be negative.  If you

		specify this to be non-zero when torture_type indicates a

		synchronous RCU implementation (one for which a member of

		the synchronize_rcu() rather than the call_rcu() family is

		used -- see the documentation for torture_type below), an

		error will be reported and no testing will be carried out.



nfakewriters	This is the number of RCU fake writer threads to run.  Fake

		writer threads repeatedly use the synchronous "wait for

		current readers" function of the interface selected by
@@ -188,7 +198,7 @@ OUTPUT 
The statistics output is as follows:



	rcu-torture:--- Start of test: nreaders=16 nfakewriters=4 stat_interval=30 verbose=0 test_no_idle_hz=1 shuffle_interval=3 stutter=5 irqreader=1 fqs_duration=0 fqs_holdoff=0 fqs_stutter=3 test_boost=1/0 test_boost_interval=7 test_boost_duration=4

	rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767

	rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767

	rcu-torture: Reader Pipe:  727860534 34213 0 0 0 0 0 0 0 0 0

	rcu-torture: Reader Batch:  727877838 17003 0 0 0 0 0 0 0 0 0

	rcu-torture: Free-Block Circulation:  155440 155440 155440 155440 155440 155440 155440 155440 155440 155440 0
@@ -230,6 +240,9 @@ o "rtmbe": A non-zero value indicates that rcutorture believes that 
	rcu_assign_pointer() and rcu_dereference() are not working

	correctly.  This value should be zero.



o	"rtbe": A non-zero value indicates that one of the rcu_barrier()

	family of functions is not working correctly.



o	"rtbke": rcutorture was unable to create the real-time kthreads

	used to force RCU priority inversion.  This value should be zero.

kernel/rcutorture.c

+186 −8
Original line number Diff line number Diff line
@@ -64,6 +64,7 @@ static int irqreader = 1; /* RCU readers from irq (timers). */ 
static int fqs_duration;	/* Duration of bursts (us), 0 to disable. */

static int fqs_holdoff;		/* Hold time within burst (us). */

static int fqs_stutter = 3;	/* Wait time between bursts (s). */

static int n_barrier_cbs;	/* Number of callbacks to test RCU barriers. */

static int onoff_interval;	/* Wait time between CPU hotplugs, 0=disable. */

static int onoff_holdoff;	/* Seconds after boot before CPU hotplugs. */

static int shutdown_secs;	/* Shutdown time (s).  <=0 for no shutdown. */
@@ -96,6 +97,8 @@ module_param(fqs_holdoff, int, 0444); 
MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");

module_param(fqs_stutter, int, 0444);

MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");

module_param(n_barrier_cbs, int, 0444);

MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");

module_param(onoff_interval, int, 0444);

MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");

module_param(onoff_holdoff, int, 0444);
@@ -139,6 +142,8 @@ static struct task_struct *shutdown_task; 
static struct task_struct *onoff_task;

#endif /* #ifdef CONFIG_HOTPLUG_CPU */

static struct task_struct *stall_task;

static struct task_struct **barrier_cbs_tasks;

static struct task_struct *barrier_task;



#define RCU_TORTURE_PIPE_LEN 10
@@ -164,6 +169,7 @@ static atomic_t n_rcu_torture_alloc_fail; 
static atomic_t n_rcu_torture_free;

static atomic_t n_rcu_torture_mberror;

static atomic_t n_rcu_torture_error;

static long n_rcu_torture_barrier_error;

static long n_rcu_torture_boost_ktrerror;

static long n_rcu_torture_boost_rterror;

static long n_rcu_torture_boost_failure;
@@ -173,6 +179,8 @@ static long n_offline_attempts; 
static long n_offline_successes;

static long n_online_attempts;

static long n_online_successes;

static long n_barrier_attempts;

static long n_barrier_successes;

static struct list_head rcu_torture_removed;

static cpumask_var_t shuffle_tmp_mask;
@@ -197,6 +205,10 @@ static unsigned long shutdown_time; /* jiffies to system shutdown. */ 
static unsigned long boost_starttime;	/* jiffies of next boost test start. */

DEFINE_MUTEX(boost_mutex);		/* protect setting boost_starttime */

					/*  and boost task create/destroy. */

static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */

static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */

static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */

static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);



/* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
@@ -327,6 +339,7 @@ struct rcu_torture_ops { 
	int (*completed)(void);

	void (*deferred_free)(struct rcu_torture *p);

	void (*sync)(void);

	void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));

	void (*cb_barrier)(void);

	void (*fqs)(void);

	int (*stats)(char *page);
@@ -417,6 +430,7 @@ static struct rcu_torture_ops rcu_ops = { 
	.completed	= rcu_torture_completed,

	.deferred_free	= rcu_torture_deferred_free,

	.sync		= synchronize_rcu,

	.call		= call_rcu,

	.cb_barrier	= rcu_barrier,

	.fqs		= rcu_force_quiescent_state,

	.stats		= NULL,
@@ -460,6 +474,7 @@ static struct rcu_torture_ops rcu_sync_ops = { 
	.completed	= rcu_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= synchronize_rcu,

	.call		= NULL,

	.cb_barrier	= NULL,

	.fqs		= rcu_force_quiescent_state,

	.stats		= NULL,
@@ -477,6 +492,7 @@ static struct rcu_torture_ops rcu_expedited_ops = { 
	.completed	= rcu_no_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= synchronize_rcu_expedited,

	.call		= NULL,

	.cb_barrier	= NULL,

	.fqs		= rcu_force_quiescent_state,

	.stats		= NULL,
@@ -519,6 +535,7 @@ static struct rcu_torture_ops rcu_bh_ops = { 
	.completed	= rcu_bh_torture_completed,

	.deferred_free	= rcu_bh_torture_deferred_free,

	.sync		= synchronize_rcu_bh,

	.call		= call_rcu_bh,

	.cb_barrier	= rcu_barrier_bh,

	.fqs		= rcu_bh_force_quiescent_state,

	.stats		= NULL,
@@ -535,6 +552,7 @@ static struct rcu_torture_ops rcu_bh_sync_ops = { 
	.completed	= rcu_bh_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= synchronize_rcu_bh,

	.call		= NULL,

	.cb_barrier	= NULL,

	.fqs		= rcu_bh_force_quiescent_state,

	.stats		= NULL,
@@ -551,6 +569,7 @@ static struct rcu_torture_ops rcu_bh_expedited_ops = { 
	.completed	= rcu_bh_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= synchronize_rcu_bh_expedited,

	.call		= NULL,

	.cb_barrier	= NULL,

	.fqs		= rcu_bh_force_quiescent_state,

	.stats		= NULL,
@@ -637,6 +656,7 @@ static struct rcu_torture_ops srcu_ops = { 
	.completed	= srcu_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= srcu_torture_synchronize,

	.call		= NULL,

	.cb_barrier	= NULL,

	.stats		= srcu_torture_stats,

	.name		= "srcu"
@@ -661,6 +681,7 @@ static struct rcu_torture_ops srcu_raw_ops = { 
	.completed	= srcu_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= srcu_torture_synchronize,

	.call		= NULL,

	.cb_barrier	= NULL,

	.stats		= srcu_torture_stats,

	.name		= "srcu_raw"
@@ -680,6 +701,7 @@ static struct rcu_torture_ops srcu_expedited_ops = { 
	.completed	= srcu_torture_completed,

	.deferred_free	= rcu_sync_torture_deferred_free,

	.sync		= srcu_torture_synchronize_expedited,

	.call		= NULL,

	.cb_barrier	= NULL,

	.stats		= srcu_torture_stats,

	.name		= "srcu_expedited"
@@ -1129,7 +1151,8 @@ rcu_torture_printk(char *page) 
		       "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d "

		       "rtmbe: %d rtbke: %ld rtbre: %ld "

		       "rtbf: %ld rtb: %ld nt: %ld "

		       "onoff: %ld/%ld:%ld/%ld",

		       "onoff: %ld/%ld:%ld/%ld "

		       "barrier: %ld/%ld:%ld",

		       rcu_torture_current,

		       rcu_torture_current_version,

		       list_empty(&rcu_torture_freelist),
@@ -1145,14 +1168,17 @@ rcu_torture_printk(char *page) 
		       n_online_successes,

		       n_online_attempts,

		       n_offline_successes,

		       n_offline_attempts);

		       n_offline_attempts,

		       n_barrier_successes,

		       n_barrier_attempts,

		       n_rcu_torture_barrier_error);

	cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);

	if (atomic_read(&n_rcu_torture_mberror) != 0 ||

	    n_rcu_torture_barrier_error != 0 ||

	    n_rcu_torture_boost_ktrerror != 0 ||

	    n_rcu_torture_boost_rterror != 0 ||

	    n_rcu_torture_boost_failure != 0)

		cnt += sprintf(&page[cnt], " !!!");

	cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);

	if (i > 1) {

	    n_rcu_torture_boost_failure != 0 ||

	    i > 1) {

		cnt += sprintf(&page[cnt], "!!! ");

		atomic_inc(&n_rcu_torture_error);

		WARN_ON_ONCE(1);
@@ -1560,6 +1586,151 @@ static void rcu_torture_stall_cleanup(void) 
	stall_task = NULL;

}



/* Callback function for RCU barrier testing. */

void rcu_torture_barrier_cbf(struct rcu_head *rcu)

{

	atomic_inc(&barrier_cbs_invoked);

}



/* kthread function to register callbacks used to test RCU barriers. */

static int rcu_torture_barrier_cbs(void *arg)

{

	long myid = (long)arg;

	struct rcu_head rcu;



	init_rcu_head_on_stack(&rcu);

	VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");

	set_user_nice(current, 19);

	do {

		wait_event(barrier_cbs_wq[myid],

			   atomic_read(&barrier_cbs_count) == n_barrier_cbs ||

			   kthread_should_stop() ||

			   fullstop != FULLSTOP_DONTSTOP);

		if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)

			break;

		cur_ops->call(&rcu, rcu_torture_barrier_cbf);

		if (atomic_dec_and_test(&barrier_cbs_count))

			wake_up(&barrier_wq);

	} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);

	VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");

	rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");

	while (!kthread_should_stop())

		schedule_timeout_interruptible(1);

	cur_ops->cb_barrier();

	destroy_rcu_head_on_stack(&rcu);

	return 0;

}



/* kthread function to drive and coordinate RCU barrier testing. */

static int rcu_torture_barrier(void *arg)

{

	int i;



	VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");

	do {

		atomic_set(&barrier_cbs_invoked, 0);

		atomic_set(&barrier_cbs_count, n_barrier_cbs);

		/* wake_up() path contains the required barriers. */

		for (i = 0; i < n_barrier_cbs; i++)

			wake_up(&barrier_cbs_wq[i]);

		wait_event(barrier_wq,

			   atomic_read(&barrier_cbs_count) == 0 ||

			   kthread_should_stop() ||

			   fullstop != FULLSTOP_DONTSTOP);

		if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)

			break;

		n_barrier_attempts++;

		cur_ops->cb_barrier();

		if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {

			n_rcu_torture_barrier_error++;

			WARN_ON_ONCE(1);

		}

		n_barrier_successes++;

		schedule_timeout_interruptible(HZ / 10);

	} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);

	VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");

	rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");

	while (!kthread_should_stop())

		schedule_timeout_interruptible(1);

	return 0;

}



/* Initialize RCU barrier testing. */

static int rcu_torture_barrier_init(void)

{

	int i;

	int ret;



	if (n_barrier_cbs == 0)

		return 0;

	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {

		printk(KERN_ALERT "%s" TORTURE_FLAG

		       " Call or barrier ops missing for %s,\n",

		       torture_type, cur_ops->name);

		printk(KERN_ALERT "%s" TORTURE_FLAG

		       " RCU barrier testing omitted from run.\n",

		       torture_type);

		return 0;

	}

	atomic_set(&barrier_cbs_count, 0);

	atomic_set(&barrier_cbs_invoked, 0);

	barrier_cbs_tasks =

		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),

			GFP_KERNEL);

	barrier_cbs_wq =

		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),

			GFP_KERNEL);

	if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0)

		return -ENOMEM;

	for (i = 0; i < n_barrier_cbs; i++) {

		init_waitqueue_head(&barrier_cbs_wq[i]);

		barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,

						   (void *)i,

						   "rcu_torture_barrier_cbs");

		if (IS_ERR(barrier_cbs_tasks[i])) {

			ret = PTR_ERR(barrier_cbs_tasks[i]);

			VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");

			barrier_cbs_tasks[i] = NULL;

			return ret;

		}

	}

	barrier_task = kthread_run(rcu_torture_barrier, NULL,

				   "rcu_torture_barrier");

	if (IS_ERR(barrier_task)) {

		ret = PTR_ERR(barrier_task);

		VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");

		barrier_task = NULL;

	}

	return 0;

}



/* Clean up after RCU barrier testing. */

static void rcu_torture_barrier_cleanup(void)

{

	int i;



	if (barrier_task != NULL) {

		VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");

		kthread_stop(barrier_task);

		barrier_task = NULL;

	}

	if (barrier_cbs_tasks != NULL) {

		for (i = 0; i < n_barrier_cbs; i++) {

			if (barrier_cbs_tasks[i] != NULL) {

				VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");

				kthread_stop(barrier_cbs_tasks[i]);

				barrier_cbs_tasks[i] = NULL;

			}

		}

		kfree(barrier_cbs_tasks);

		barrier_cbs_tasks = NULL;

	}

	if (barrier_cbs_wq != NULL) {

		kfree(barrier_cbs_wq);

		barrier_cbs_wq = NULL;

	}

}



static int rcutorture_cpu_notify(struct notifier_block *self,

				 unsigned long action, void *hcpu)

{
@@ -1602,6 +1773,7 @@ rcu_torture_cleanup(void) 
	fullstop = FULLSTOP_RMMOD;

	mutex_unlock(&fullstop_mutex);

	unregister_reboot_notifier(&rcutorture_shutdown_nb);

	rcu_torture_barrier_cleanup();

	rcu_torture_stall_cleanup();

	if (stutter_task) {

		VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
@@ -1681,7 +1853,7 @@ rcu_torture_cleanup(void) 


	if (cur_ops->cleanup)

		cur_ops->cleanup();

	if (atomic_read(&n_rcu_torture_error))

	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)

		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");

	else if (n_online_successes != n_online_attempts ||

		 n_offline_successes != n_offline_attempts)
@@ -1697,6 +1869,7 @@ rcu_torture_init(void) 
	int i;

	int cpu;

	int firsterr = 0;

	int retval;

	static struct rcu_torture_ops *torture_ops[] =

		{ &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,

		  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
@@ -1754,6 +1927,7 @@ rcu_torture_init(void) 
	atomic_set(&n_rcu_torture_free, 0);

	atomic_set(&n_rcu_torture_mberror, 0);

	atomic_set(&n_rcu_torture_error, 0);

	n_rcu_torture_barrier_error = 0;

	n_rcu_torture_boost_ktrerror = 0;

	n_rcu_torture_boost_rterror = 0;

	n_rcu_torture_boost_failure = 0;
@@ -1877,7 +2051,6 @@ rcu_torture_init(void) 
		test_boost_duration = 2;

	if ((test_boost == 1 && cur_ops->can_boost) ||

	    test_boost == 2) {

		int retval;



		boost_starttime = jiffies + test_boost_interval * HZ;

		register_cpu_notifier(&rcutorture_cpu_nb);
@@ -1913,6 +2086,11 @@ rcu_torture_init(void) 
		firsterr = i;

		goto unwind;

	}

	retval = rcu_torture_barrier_init();

	if (retval != 0) {

		firsterr = retval;

		goto unwind;

	}

	rcutorture_record_test_transition();

	mutex_unlock(&fullstop_mutex);

	return 0;