Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 551d55a9 authored by Mathieu Desnoyers's avatar Mathieu Desnoyers Committed by Paul E. McKenney
Browse files

tree/tiny rcu: Add debug RCU head objects 



Helps finding racy users of call_rcu(), which results in hangs because list
entries are overwritten and/or skipped.

Changelog since v4:
- Bissectability is now OK
- Now generate a WARN_ON_ONCE() for non-initialized rcu_head passed to
  call_rcu(). Statically initialized objects are detected with
  object_is_static().
- Rename rcu_head_init_on_stack to init_rcu_head_on_stack.
- Remove init_rcu_head() completely.

Changelog since v3:
- Include comments from Lai Jiangshan

This new patch version is based on the debugobjects with the newly introduced
"active state" tracker.

Non-initialized entries are all considered as "statically initialized". An
activation fixup (triggered by call_rcu()) takes care of performing the debug
object initialization without issuing any warning. Since we cannot increase the
size of struct rcu_head, I don't see much room to put an identifier for
statically initialized rcu_head structures. So for now, we have to live without
"activation without explicit init" detection. But the main purpose of this debug
option is to detect double-activations (double call_rcu() use of a rcu_head
before the callback is executed), which is correctly addressed here.

This also detects potential internal RCU callback corruption, which would cause
the callbacks to be executed twice.

Signed-off-by: default avatarMathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: David S. Miller <davem@davemloft.net>
CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
CC: akpm@linux-foundation.org
CC: mingo@elte.hu
CC: laijs@cn.fujitsu.com
CC: dipankar@in.ibm.com
CC: josh@joshtriplett.org
CC: dvhltc@us.ibm.com
CC: niv@us.ibm.com
CC: tglx@linutronix.de
CC: peterz@infradead.org
CC: rostedt@goodmis.org
CC: Valdis.Kletnieks@vt.edu
CC: dhowells@redhat.com
CC: eric.dumazet@gmail.com
CC: Alexey Dobriyan <adobriyan@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarLai Jiangshan <laijs@cn.fujitsu.com>
parent 875352c9

include/linux/rcupdate.h

+49 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 
#include <linux/seqlock.h>

#include <linux/lockdep.h>

#include <linux/completion.h>

#include <linux/debugobjects.h>



#ifdef CONFIG_RCU_TORTURE_TEST

extern int rcutorture_runnable; /* for sysctl */
@@ -79,6 +80,16 @@ extern void rcu_init(void); 
       (ptr)->next = NULL; (ptr)->func = NULL; \

} while (0)



/*

 * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic

 * initialization and destruction of rcu_head on the stack. rcu_head structures

 * allocated dynamically in the heap or defined statically don't need any

 * initialization.

 */

#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD

extern void init_rcu_head_on_stack(struct rcu_head *head);

extern void destroy_rcu_head_on_stack(struct rcu_head *head);

#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */

static inline void init_rcu_head_on_stack(struct rcu_head *head)

{

}
@@ -86,6 +97,7 @@ static inline void init_rcu_head_on_stack(struct rcu_head *head) 
static inline void destroy_rcu_head_on_stack(struct rcu_head *head)

{

}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -517,4 +529,41 @@ extern void call_rcu(struct rcu_head *head, 
extern void call_rcu_bh(struct rcu_head *head,

			void (*func)(struct rcu_head *head));



/*

 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally

 * by call_rcu() and rcu callback execution, and are therefore not part of the

 * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.

 */



#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD

# define STATE_RCU_HEAD_READY	0

# define STATE_RCU_HEAD_QUEUED	1



extern struct debug_obj_descr rcuhead_debug_descr;



static inline void debug_rcu_head_queue(struct rcu_head *head)

{

	debug_object_activate(head, &rcuhead_debug_descr);

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_READY,

				  STATE_RCU_HEAD_QUEUED);

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_QUEUED,

				  STATE_RCU_HEAD_READY);

	debug_object_deactivate(head, &rcuhead_debug_descr);

}

#else	/* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */

static inline void debug_rcu_head_queue(struct rcu_head *head)

{

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



#endif /* __LINUX_RCUPDATE_H */

kernel/rcupdate.c

+160 −0
Original line number Diff line number Diff line
@@ -114,3 +114,163 @@ int rcu_my_thread_group_empty(void) 
}

EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);

#endif /* #ifdef CONFIG_PROVE_RCU */



#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD

static inline void debug_init_rcu_head(struct rcu_head *head)

{

	debug_object_init(head, &rcuhead_debug_descr);

}



static inline void debug_rcu_head_free(struct rcu_head *head)

{

	debug_object_free(head, &rcuhead_debug_descr);

}



/*

 * fixup_init is called when:

 * - an active object is initialized

 */

static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)

{

	struct rcu_head *head = addr;



	switch (state) {

	case ODEBUG_STATE_ACTIVE:

		/*

		 * Ensure that queued callbacks are all executed.

		 * If we detect that we are nested in a RCU read-side critical

		 * section, we should simply fail, otherwise we would deadlock.

		 */

		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||

		    irqs_disabled()) {

			WARN_ON(1);

			return 0;

		}

		rcu_barrier();

		rcu_barrier_sched();

		rcu_barrier_bh();

		debug_object_init(head, &rcuhead_debug_descr);

		return 1;

	default:

		return 0;

	}

}



/*

 * fixup_activate is called when:

 * - an active object is activated

 * - an unknown object is activated (might be a statically initialized object)

 * Activation is performed internally by call_rcu().

 */

static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)

{

	struct rcu_head *head = addr;



	switch (state) {



	case ODEBUG_STATE_NOTAVAILABLE:

		/*

		 * This is not really a fixup. We just make sure that it is

		 * tracked in the object tracker.

		 */

		debug_object_init(head, &rcuhead_debug_descr);

		debug_object_activate(head, &rcuhead_debug_descr);

		return 0;



	case ODEBUG_STATE_ACTIVE:

		/*

		 * Ensure that queued callbacks are all executed.

		 * If we detect that we are nested in a RCU read-side critical

		 * section, we should simply fail, otherwise we would deadlock.

		 */

		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||

		    irqs_disabled()) {

			WARN_ON(1);

			return 0;

		}

		rcu_barrier();

		rcu_barrier_sched();

		rcu_barrier_bh();

		debug_object_activate(head, &rcuhead_debug_descr);

		return 1;

	default:

		return 0;

	}

}



/*

 * fixup_free is called when:

 * - an active object is freed

 */

static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)

{

	struct rcu_head *head = addr;



	switch (state) {

	case ODEBUG_STATE_ACTIVE:

		/*

		 * Ensure that queued callbacks are all executed.

		 * If we detect that we are nested in a RCU read-side critical

		 * section, we should simply fail, otherwise we would deadlock.

		 */

#ifndef CONFIG_PREEMPT

		WARN_ON(1);

		return 0;

#else

		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||

		    irqs_disabled()) {

			WARN_ON(1);

			return 0;

		}

		rcu_barrier();

		rcu_barrier_sched();

		rcu_barrier_bh();

		debug_object_free(head, &rcuhead_debug_descr);

		return 1;

#endif

	default:

		return 0;

	}

}



/**

 * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects

 * @head: pointer to rcu_head structure to be initialized

 *

 * This function informs debugobjects of a new rcu_head structure that

 * has been allocated as an auto variable on the stack.  This function

 * is not required for rcu_head structures that are statically defined or

 * that are dynamically allocated on the heap.  This function has no

 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.

 */

void init_rcu_head_on_stack(struct rcu_head *head)

{

	debug_object_init_on_stack(head, &rcuhead_debug_descr);

}

EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);



/**

 * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects

 * @head: pointer to rcu_head structure to be initialized

 *

 * This function informs debugobjects that an on-stack rcu_head structure

 * is about to go out of scope.  As with init_rcu_head_on_stack(), this

 * function is not required for rcu_head structures that are statically

 * defined or that are dynamically allocated on the heap.  Also as with

 * init_rcu_head_on_stack(), this function has no effect for

 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.

 */

void destroy_rcu_head_on_stack(struct rcu_head *head)

{

	debug_object_free(head, &rcuhead_debug_descr);

}

EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);



struct debug_obj_descr rcuhead_debug_descr = {

	.name = "rcu_head",

	.fixup_init = rcuhead_fixup_init,

	.fixup_activate = rcuhead_fixup_activate,

	.fixup_free = rcuhead_fixup_free,

};

EXPORT_SYMBOL_GPL(rcuhead_debug_descr);

#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

kernel/rcutiny.c

+2 −0
Original line number Diff line number Diff line
@@ -169,6 +169,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) 
	while (list) {

		next = list->next;

		prefetch(next);

		debug_rcu_head_unqueue(list);

		list->func(list);

		list = next;

	}
@@ -211,6 +212,7 @@ static void __call_rcu(struct rcu_head *head, 
{

	unsigned long flags;



	debug_rcu_head_queue(head);

	head->func = func;

	head->next = NULL;

kernel/rcutree.c

+2 −0
Original line number Diff line number Diff line
@@ -1112,6 +1112,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) 
	while (list) {

		next = list->next;

		prefetch(next);

		debug_rcu_head_unqueue(list);

		list->func(list);

		list = next;

		if (++count >= rdp->blimit)
@@ -1388,6 +1389,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	unsigned long flags;

	struct rcu_data *rdp;



	debug_rcu_head_queue(head);

	head->func = func;

	head->next = NULL;

lib/Kconfig.debug

+6 −0
Original line number Diff line number Diff line
@@ -307,6 +307,12 @@ config DEBUG_OBJECTS_WORK 
	  work queue routines to track the life time of work objects and

	  validate the work operations.



config DEBUG_OBJECTS_RCU_HEAD

	bool "Debug RCU callbacks objects"

	depends on DEBUG_OBJECTS && PREEMPT

	help

	  Enable this to turn on debugging of RCU list heads (call_rcu() usage).



config DEBUG_OBJECTS_ENABLE_DEFAULT

	int "debug_objects bootup default value (0-1)"

        range 0 1