Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 29c00b4a authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Add event-tracing for RCU callback invocation 



There was recently some controversy about the overhead of invoking RCU
callbacks.  Add TRACE_EVENT()s to obtain fine-grained timings for the
start and stop of a batch of callbacks and also for each callback invoked.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 9d68197c

include/linux/rcupdate.h

+0 −50
Original line number Diff line number Diff line
@@ -794,44 +794,6 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) 
#define RCU_INIT_POINTER(p, v) \

		p = (typeof(*v) __force __rcu *)(v)



/*

 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally

 * by call_rcu() and rcu callback execution, and are therefore not part of the

 * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.

 */



#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD

# define STATE_RCU_HEAD_READY	0

# define STATE_RCU_HEAD_QUEUED	1



extern struct debug_obj_descr rcuhead_debug_descr;



static inline void debug_rcu_head_queue(struct rcu_head *head)

{

	WARN_ON_ONCE((unsigned long)head & 0x3);

	debug_object_activate(head, &rcuhead_debug_descr);

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_READY,

				  STATE_RCU_HEAD_QUEUED);

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_QUEUED,

				  STATE_RCU_HEAD_READY);

	debug_object_deactivate(head, &rcuhead_debug_descr);

}

#else	/* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */

static inline void debug_rcu_head_queue(struct rcu_head *head)

{

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)

{

	return offset < 4096;
@@ -850,18 +812,6 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset) 
	call_rcu(head, (rcu_callback)offset);

}



extern void kfree(const void *);



static inline void __rcu_reclaim(struct rcu_head *head)

{

	unsigned long offset = (unsigned long)head->func;



	if (__is_kfree_rcu_offset(offset))

		kfree((void *)head - offset);

	else

		head->func(head);

}



/**

 * kfree_rcu() - kfree an object after a grace period.

 * @ptr:	pointer to kfree

include/trace/events/rcu.h

0 → 100644
+98 −0
Original line number Diff line number Diff line 
#undef TRACE_SYSTEM

#define TRACE_SYSTEM rcu



#if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)

#define _TRACE_RCU_H



#include <linux/tracepoint.h>



/*

 * Tracepoint for calling rcu_do_batch, performed to start callback invocation:

 */

TRACE_EVENT(rcu_batch_start,



	TP_PROTO(long callbacks_ready, int blimit),



	TP_ARGS(callbacks_ready, blimit),



	TP_STRUCT__entry(

		__field(	long,	callbacks_ready		)

		__field(	int,	blimit			)

	),



	TP_fast_assign(

		__entry->callbacks_ready	= callbacks_ready;

		__entry->blimit			= blimit;

	),



	TP_printk("CBs=%ld bl=%d", __entry->callbacks_ready, __entry->blimit)

);



/*

 * Tracepoint for the invocation of a single RCU callback

 */

TRACE_EVENT(rcu_invoke_callback,



	TP_PROTO(struct rcu_head *rhp),



	TP_ARGS(rhp),



	TP_STRUCT__entry(

		__field(	void *,	rhp	)

		__field(	void *,	func	)

	),



	TP_fast_assign(

		__entry->rhp	= rhp;

		__entry->func	= rhp->func;

	),



	TP_printk("rhp=%p func=%pf", __entry->rhp, __entry->func)

);



/*

 * Tracepoint for the invocation of a single RCU kfree callback

 */

TRACE_EVENT(rcu_invoke_kfree_callback,



	TP_PROTO(struct rcu_head *rhp, unsigned long offset),



	TP_ARGS(rhp, offset),



	TP_STRUCT__entry(

		__field(void *,	rhp	)

		__field(unsigned long,	offset	)

	),



	TP_fast_assign(

		__entry->rhp	= rhp;

		__entry->offset	= offset;

	),



	TP_printk("rhp=%p func=%ld", __entry->rhp, __entry->offset)

);



/*

 * Tracepoint for leaving rcu_do_batch, performed after callback invocation:

 */

TRACE_EVENT(rcu_batch_end,



	TP_PROTO(int callbacks_invoked),



	TP_ARGS(callbacks_invoked),



	TP_STRUCT__entry(

		__field(	int,	callbacks_invoked		)

	),



	TP_fast_assign(

		__entry->callbacks_invoked	= callbacks_invoked;

	),



	TP_printk("CBs-invoked=%d", __entry->callbacks_invoked)

);



#endif /* _TRACE_RCU_H */



/* This part must be outside protection */

#include <trace/define_trace.h>

kernel/rcu.h

0 → 100644
+79 −0
Original line number Diff line number Diff line 
/*

 * Read-Copy Update definitions shared among RCU implementations.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

 *

 * Copyright IBM Corporation, 2011

 *

 * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

 */



#ifndef __LINUX_RCU_H

#define __LINUX_RCU_H



/*

 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally

 * by call_rcu() and rcu callback execution, and are therefore not part of the

 * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.

 */



#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD

# define STATE_RCU_HEAD_READY	0

# define STATE_RCU_HEAD_QUEUED	1



extern struct debug_obj_descr rcuhead_debug_descr;



static inline void debug_rcu_head_queue(struct rcu_head *head)

{

	WARN_ON_ONCE((unsigned long)head & 0x3);

	debug_object_activate(head, &rcuhead_debug_descr);

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_READY,

				  STATE_RCU_HEAD_QUEUED);

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

	debug_object_active_state(head, &rcuhead_debug_descr,

				  STATE_RCU_HEAD_QUEUED,

				  STATE_RCU_HEAD_READY);

	debug_object_deactivate(head, &rcuhead_debug_descr);

}

#else	/* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */

static inline void debug_rcu_head_queue(struct rcu_head *head)

{

}



static inline void debug_rcu_head_unqueue(struct rcu_head *head)

{

}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



extern void kfree(const void *);



static inline void __rcu_reclaim(struct rcu_head *head)

{

	unsigned long offset = (unsigned long)head->func;



	if (__is_kfree_rcu_offset(offset)) {

		trace_rcu_invoke_kfree_callback(head, offset);

		kfree((void *)head - offset);

	} else {

		trace_rcu_invoke_callback(head);

		head->func(head);

	}

}



#endif /* __LINUX_RCU_H */

kernel/rcupdate.c

+5 −0
Original line number Diff line number Diff line
@@ -46,6 +46,11 @@ 
#include <linux/module.h>

#include <linux/hardirq.h>



#define CREATE_TRACE_POINTS

#include <trace/events/rcu.h>



#include "rcu.h"



#ifdef CONFIG_DEBUG_LOCK_ALLOC

static struct lock_class_key rcu_lock_key;

struct lockdep_map rcu_lock_map =

kernel/rcutiny.c

+25 −1
Original line number Diff line number Diff line
@@ -37,6 +37,25 @@ 
#include <linux/cpu.h>

#include <linux/prefetch.h>



#ifdef CONFIG_RCU_TRACE



#include <trace/events/rcu.h>



#else /* #ifdef CONFIG_RCU_TRACE */



/* No by-default tracing in TINY_RCU: Keep TINY_RCU tiny! */

static void trace_rcu_invoke_kfree_callback(struct rcu_head *rhp,

					    unsigned long offset)

{

}

static void trace_rcu_invoke_callback(struct rcu_head *head)

{

}



#endif /* #else #ifdef CONFIG_RCU_TRACE */



#include "rcu.h"



/* Controls for rcu_kthread() kthread, replacing RCU_SOFTIRQ used previously. */

static struct task_struct *rcu_kthread_task;

static DECLARE_WAIT_QUEUE_HEAD(rcu_kthread_wq);
@@ -161,11 +180,15 @@ static void rcu_process_callbacks(struct rcu_ctrlblk *rcp) 
	RCU_TRACE(int cb_count = 0);



	/* If no RCU callbacks ready to invoke, just return. */

	if (&rcp->rcucblist == rcp->donetail)

	if (&rcp->rcucblist == rcp->donetail) {

		RCU_TRACE(trace_rcu_batch_start(0, -1));

		RCU_TRACE(trace_rcu_batch_end(0));

		return;

	}



	/* Move the ready-to-invoke callbacks to a local list. */

	local_irq_save(flags);

	RCU_TRACE(trace_rcu_batch_start(0, -1));

	list = rcp->rcucblist;

	rcp->rcucblist = *rcp->donetail;

	*rcp->donetail = NULL;
@@ -187,6 +210,7 @@ static void rcu_process_callbacks(struct rcu_ctrlblk *rcp) 
		RCU_TRACE(cb_count++);

	}

	RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count));

	RCU_TRACE(trace_rcu_batch_end(cb_count));

}



/*