Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 486e2593 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Avoid waking up CPUs having only kfree_rcu() callbacks 



When CONFIG_RCU_FAST_NO_HZ is enabled, RCU will allow a given CPU to
enter dyntick-idle mode even if it still has RCU callbacks queued.
RCU avoids system hangs in this case by scheduling a timer for several
jiffies in the future.  However, if all of the callbacks on that CPU
are from kfree_rcu(), there is no reason to wake the CPU up, as it is
not a problem to defer freeing of memory.

This commit therefore tracks the number of callbacks on a given CPU
that are from kfree_rcu(), and avoids scheduling the timer if all of
a given CPU's callbacks are from kfree_rcu().

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 0bb7b59d

include/linux/rcupdate.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -841,7 +841,7 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset) 
	/* See the kfree_rcu() header comment. */

	/* See the kfree_rcu() header comment. */

	BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));

	BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));





	call_rcu(head, (rcu_callback)offset);

	kfree_call_rcu(head, (rcu_callback)offset);

}

}





/**

/**

include/linux/rcutiny.h

+6 −0
Original line number Original line Diff line number Diff line
@@ -83,6 +83,12 @@ static inline void synchronize_sched_expedited(void) 
	synchronize_sched();

	synchronize_sched();

}

}





static inline void kfree_call_rcu(struct rcu_head *head,

				  void (*func)(struct rcu_head *rcu))

{

	call_rcu(head, func);

}



#ifdef CONFIG_TINY_RCU

#ifdef CONFIG_TINY_RCU





static inline void rcu_preempt_note_context_switch(void)

static inline void rcu_preempt_note_context_switch(void)

include/linux/rcutree.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -61,6 +61,8 @@ extern void synchronize_rcu_bh(void); 
extern void synchronize_sched_expedited(void);

extern void synchronize_sched_expedited(void);

extern void synchronize_rcu_expedited(void);

extern void synchronize_rcu_expedited(void);





void kfree_call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));



static inline void synchronize_rcu_bh_expedited(void)

static inline void synchronize_rcu_bh_expedited(void)

{

{

	synchronize_sched_expedited();

	synchronize_sched_expedited();

include/trace/events/rcu.h

+39 −24
Original line number Original line Diff line number Diff line
@@ -313,19 +313,22 @@ TRACE_EVENT(rcu_prep_idle, 
/*

/*

 * Tracepoint for the registration of a single RCU callback function.

 * Tracepoint for the registration of a single RCU callback function.

 * The first argument is the type of RCU, the second argument is

 * The first argument is the type of RCU, the second argument is

 * a pointer to the RCU callback itself, and the third element is the

 * a pointer to the RCU callback itself, the third element is the

 * new RCU callback queue length for the current CPU.

 * number of lazy callbacks queued, and the fourth element is the

 * total number of callbacks queued.

 */

 */

TRACE_EVENT(rcu_callback,

TRACE_EVENT(rcu_callback,





	TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),

	TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen_lazy,

		 long qlen),





	TP_ARGS(rcuname, rhp, qlen),

	TP_ARGS(rcuname, rhp, qlen_lazy, qlen),





	TP_STRUCT__entry(

	TP_STRUCT__entry(

		__field(char *, rcuname)

		__field(char *, rcuname)

		__field(void *, rhp)

		__field(void *, rhp)

		__field(void *, func)

		__field(void *, func)

		__field(long, qlen_lazy)

		__field(long, qlen)

		__field(long, qlen)

	),

	),
@@ -333,11 +336,13 @@ TRACE_EVENT(rcu_callback, 
		__entry->rcuname = rcuname;

		__entry->rcuname = rcuname;

		__entry->rhp = rhp;

		__entry->rhp = rhp;

		__entry->func = rhp->func;

		__entry->func = rhp->func;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

		__entry->qlen = qlen;

	),

	),





	TP_printk("%s rhp=%p func=%pf %ld",

	TP_printk("%s rhp=%p func=%pf %ld/%ld",

		  __entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)

		  __entry->rcuname, __entry->rhp, __entry->func,

		  __entry->qlen_lazy, __entry->qlen)

);

);





/*

/*
@@ -345,20 +350,21 @@ TRACE_EVENT(rcu_callback, 
 * kfree() form.  The first argument is the RCU type, the second argument

 * kfree() form.  The first argument is the RCU type, the second argument

 * is a pointer to the RCU callback, the third argument is the offset

 * is a pointer to the RCU callback, the third argument is the offset

 * of the callback within the enclosing RCU-protected data structure,

 * of the callback within the enclosing RCU-protected data structure,

 * and the fourth argument is the new RCU callback queue length for the

 * the fourth argument is the number of lazy callbacks queued, and the

 * current CPU.

 * fifth argument is the total number of callbacks queued.

 */

 */

TRACE_EVENT(rcu_kfree_callback,

TRACE_EVENT(rcu_kfree_callback,





	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,

	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,

		 long qlen),

		 long qlen_lazy, long qlen),





	TP_ARGS(rcuname, rhp, offset, qlen),

	TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),





	TP_STRUCT__entry(

	TP_STRUCT__entry(

		__field(char *, rcuname)

		__field(char *, rcuname)

		__field(void *, rhp)

		__field(void *, rhp)

		__field(unsigned long, offset)

		__field(unsigned long, offset)

		__field(long, qlen_lazy)

		__field(long, qlen)

		__field(long, qlen)

	),

	),
@@ -366,41 +372,45 @@ TRACE_EVENT(rcu_kfree_callback, 
		__entry->rcuname = rcuname;

		__entry->rcuname = rcuname;

		__entry->rhp = rhp;

		__entry->rhp = rhp;

		__entry->offset = offset;

		__entry->offset = offset;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

		__entry->qlen = qlen;

	),

	),





	TP_printk("%s rhp=%p func=%ld %ld",

	TP_printk("%s rhp=%p func=%ld %ld/%ld",

		  __entry->rcuname, __entry->rhp, __entry->offset,

		  __entry->rcuname, __entry->rhp, __entry->offset,

		  __entry->qlen)

		  __entry->qlen_lazy, __entry->qlen)

);

);





/*

/*

 * Tracepoint for marking the beginning rcu_do_batch, performed to start

 * Tracepoint for marking the beginning rcu_do_batch, performed to start

 * RCU callback invocation.  The first argument is the RCU flavor,

 * RCU callback invocation.  The first argument is the RCU flavor,

 * the second is the total number of callbacks (including those that

 * the second is the number of lazy callbacks queued, the third is

 * are not yet ready to be invoked), and the third argument is the

 * the total number of callbacks queued, and the fourth argument is

 * current RCU-callback batch limit.

 * the current RCU-callback batch limit.

 */

 */

TRACE_EVENT(rcu_batch_start,

TRACE_EVENT(rcu_batch_start,





	TP_PROTO(char *rcuname, long qlen, int blimit),

	TP_PROTO(char *rcuname, long qlen_lazy, long qlen, int blimit),





	TP_ARGS(rcuname, qlen, blimit),

	TP_ARGS(rcuname, qlen_lazy, qlen, blimit),





	TP_STRUCT__entry(

	TP_STRUCT__entry(

		__field(char *, rcuname)

		__field(char *, rcuname)

		__field(long, qlen_lazy)

		__field(long, qlen)

		__field(long, qlen)

		__field(int, blimit)

		__field(int, blimit)

	),

	),





	TP_fast_assign(

	TP_fast_assign(

		__entry->rcuname = rcuname;

		__entry->rcuname = rcuname;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

		__entry->qlen = qlen;

		__entry->blimit = blimit;

		__entry->blimit = blimit;

	),

	),





	TP_printk("%s CBs=%ld bl=%d",

	TP_printk("%s CBs=%ld/%ld bl=%d",

		  __entry->rcuname, __entry->qlen, __entry->blimit)

		  __entry->rcuname, __entry->qlen_lazy, __entry->qlen,

		  __entry->blimit)

);

);





/*

/*
@@ -531,16 +541,21 @@ TRACE_EVENT(rcu_torture_read, 
#else /* #ifdef CONFIG_RCU_TRACE */

#else /* #ifdef CONFIG_RCU_TRACE */





#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)

#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)

#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)

#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, \

				    qsmask) do { } while (0)

#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)

#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)

#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)

#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)

#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)

#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, \

					 grplo, grphi, gp_tasks) do { } \

	while (0)

#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)

#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)

#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)

#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)

#define trace_rcu_prep_idle(reason) do { } while (0)

#define trace_rcu_prep_idle(reason) do { } while (0)

#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)

#define trace_rcu_callback(rcuname, rhp, qlen_lazy, qlen) do { } while (0)

#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)

#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen_lazy, qlen) \

#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)

	do { } while (0)

#define trace_rcu_batch_start(rcuname, qlen_lazy, qlen, blimit) \

	do { } while (0)

#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)

#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)

#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)

#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)

#define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \

#define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \

kernel/rcu.h

+3 −1
Original line number Original line Diff line number Diff line
@@ -76,16 +76,18 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) 




extern void kfree(const void *);

extern void kfree(const void *);





static inline void __rcu_reclaim(char *rn, struct rcu_head *head)

static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)

{

{

	unsigned long offset = (unsigned long)head->func;

	unsigned long offset = (unsigned long)head->func;





	if (__is_kfree_rcu_offset(offset)) {

	if (__is_kfree_rcu_offset(offset)) {

		RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));

		RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));

		kfree((void *)head - offset);

		kfree((void *)head - offset);

		return 1;

	} else {

	} else {

		RCU_TRACE(trace_rcu_invoke_callback(rn, head));

		RCU_TRACE(trace_rcu_invoke_callback(rn, head));

		head->func(head);

		head->func(head);

		return 0;

	}

	}

}

}