Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 29154c57 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Split RCU core processing out of __call_rcu() 



The __call_rcu() function is a bit overweight, so this commit splits
it into actual enqueuing of and accounting for the callback (__call_rcu())
and associated RCU-core processing (__call_rcu_core()).

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent a16b7a69

kernel/rcutree.c

+49 −41
Original line number Diff line number Diff line
@@ -1861,45 +1861,12 @@ static void invoke_rcu_core(void) 
	raise_softirq(RCU_SOFTIRQ);

}



static void

__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),

	   struct rcu_state *rsp, bool lazy)

{

	unsigned long flags;

	struct rcu_data *rdp;



	WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */

	debug_rcu_head_queue(head);

	head->func = func;

	head->next = NULL;



	smp_mb(); /* Ensure RCU update seen before callback registry. */



/*

	 * Opportunistically note grace-period endings and beginnings.

	 * Note that we might see a beginning right after we see an

	 * end, but never vice versa, since this CPU has to pass through

	 * a quiescent state betweentimes.

 * Handle any core-RCU processing required by a call_rcu() invocation.

 */

	local_irq_save(flags);

	rdp = this_cpu_ptr(rsp->rda);



	/* Add the callback to our list. */

	ACCESS_ONCE(rdp->qlen)++;

	if (lazy)

		rdp->qlen_lazy++;

	else

		rcu_idle_count_callbacks_posted();

	smp_mb();  /* Count before adding callback for rcu_barrier(). */

	*rdp->nxttail[RCU_NEXT_TAIL] = head;

	rdp->nxttail[RCU_NEXT_TAIL] = &head->next;



	if (__is_kfree_rcu_offset((unsigned long)func))

		trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,

					 rdp->qlen_lazy, rdp->qlen);

	else

		trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);



static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,

			    struct rcu_head *head, unsigned long flags)

{

	/*

	 * If called from an extended quiescent state, invoke the RCU

	 * core in order to force a re-evaluation of RCU's idleness.
@@ -1908,10 +1875,8 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
		invoke_rcu_core();



	/* If interrupts were disabled or CPU offline, don't invoke RCU core. */

	if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id())) {

		local_irq_restore(flags);

	if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))

		return;

	}



	/*

	 * Force the grace period if too many callbacks or too long waiting.
@@ -1944,6 +1909,49 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
		}

	} else if (ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs), jiffies))

		force_quiescent_state(rsp, 1);

}



static void

__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),

	   struct rcu_state *rsp, bool lazy)

{

	unsigned long flags;

	struct rcu_data *rdp;



	WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */

	debug_rcu_head_queue(head);

	head->func = func;

	head->next = NULL;



	smp_mb(); /* Ensure RCU update seen before callback registry. */



	/*

	 * Opportunistically note grace-period endings and beginnings.

	 * Note that we might see a beginning right after we see an

	 * end, but never vice versa, since this CPU has to pass through

	 * a quiescent state betweentimes.

	 */

	local_irq_save(flags);

	rdp = this_cpu_ptr(rsp->rda);



	/* Add the callback to our list. */

	ACCESS_ONCE(rdp->qlen)++;

	if (lazy)

		rdp->qlen_lazy++;

	else

		rcu_idle_count_callbacks_posted();

	smp_mb();  /* Count before adding callback for rcu_barrier(). */

	*rdp->nxttail[RCU_NEXT_TAIL] = head;

	rdp->nxttail[RCU_NEXT_TAIL] = &head->next;



	if (__is_kfree_rcu_offset((unsigned long)func))

		trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,

					 rdp->qlen_lazy, rdp->qlen);

	else

		trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);



	/* Go handle any RCU core processing required. */

	__call_rcu_core(rsp, rdp, head, flags);

	local_irq_restore(flags);

}