Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b52573d2 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: reduce __call_rcu()-induced contention on rcu_node structures 



When the current __call_rcu() function was written, the expedited
APIs did not exist.  The __call_rcu() implementation therefore went
to great lengths to detect the end of old grace periods and to start
new ones, all in the name of reducing grace-period latency.  Now the
expedited APIs do exist, and the usage of __call_rcu() has increased
considerably.  This commit therefore causes __call_rcu() to avoid
worrying about grace periods unless there are a large number of
RCU callbacks stacked up on the current CPU.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 0209f649

kernel/rcutree.c

+21 −17
Original line number Diff line number Diff line
@@ -1435,22 +1435,11 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	 */

	local_irq_save(flags);

	rdp = this_cpu_ptr(rsp->rda);

	rcu_process_gp_end(rsp, rdp);

	check_for_new_grace_period(rsp, rdp);



	/* Add the callback to our list. */

	*rdp->nxttail[RCU_NEXT_TAIL] = head;

	rdp->nxttail[RCU_NEXT_TAIL] = &head->next;



	/* Start a new grace period if one not already started. */

	if (!rcu_gp_in_progress(rsp)) {

		unsigned long nestflag;

		struct rcu_node *rnp_root = rcu_get_root(rsp);



		raw_spin_lock_irqsave(&rnp_root->lock, nestflag);

		rcu_start_gp(rsp, nestflag);  /* releases rnp_root->lock. */

	}



	/*

	 * Force the grace period if too many callbacks or too long waiting.

	 * Enforce hysteresis, and don't invoke force_quiescent_state()
@@ -1459,12 +1448,27 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	 * is the only one waiting for a grace period to complete.

	 */

	if (unlikely(++rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) {



		/* Are we ignoring a completed grace period? */

		rcu_process_gp_end(rsp, rdp);

		check_for_new_grace_period(rsp, rdp);



		/* Start a new grace period if one not already started. */

		if (!rcu_gp_in_progress(rsp)) {

			unsigned long nestflag;

			struct rcu_node *rnp_root = rcu_get_root(rsp);



			raw_spin_lock_irqsave(&rnp_root->lock, nestflag);

			rcu_start_gp(rsp, nestflag);  /* rlses rnp_root->lock */

		} else {

			/* Give the grace period a kick. */

			rdp->blimit = LONG_MAX;

			if (rsp->n_force_qs == rdp->n_force_qs_snap &&

			    *rdp->nxttail[RCU_DONE_TAIL] != head)

				force_quiescent_state(rsp, 0);

			rdp->n_force_qs_snap = rsp->n_force_qs;

			rdp->qlen_last_fqs_check = rdp->qlen;

		}

	} else if (ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs), jiffies))

		force_quiescent_state(rsp, 1);

	local_irq_restore(flags);