Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 34404ca8 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs 



When a CPU is first determined to be a no-CBs CPUs, this commit causes
any early boot callbacks to be moved to the no-CBs callback list,
allowing them to be invoked.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 39c8d313

kernel/rcu/tree.c

+1 −0
Original line number Diff line number Diff line
@@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
		 * and then drop through to queue the callback.

		 */

		BUG_ON(cpu != -1);

		WARN_ON_ONCE(!rcu_is_watching());

		if (!likely(rdp->nxtlist))

			init_default_callback_list(rdp);

	}

kernel/rcu/tree_plugin.h

+12 −12
Original line number Diff line number Diff line
@@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void) 
		pr_info("\tPoll for callbacks from no-CBs CPUs.\n");



	for_each_rcu_flavor(rsp) {

		for_each_cpu(cpu, rcu_nocb_mask) {

			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);



			/*

			 * If there are early callbacks, they will need

			 * to be moved to the nocb lists.

			 */

			WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=

				     &rdp->nxtlist &&

				     rdp->nxttail[RCU_NEXT_TAIL] != NULL);

			init_nocb_callback_list(rdp);

		}

		for_each_cpu(cpu, rcu_nocb_mask)

			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));

		rcu_organize_nocb_kthreads(rsp);

	}

}
@@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) 
	if (!rcu_is_nocb_cpu(rdp->cpu))

		return false;



	/* If there are early-boot callbacks, move them to nocb lists. */

	if (rdp->nxtlist) {

		rdp->nocb_head = rdp->nxtlist;

		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];

		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);

		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);

		rdp->nxtlist = NULL;

		rdp->qlen = 0;

		rdp->qlen_lazy = 0;

	}

	rdp->nxttail[RCU_NEXT_TAIL] = NULL;

	return true;

}