Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e5601400 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Simplify offline processing 



Move ->qsmaskinit and blkd_tasks[] manipulation to the CPU_DYING
notifier.  This simplifies the code by eliminating a potential
deadlock and by reducing the responsibilities of force_quiescent_state().
Also rename functions to make their connection to the CPU-hotplug
stages explicit.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent ae1f18e4

kernel/rcutree.c

+75 −85
Original line number Diff line number Diff line
@@ -943,6 +943,10 @@ rcu_start_gp_per_cpu(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat 
 * in preparation for detecting the next grace period.  The caller must hold

 * the root node's ->lock, which is released before return.  Hard irqs must

 * be disabled.

 *

 * Note that it is legal for a dying CPU (which is marked as offline) to

 * invoke this function.  This can happen when the dying CPU reports its

 * quiescent state.

 */

static void

rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
@@ -1245,22 +1249,32 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 


/*

 * Move a dying CPU's RCU callbacks to online CPU's callback list.

 * Synchronization is not required because this function executes

 * in stop_machine() context.

 * Also record a quiescent state for this CPU for the current grace period.

 * Synchronization and interrupt disabling are not required because

 * this function executes in stop_machine() context.  Therefore, cleanup

 * operations that might block must be done later from the CPU_DEAD

 * notifier.

 *

 * Note that the outgoing CPU's bit has already been cleared in the

 * cpu_online_mask.  This allows us to randomly pick a callback

 * destination from the bits set in that mask.

 */

static void rcu_send_cbs_to_online(struct rcu_state *rsp)

static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)

{

	unsigned long flags;

	int i;

	/* current DYING CPU is cleared in the cpu_online_mask */

	unsigned long mask;

	int need_report;

	int receive_cpu = cpumask_any(cpu_online_mask);

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);

	struct rcu_data *receive_rdp = per_cpu_ptr(rsp->rda, receive_cpu);

	struct rcu_node *rnp = rdp->mynode; /* For dying CPU. */



	if (rdp->nxtlist == NULL)

		return;  /* irqs disabled, so comparison is stable. */



	/* Move callbacks to some other CPU. */

	if (rdp->nxtlist != NULL) {

		*receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;

	receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];

		receive_rdp->nxttail[RCU_NEXT_TAIL] =

				rdp->nxttail[RCU_NEXT_TAIL];

		receive_rdp->qlen_lazy += rdp->qlen_lazy;

		receive_rdp->qlen += rdp->qlen;

		receive_rdp->n_cbs_adopted += rdp->qlen;
@@ -1273,90 +1287,63 @@ static void rcu_send_cbs_to_online(struct rcu_state *rsp) 
		rdp->qlen = 0;

	}



	/* Record a quiescent state for the dying CPU. */

	mask = rdp->grpmask;	/* rnp->grplo is constant. */

	trace_rcu_grace_period(rsp->name,

			       rnp->gpnum + 1 - !!(rnp->qsmask & mask),

			       "cpuofl");

	rcu_report_qs_rdp(smp_processor_id(), rsp, rdp, rsp->gpnum);

	/* Note that rcu_report_qs_rdp() might call trace_rcu_grace_period(). */



	/*

 * Remove the outgoing CPU from the bitmasks in the rcu_node hierarchy

 * and move all callbacks from the outgoing CPU to the current one.

 * There can only be one CPU hotplug operation at a time, so no other

 * CPU can be attempting to update rcu_cpu_kthread_task.

	 * Remove the dying CPU from the bitmasks in the rcu_node

	 * hierarchy.  Because we are in stop_machine() context, we

	 * automatically exclude ->onofflock critical sections.

	 */

static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)

{

	unsigned long flags;

	unsigned long mask;

	int need_report = 0;

	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);

	struct rcu_node *rnp;



	rcu_stop_cpu_kthread(cpu);



	/* Exclude any attempts to start a new grace period. */

	raw_spin_lock_irqsave(&rsp->onofflock, flags);



	/* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */

	rnp = rdp->mynode;	/* this is the outgoing CPU's rnp. */

	mask = rdp->grpmask;	/* rnp->grplo is constant. */

	do {

		raw_spin_lock(&rnp->lock);	/* irqs already disabled. */

		raw_spin_lock_irqsave(&rnp->lock, flags);

		rnp->qsmaskinit &= ~mask;

		if (rnp->qsmaskinit != 0) {

			if (rnp != rdp->mynode)

				raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */

			else

				trace_rcu_grace_period(rsp->name,

						       rnp->gpnum + 1 -

						       !!(rnp->qsmask & mask),

						       "cpuofl");

			raw_spin_unlock_irqrestore(&rnp->lock, flags);

			break;

		}

		if (rnp == rdp->mynode) {

			trace_rcu_grace_period(rsp->name,

					       rnp->gpnum + 1 -

					       !!(rnp->qsmask & mask),

					       "cpuofl");

			need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);

		} else

			raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */

		mask = rnp->grpmask;

		rnp = rnp->parent;

	} while (rnp != NULL);



	/*

	 * We still hold the leaf rcu_node structure lock here, and

	 * irqs are still disabled.  The reason for this subterfuge is

	 * because invoking rcu_report_unblock_qs_rnp() with ->onofflock

	 * held leads to deadlock.

	 */

	raw_spin_unlock(&rsp->onofflock); /* irqs remain disabled. */

	rnp = rdp->mynode;

			if (need_report & RCU_OFL_TASKS_NORM_GP)

				rcu_report_unblock_qs_rnp(rnp, flags);

			else

				raw_spin_unlock_irqrestore(&rnp->lock, flags);

			if (need_report & RCU_OFL_TASKS_EXP_GP)

				rcu_report_exp_rnp(rsp, rnp, true);

	rcu_node_kthread_setaffinity(rnp, -1);

		} else

			raw_spin_unlock_irqrestore(&rnp->lock, flags);

		mask = rnp->grpmask;

		rnp = rnp->parent;

	} while (rnp != NULL);

}



/*

 * Remove the specified CPU from the RCU hierarchy and move any pending

 * callbacks that it might have to the current CPU.  This code assumes

 * that at least one CPU in the system will remain running at all times.

 * Any attempt to offline -all- CPUs is likely to strand RCU callbacks.

 * The CPU has been completely removed, and some other CPU is reporting

 * this fact from process context.  Do the remainder of the cleanup.

 * There can only be one CPU hotplug operation at a time, so no other

 * CPU can be attempting to update rcu_cpu_kthread_task.

 */

static void rcu_offline_cpu(int cpu)

static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)

{

	__rcu_offline_cpu(cpu, &rcu_sched_state);

	__rcu_offline_cpu(cpu, &rcu_bh_state);

	rcu_preempt_offline_cpu(cpu);

	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);

	struct rcu_node *rnp = rdp->mynode;



	rcu_stop_cpu_kthread(cpu);

	rcu_node_kthread_setaffinity(rnp, -1);

}



#else /* #ifdef CONFIG_HOTPLUG_CPU */



static void rcu_send_cbs_to_online(struct rcu_state *rsp)

static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)

{

}



static void rcu_offline_cpu(int cpu)

static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)

{

}
@@ -1725,6 +1712,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	 * a quiescent state betweentimes.

	 */

	local_irq_save(flags);

	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

	rdp = this_cpu_ptr(rsp->rda);



	/* Add the callback to our list. */
@@ -2155,16 +2143,18 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self, 
		 * touch any data without introducing corruption. We send the

		 * dying CPU's callbacks to an arbitrarily chosen online CPU.

		 */

		rcu_send_cbs_to_online(&rcu_bh_state);

		rcu_send_cbs_to_online(&rcu_sched_state);

		rcu_preempt_send_cbs_to_online();

		rcu_cleanup_dying_cpu(&rcu_bh_state);

		rcu_cleanup_dying_cpu(&rcu_sched_state);

		rcu_preempt_cleanup_dying_cpu();

		rcu_cleanup_after_idle(cpu);

		break;

	case CPU_DEAD:

	case CPU_DEAD_FROZEN:

	case CPU_UP_CANCELED:

	case CPU_UP_CANCELED_FROZEN:

		rcu_offline_cpu(cpu);

		rcu_cleanup_dead_cpu(cpu, &rcu_bh_state);

		rcu_cleanup_dead_cpu(cpu, &rcu_sched_state);

		rcu_preempt_cleanup_dead_cpu(cpu);

		break;

	default:

		break;

kernel/rcutree.h

+2 −2
Original line number Diff line number Diff line
@@ -439,8 +439,8 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); 
static int rcu_preempt_offline_tasks(struct rcu_state *rsp,

				     struct rcu_node *rnp,

				     struct rcu_data *rdp);

static void rcu_preempt_offline_cpu(int cpu);

#endif /* #ifdef CONFIG_HOTPLUG_CPU */

static void rcu_preempt_cleanup_dead_cpu(int cpu);

static void rcu_preempt_check_callbacks(int cpu);

static void rcu_preempt_process_callbacks(void);

void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
@@ -451,7 +451,7 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, 
static int rcu_preempt_pending(int cpu);

static int rcu_preempt_needs_cpu(int cpu);

static void __cpuinit rcu_preempt_init_percpu_data(int cpu);

static void rcu_preempt_send_cbs_to_online(void);

static void rcu_preempt_cleanup_dying_cpu(void);

static void __init __rcu_init_preempt(void);

static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);

static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);

kernel/rcutree_plugin.h

+13 −12
Original line number Diff line number Diff line
@@ -618,16 +618,16 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, 
	return retval;

}



#endif /* #ifdef CONFIG_HOTPLUG_CPU */



/*

 * Do CPU-offline processing for preemptible RCU.

 */

static void rcu_preempt_offline_cpu(int cpu)

static void rcu_preempt_cleanup_dead_cpu(int cpu)

{

	__rcu_offline_cpu(cpu, &rcu_preempt_state);

	rcu_cleanup_dead_cpu(cpu, &rcu_preempt_state);

}



#endif /* #ifdef CONFIG_HOTPLUG_CPU */



/*

 * Check for a quiescent state from the current CPU.  When a task blocks,

 * the task is recorded in the corresponding CPU's rcu_node structure,
@@ -912,11 +912,12 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) 
}



/*

 * Move preemptible RCU's callbacks from dying CPU to other online CPU.

 * Move preemptible RCU's callbacks from dying CPU to other online CPU

 * and record a quiescent state.

 */

static void rcu_preempt_send_cbs_to_online(void)

static void rcu_preempt_cleanup_dying_cpu(void)

{

	rcu_send_cbs_to_online(&rcu_preempt_state);

	rcu_cleanup_dying_cpu(&rcu_preempt_state);

}



/*
@@ -1052,16 +1053,16 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, 
	return 0;

}



#endif /* #ifdef CONFIG_HOTPLUG_CPU */



/*

 * Because preemptible RCU does not exist, it never needs CPU-offline

 * processing.

 */

static void rcu_preempt_offline_cpu(int cpu)

static void rcu_preempt_cleanup_dead_cpu(int cpu)

{

}



#endif /* #ifdef CONFIG_HOTPLUG_CPU */



/*

 * Because preemptible RCU does not exist, it never has any callbacks

 * to check.
@@ -1153,9 +1154,9 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) 
}



/*

 * Because there is no preemptible RCU, there are no callbacks to move.

 * Because there is no preemptible RCU, there is no cleanup to do.

 */

static void rcu_preempt_send_cbs_to_online(void)

static void rcu_preempt_cleanup_dying_cpu(void)

{

}