Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 910ee45d authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Make rcu_accelerate_cbs() note need for future grace periods 



Now that rcu_start_future_gp() has been abstracted from
rcu_nocb_wait_gp(), rcu_accelerate_cbs() can invoke rcu_start_future_gp()
so as to register the need for any future grace periods needed by a
CPU about to enter dyntick-idle mode.  This commit makes this change.
Note that some refactoring of rcu_start_gp() is carried out to avoid
recursion and subsequent self-deadlocks.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 0446be48

kernel/rcutree.c

+32 −18
Original line number Diff line number Diff line
@@ -224,7 +224,8 @@ static ulong jiffies_till_next_fqs = RCU_JIFFIES_TILL_FORCE_QS; 
module_param(jiffies_till_first_fqs, ulong, 0644);

module_param(jiffies_till_next_fqs, ulong, 0644);



static void rcu_start_gp(struct rcu_state *rsp);

static void rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,

				  struct rcu_data *rdp);

static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *));

static void force_quiescent_state(struct rcu_state *rsp);

static int rcu_pending(int cpu);
@@ -1162,7 +1163,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp) 
		trace_rcu_future_gp(rnp, rdp, c, "Startedleafroot");

	} else {

		trace_rcu_future_gp(rnp, rdp, c, "Startedroot");

		rcu_start_gp(rdp->rsp);

		rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp);

	}

unlock_out:

	if (rnp != rnp_root)
@@ -1248,6 +1249,8 @@ static void rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, 
		rdp->nxttail[i] = rdp->nxttail[RCU_NEXT_TAIL];

		rdp->nxtcompleted[i] = c;

	}

	/* Record any needed additional grace periods. */

	rcu_start_future_gp(rnp, rdp);



	/* Trace depending on how much we were able to accelerate. */

	if (!*rdp->nxttail[RCU_WAIT_TAIL])
@@ -1609,20 +1612,9 @@ static int __noreturn rcu_gp_kthread(void *arg) 
 * quiescent state.

 */

static void

rcu_start_gp(struct rcu_state *rsp)

rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,

		      struct rcu_data *rdp)

{

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);

	struct rcu_node *rnp = rcu_get_root(rsp);



	/*

	 * If there is no grace period in progress right now, any

	 * callbacks we have up to this point will be satisfied by the

	 * next grace period.  Also, advancing the callbacks reduces the

	 * probability of false positives from cpu_needs_another_gp()

	 * resulting in pointless grace periods.  So, advance callbacks!

	 */

	rcu_advance_cbs(rsp, rnp, rdp);



	if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) {

		/*

		 * Either we have not yet spawned the grace-period
@@ -1634,13 +1626,35 @@ rcu_start_gp(struct rcu_state *rsp) 
	}

	rsp->gp_flags = RCU_GP_FLAG_INIT;



	/* Ensure that CPU is aware of completion of last grace period. */

	__rcu_process_gp_end(rsp, rdp->mynode, rdp);



	/* Wake up rcu_gp_kthread() to start the grace period. */

	wake_up(&rsp->gp_wq);

}



/*

 * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's

 * callbacks.  Note that rcu_start_gp_advanced() cannot do this because it

 * is invoked indirectly from rcu_advance_cbs(), which would result in

 * endless recursion -- or would do so if it wasn't for the self-deadlock

 * that is encountered beforehand.

 */

static void

rcu_start_gp(struct rcu_state *rsp)

{

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);

	struct rcu_node *rnp = rcu_get_root(rsp);



	/*

	 * If there is no grace period in progress right now, any

	 * callbacks we have up to this point will be satisfied by the

	 * next grace period.  Also, advancing the callbacks reduces the

	 * probability of false positives from cpu_needs_another_gp()

	 * resulting in pointless grace periods.  So, advance callbacks

	 * then start the grace period!

	 */

	rcu_advance_cbs(rsp, rnp, rdp);

	rcu_start_gp_advanced(rsp, rnp, rdp);

}



/*

 * Report a full set of quiescent states to the specified rcu_state

 * data structure.  This involves cleaning up after the prior grace