Merge "rcu: Stop disabling interrupts in scheduler fastpaths" (fe470ca9) · Commits · e / devices / android_kernel_sony_msm8998

include/linux/rcutree.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -37,7 +37,7 @@ void rcu_cpu_stall_reset(void);
		/*
		* Note a virtualization-based context switch. This is simply a
		* wrapper around rcu_note_context_switch(), which allows TINY_RCU
		* to save a few bytes.
		* to save a few bytes. The caller must have disabled interrupts.
		*/
		static inline void rcu_virt_note_context_switch(int cpu)
		{

kernel/rcu/tree.c

+20 −24

Original line number	Diff line number	Diff line
		@@ -248,24 +248,17 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
		*/
		void rcu_sched_qs(void)
		{
		unsigned long flags;

		if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {
		if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
		return;
		trace_rcu_grace_period(TPS("rcu_sched"),
		__this_cpu_read(rcu_sched_data.gpnum),
		TPS("cpuqs"));
		__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
		if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
		return;
		local_irq_save(flags);
		if (__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) {
		__this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
		rcu_report_exp_rdp(&rcu_sched_state,
		this_cpu_ptr(&rcu_sched_data),
		true);
		}
		local_irq_restore(flags);
		}
		this_cpu_ptr(&rcu_sched_data), true);
		}

		void rcu_bh_qs(void)
		@@ -302,17 +295,16 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
		* We inform the RCU core by emulating a zero-duration dyntick-idle
		* period, which we in turn do by incrementing the ->dynticks counter
		* by two.
		*
		* The caller must have disabled interrupts.
		*/
		static void rcu_momentary_dyntick_idle(void)
		{
		unsigned long flags;
		struct rcu_data *rdp;
		struct rcu_dynticks *rdtp;
		int resched_mask;
		struct rcu_state *rsp;

		local_irq_save(flags);

		/*
		* Yes, we can lose flag-setting operations. This is OK, because
		* the flag will be set again after some delay.
		@@ -342,13 +334,12 @@ static void rcu_momentary_dyntick_idle(void)
		smp_mb__after_atomic(); /* Later stuff after QS. */
		break;
		}
		local_irq_restore(flags);
		}

		/*
		* Note a context switch. This is a quiescent state for RCU-sched,
		* and requires special handling for preemptible RCU.
		* The caller must have disabled preemption.
		* The caller must have disabled interrupts.
		*/
		void rcu_note_context_switch(void)
		{
		@@ -378,9 +369,14 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);
		*/
		void rcu_all_qs(void)
		{
		unsigned long flags;

		barrier(); /* Avoid RCU read-side critical sections leaking down. */
		if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
		if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) {
		local_irq_save(flags);
		rcu_momentary_dyntick_idle();
		local_irq_restore(flags);
		}
		this_cpu_inc(rcu_qs_ctr);
		barrier(); /* Avoid RCU read-side critical sections leaking up. */
		}

kernel/rcu/tree_plugin.h

+6 −8

Original line number	Diff line number	Diff line
		@@ -147,8 +147,8 @@ static void __init rcu_bootup_announce(void)
		* the corresponding expedited grace period will also be the end of the
		* normal grace period.
		*/
		static void rcu_preempt_ctxt_queue(struct rcu_node rnp, struct rcu_data rdp,
		unsigned long flags) __releases(rnp->lock)
		static void rcu_preempt_ctxt_queue(struct rcu_node rnp, struct rcu_data rdp)
		__releases(rnp->lock) /* But leaves rrupts disabled. */
		{
		int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
		(rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
		@@ -236,7 +236,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node rnp, struct rcu_data rdp,
		rnp->gp_tasks = &t->rcu_node_entry;
		if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
		rnp->exp_tasks = &t->rcu_node_entry;
		raw_spin_unlock(&rnp->lock);
		raw_spin_unlock(&rnp->lock); /* rrupts remain disabled. */

		/*
		* Report the quiescent state for the expedited GP. This expedited
		@@ -251,7 +251,6 @@ static void rcu_preempt_ctxt_queue(struct rcu_node rnp, struct rcu_data rdp,
		} else {
		WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
		}
		local_irq_restore(flags);
		}

		/*
		@@ -286,12 +285,11 @@ static void rcu_preempt_qs(void)
		* predating the current grace period drain, in other words, until
		* rnp->gp_tasks becomes NULL.
		*
		* Caller must disable preemption.
		* Caller must disable interrupts.
		*/
		static void rcu_preempt_note_context_switch(void)
		{
		struct task_struct *t = current;
		unsigned long flags;
		struct rcu_data *rdp;
		struct rcu_node *rnp;

		@@ -301,7 +299,7 @@ static void rcu_preempt_note_context_switch(void)
		/* Possibly blocking in an RCU read-side critical section. */
		rdp = this_cpu_ptr(rcu_state_p->rda);
		rnp = rdp->mynode;
		raw_spin_lock_irqsave(&rnp->lock, flags);
		raw_spin_lock(&rnp->lock);
		smp_mb__after_unlock_lock();
		t->rcu_read_unlock_special.b.blocked = true;
		t->rcu_blocked_node = rnp;
		@@ -318,7 +316,7 @@ static void rcu_preempt_note_context_switch(void)
		(rnp->qsmask & rdp->grpmask)
		? rnp->gpnum
		: rnp->gpnum + 1);
		rcu_preempt_ctxt_queue(rnp, rdp, flags);
		rcu_preempt_ctxt_queue(rnp, rdp);
		} else if (t->rcu_read_lock_nesting < 0 &&
		t->rcu_read_unlock_special.s) {

kernel/sched/core.c

+4 −2

Original line number	Diff line number	Diff line
		@@ -3511,7 +3511,6 @@ static void __sched notrace __schedule(bool preempt)

		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
		rcu_note_context_switch();
		prev = rq->curr;

		/*
		@@ -3530,13 +3529,16 @@ static void __sched notrace __schedule(bool preempt)
		if (sched_feat(HRTICK))
		hrtick_clear(rq);

		local_irq_disable();
		rcu_note_context_switch();

		/*
		* Make sure that signal_pending_state()->signal_pending() below
		* can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
		* done by the caller to avoid the race with signal_wake_up().
		*/
		smp_mb__before_spinlock();
		raw_spin_lock_irq(&rq->lock);
		raw_spin_lock(&rq->lock);
		lockdep_pin_lock(&rq->lock);

		rq->clock_skip_update <<= 1; /* promote REQ to ACT */