rcu: Move synchronize_sched_expedited() to rcutree.c (3d3b7db0) · Commits · e / devices / android_kernel_xiaomi_markw

kernel/rcutree.c

+117 −0

Original line number	Original line	Diff line number	Diff line
	@@ -50,6 +50,8 @@
	#include <linux/wait.h>		#include <linux/wait.h>
	#include <linux/kthread.h>		#include <linux/kthread.h>
	#include <linux/prefetch.h>		#include <linux/prefetch.h>
			#include <linux/delay.h>
			#include <linux/stop_machine.h>

	#include "rcutree.h"		#include "rcutree.h"
	#include <trace/events/rcu.h>		#include <trace/events/rcu.h>
	@@ -1918,6 +1920,121 @@ void synchronize_rcu_bh(void)
	}		}
	EXPORT_SYMBOL_GPL(synchronize_rcu_bh);		EXPORT_SYMBOL_GPL(synchronize_rcu_bh);

			static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
			static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);

			static int synchronize_sched_expedited_cpu_stop(void *data)
			{
			/*
			* There must be a full memory barrier on each affected CPU
			* between the time that try_stop_cpus() is called and the
			* time that it returns.
			*
			* In the current initial implementation of cpu_stop, the
			* above condition is already met when the control reaches
			* this point and the following smp_mb() is not strictly
			* necessary. Do smp_mb() anyway for documentation and
			* robustness against future implementation changes.
			*/
			smp_mb(); /* See above comment block. */
			return 0;
			}

			/*
			* Wait for an rcu-sched grace period to elapse, but use "big hammer"
			* approach to force grace period to end quickly. This consumes
			* significant time on all CPUs, and is thus not recommended for
			* any sort of common-case code.
			*
			* Note that it is illegal to call this function while holding any
			* lock that is acquired by a CPU-hotplug notifier. Failing to
			* observe this restriction will result in deadlock.
			*
			* This implementation can be thought of as an application of ticket
			* locking to RCU, with sync_sched_expedited_started and
			* sync_sched_expedited_done taking on the roles of the halves
			* of the ticket-lock word. Each task atomically increments
			* sync_sched_expedited_started upon entry, snapshotting the old value,
			* then attempts to stop all the CPUs. If this succeeds, then each
			* CPU will have executed a context switch, resulting in an RCU-sched
			* grace period. We are then done, so we use atomic_cmpxchg() to
			* update sync_sched_expedited_done to match our snapshot -- but
			* only if someone else has not already advanced past our snapshot.
			*
			* On the other hand, if try_stop_cpus() fails, we check the value
			* of sync_sched_expedited_done. If it has advanced past our
			* initial snapshot, then someone else must have forced a grace period
			* some time after we took our snapshot. In this case, our work is
			* done for us, and we can simply return. Otherwise, we try again,
			* but keep our initial snapshot for purposes of checking for someone
			* doing our work for us.
			*
			* If we fail too many times in a row, we fall back to synchronize_sched().
			*/
			void synchronize_sched_expedited(void)
			{
			int firstsnap, s, snap, trycount = 0;

			/* Note that atomic_inc_return() implies full memory barrier. */
			firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
			get_online_cpus();
			WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

			/*
			* Each pass through the following loop attempts to force a
			* context switch on each CPU.
			*/
			while (try_stop_cpus(cpu_online_mask,
			synchronize_sched_expedited_cpu_stop,
			NULL) == -EAGAIN) {
			put_online_cpus();

			/* No joy, try again later. Or just synchronize_sched(). */
			if (trycount++ < 10)
			udelay(trycount * num_online_cpus());
			else {
			synchronize_sched();
			return;
			}

			/* Check to see if someone else did our work for us. */
			s = atomic_read(&sync_sched_expedited_done);
			if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
			smp_mb(); /* ensure test happens before caller kfree */
			return;
			}

			/*
			* Refetching sync_sched_expedited_started allows later
			* callers to piggyback on our grace period. We subtract
			* 1 to get the same token that the last incrementer got.
			* We retry after they started, so our grace period works
			* for them, and they started after our first try, so their
			* grace period works for us.
			*/
			get_online_cpus();
			snap = atomic_read(&sync_sched_expedited_started);
			smp_mb(); /* ensure read is before try_stop_cpus(). */
			}

			/*
			* Everyone up to our most recent fetch is covered by our grace
			* period. Update the counter, but only if our work is still
			* relevant -- which it won't be if someone who started later
			* than we did beat us to the punch.
			*/
			do {
			s = atomic_read(&sync_sched_expedited_done);
			if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
			smp_mb(); /* ensure test happens before caller kfree */
			break;
			}
			} while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);

			put_online_cpus();
			}
			EXPORT_SYMBOL_GPL(synchronize_sched_expedited);

	/*		/*
	* Check to see if there is any immediate RCU-related work to be done		* Check to see if there is any immediate RCU-related work to be done
	* by the current CPU, for the specified type of RCU, returning 1 if so.		* by the current CPU, for the specified type of RCU, returning 1 if so.

kernel/rcutree_plugin.h

+0 −116

Original line number	Original line	Diff line number	Diff line
	@@ -25,7 +25,6 @@
	*/		*/

	#include <linux/delay.h>		#include <linux/delay.h>
	#include <linux/stop_machine.h>

	#define RCU_KTHREAD_PRIO 1		#define RCU_KTHREAD_PRIO 1

	@@ -1888,121 +1887,6 @@ static void __cpuinit rcu_prepare_kthreads(int cpu)

	#endif /* #else #ifdef CONFIG_RCU_BOOST */		#endif /* #else #ifdef CONFIG_RCU_BOOST */

	static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
	static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);

	static int synchronize_sched_expedited_cpu_stop(void *data)
	{
	/*
	* There must be a full memory barrier on each affected CPU
	* between the time that try_stop_cpus() is called and the
	* time that it returns.
	*
	* In the current initial implementation of cpu_stop, the
	* above condition is already met when the control reaches
	* this point and the following smp_mb() is not strictly
	* necessary. Do smp_mb() anyway for documentation and
	* robustness against future implementation changes.
	*/
	smp_mb(); /* See above comment block. */
	return 0;
	}

	/*
	* Wait for an rcu-sched grace period to elapse, but use "big hammer"
	* approach to force grace period to end quickly. This consumes
	* significant time on all CPUs, and is thus not recommended for
	* any sort of common-case code.
	*
	* Note that it is illegal to call this function while holding any
	* lock that is acquired by a CPU-hotplug notifier. Failing to
	* observe this restriction will result in deadlock.
	*
	* This implementation can be thought of as an application of ticket
	* locking to RCU, with sync_sched_expedited_started and
	* sync_sched_expedited_done taking on the roles of the halves
	* of the ticket-lock word. Each task atomically increments
	* sync_sched_expedited_started upon entry, snapshotting the old value,
	* then attempts to stop all the CPUs. If this succeeds, then each
	* CPU will have executed a context switch, resulting in an RCU-sched
	* grace period. We are then done, so we use atomic_cmpxchg() to
	* update sync_sched_expedited_done to match our snapshot -- but
	* only if someone else has not already advanced past our snapshot.
	*
	* On the other hand, if try_stop_cpus() fails, we check the value
	* of sync_sched_expedited_done. If it has advanced past our
	* initial snapshot, then someone else must have forced a grace period
	* some time after we took our snapshot. In this case, our work is
	* done for us, and we can simply return. Otherwise, we try again,
	* but keep our initial snapshot for purposes of checking for someone
	* doing our work for us.
	*
	* If we fail too many times in a row, we fall back to synchronize_sched().
	*/
	void synchronize_sched_expedited(void)
	{
	int firstsnap, s, snap, trycount = 0;

	/* Note that atomic_inc_return() implies full memory barrier. */
	firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
	get_online_cpus();
	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

	/*
	* Each pass through the following loop attempts to force a
	* context switch on each CPU.
	*/
	while (try_stop_cpus(cpu_online_mask,
	synchronize_sched_expedited_cpu_stop,
	NULL) == -EAGAIN) {
	put_online_cpus();

	/* No joy, try again later. Or just synchronize_sched(). */
	if (trycount++ < 10)
	udelay(trycount * num_online_cpus());
	else {
	synchronize_sched();
	return;
	}

	/* Check to see if someone else did our work for us. */
	s = atomic_read(&sync_sched_expedited_done);
	if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
	smp_mb(); /* ensure test happens before caller kfree */
	return;
	}

	/*
	* Refetching sync_sched_expedited_started allows later
	* callers to piggyback on our grace period. We subtract
	* 1 to get the same token that the last incrementer got.
	* We retry after they started, so our grace period works
	* for them, and they started after our first try, so their
	* grace period works for us.
	*/
	get_online_cpus();
	snap = atomic_read(&sync_sched_expedited_started);
	smp_mb(); /* ensure read is before try_stop_cpus(). */
	}

	/*
	* Everyone up to our most recent fetch is covered by our grace
	* period. Update the counter, but only if our work is still
	* relevant -- which it won't be if someone who started later
	* than we did beat us to the punch.
	*/
	do {
	s = atomic_read(&sync_sched_expedited_done);
	if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
	smp_mb(); /* ensure test happens before caller kfree */
	break;
	}
	} while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);

	put_online_cpus();
	}
	EXPORT_SYMBOL_GPL(synchronize_sched_expedited);

	#if !defined(CONFIG_RCU_FAST_NO_HZ)		#if !defined(CONFIG_RCU_FAST_NO_HZ)

	/*		/*