Merge "timers: Fix excessive granularity of new timers after a nohz idle" (d0807d61) · Commits · e / devices / android_kernel_fairphone_FP3

kernel/time/timer.c

+41 −9

Original line number	Diff line number	Diff line
		@@ -201,6 +201,7 @@ struct timer_base {
		bool migration_enabled;
		bool nohz_active;
		bool is_idle;
		bool must_forward_clk;
		DECLARE_BITMAP(pending_map, WHEEL_SIZE);
		struct hlist_head vectors[WHEEL_SIZE];
		} ____cacheline_aligned;
		@@ -874,13 +875,19 @@ get_target_base(struct timer_base *base, unsigned tflags)

		static inline void forward_timer_base(struct timer_base *base)
		{
		unsigned long jnow = READ_ONCE(jiffies);
		unsigned long jnow;

		/*
		* We only forward the base when it's idle and we have a delta between
		* base clock and jiffies.
		* We only forward the base when we are idle or have just come out of
		* idle (must_forward_clk logic), and have a delta between base clock
		* and jiffies. In the common case, run_timers will take care of it.
		*/
		if (!base->is_idle \|\| (long) (jnow - base->clk) < 2)
		if (likely(!base->must_forward_clk))
		return;

		jnow = READ_ONCE(jiffies);
		base->must_forward_clk = base->is_idle;
		if ((long)(jnow - base->clk) < 2)
		return;

		/*
		@@ -956,6 +963,11 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
		* same array bucket then just return:
		*/
		if (timer_pending(timer)) {
		/*
		* The downside of this optimization is that it can result in
		* larger granularity than you would get from adding a new
		* timer with this expiry.
		*/
		if (timer->expires == expires)
		return 1;

		@@ -966,6 +978,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
		* dequeue/enqueue dance.
		*/
		base = lock_timer_base(timer, &flags);
		forward_timer_base(base);

		clk = base->clk;
		idx = calc_wheel_index(expires, clk);
		@@ -982,6 +995,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
		}
		} else {
		base = lock_timer_base(timer, &flags);
		forward_timer_base(base);
		}

		ret = detach_if_pending(timer, base, false);
		@@ -1009,12 +1023,10 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
		spin_lock(&base->lock);
		WRITE_ONCE(timer->flags,
		(timer->flags & ~TIMER_BASEMASK) \| base->cpu);
		forward_timer_base(base);
		}
		}

		/* Try to forward a stale timer base clock */
		forward_timer_base(base);

		timer->expires = expires;
		/*
		* If 'idx' was calculated above and the base time did not advance
		@@ -1130,6 +1142,7 @@ void add_timer_on(struct timer_list *timer, int cpu)
		WRITE_ONCE(timer->flags,
		(timer->flags & ~TIMER_BASEMASK) \| cpu);
		}
		forward_timer_base(base);

		debug_activate(timer, timer->expires);
		internal_add_timer(base, timer);
		@@ -1546,11 +1559,17 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
		if (!is_max_delta)
		expires = basem + (nextevt - basej) * TICK_NSEC;
		/*
		* If we expect to sleep more than a tick, mark the base idle:
		*/
		if ((expires - basem) > TICK_NSEC)
		* If we expect to sleep more than a tick, mark the base idle.
		* Also the tick is stopped so any added timer must forward
		* the base clk itself to keep granularity small. This idle
		* logic is only maintained for the BASE_STD base, deferrable
		* timers may still see large granularity skew (by design).
		*/
		if ((expires - basem) > TICK_NSEC) {
		base->must_forward_clk = true;
		base->is_idle = true;
		}
		}
		spin_unlock(&base->lock);

		return cmp_next_hrtimer_event(basem, expires);
		@@ -1659,6 +1678,19 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h)
		{
		struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);

		/*
		* must_forward_clk must be cleared before running timers so that any
		* timer functions that call mod_timer will not try to forward the
		* base. idle trcking / clock forwarding logic is only used with
		* BASE_STD timers.
		*
		* The deferrable base does not do idle tracking at all, so we do
		* not forward it. This can result in very large variations in
		* granularity for deferrable timers, but they can be deferred for
		* long periods due to idle.
		*/
		base->must_forward_clk = false;

		__run_timers(base);
		if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active)
		__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));