Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 460775df authored by Frederic Weisbecker's avatar Frederic Weisbecker
Browse files

nohz: Optimize full dynticks state checks with static keys 



These APIs are frequenctly accessed and priority is given
to optimize the full dynticks off-case in order to let
distros enable this feature without suffering from
significant performance regressions.

Let's inline these APIs and optimize them with static keys.

Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Li Zhong <zhong@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Kevin Hilman <khilman@linaro.org>
parent 73867dcd

include/linux/tick.h

+23 −2
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ 
#include <linux/irqflags.h>

#include <linux/percpu.h>

#include <linux/hrtimer.h>

#include <linux/context_tracking_state.h>

#include <linux/cpumask.h>



#ifdef CONFIG_GENERIC_CLOCKEVENTS
@@ -158,15 +160,34 @@ static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; } 
# endif /* !CONFIG_NO_HZ_COMMON */



#ifdef CONFIG_NO_HZ_FULL

extern bool tick_nohz_full_running;

extern cpumask_var_t tick_nohz_full_mask;



static inline bool tick_nohz_full_enabled(void)

{

	if (!static_key_false(&context_tracking_enabled))

		return false;



	return tick_nohz_full_running;

}



static inline bool tick_nohz_full_cpu(int cpu)

{

	if (!tick_nohz_full_enabled())

		return false;



	return cpumask_test_cpu(cpu, tick_nohz_full_mask);

}



extern void tick_nohz_init(void);

extern int tick_nohz_full_cpu(int cpu);

extern void tick_nohz_full_check(void);

extern void tick_nohz_full_kick(void);

extern void tick_nohz_full_kick_all(void);

extern void tick_nohz_task_switch(struct task_struct *tsk);

#else

static inline void tick_nohz_init(void) { }

static inline int tick_nohz_full_cpu(int cpu) { return 0; }

static inline bool tick_nohz_full_enabled(void) { return false; }

static inline bool tick_nohz_full_cpu(int cpu) { return false; }

static inline void tick_nohz_full_check(void) { }

static inline void tick_nohz_full_kick(void) { }

static inline void tick_nohz_full_kick_all(void) { }

kernel/time/tick-sched.c

+2 −12
Original line number Diff line number Diff line
@@ -149,7 +149,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) 
}



#ifdef CONFIG_NO_HZ_FULL

static cpumask_var_t tick_nohz_full_mask;

cpumask_var_t tick_nohz_full_mask;

bool tick_nohz_full_running;



static bool can_stop_full_tick(void)
@@ -270,14 +270,6 @@ void tick_nohz_task_switch(struct task_struct *tsk) 
	local_irq_restore(flags);

}



int tick_nohz_full_cpu(int cpu)

{

	if (!tick_nohz_full_running)

		return 0;



	return cpumask_test_cpu(cpu, tick_nohz_full_mask);

}



/* Parse the boot-time nohz CPU list from the kernel parameters. */

static int __init tick_nohz_full_setup(char *str)

{
@@ -359,8 +351,6 @@ void __init tick_nohz_init(void) 
	cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), tick_nohz_full_mask);

	pr_info("NO_HZ: Full dynticks CPUs: %s.\n", nohz_full_buf);

}

#else

#define tick_nohz_full_running (0)

#endif



/*
@@ -738,7 +728,7 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) 
		return false;

	}



	if (tick_nohz_full_running) {

	if (tick_nohz_full_enabled()) {

		/*

		 * Keep the tick alive to guarantee timekeeping progression

		 * if there are full dynticks CPUs around