Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3a95398f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull NOHZ updates from Thomas Gleixner:
 "A few updates to the nohz infrastructure:

   - recursion protection for context tracking

   - make the TIF_NOHZ inheritance smarter

   - isolate cpus which belong to the NOHZ full set"

* 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  nohz: Set isolcpus when nohz_full is set
  nohz: Add tick_nohz_full_add_cpus_to() API
  context_tracking: Inherit TIF_NOHZ through forks instead of context switches
  context_tracking: Protect against recursion
parents 43224b96 8cb9764f

include/linux/context_tracking.h

+0 −10
Original line number Diff line number Diff line
@@ -14,8 +14,6 @@ extern void context_tracking_enter(enum ctx_state state); 
extern void context_tracking_exit(enum ctx_state state);

extern void context_tracking_user_enter(void);

extern void context_tracking_user_exit(void);

extern void __context_tracking_task_switch(struct task_struct *prev,

					   struct task_struct *next);



static inline void user_enter(void)

{
@@ -51,19 +49,11 @@ static inline void exception_exit(enum ctx_state prev_ctx) 
	}

}



static inline void context_tracking_task_switch(struct task_struct *prev,

						struct task_struct *next)

{

	if (context_tracking_is_enabled())

		__context_tracking_task_switch(prev, next);

}

#else

static inline void user_enter(void) { }

static inline void user_exit(void) { }

static inline enum ctx_state exception_enter(void) { return 0; }

static inline void exception_exit(enum ctx_state prev_ctx) { }

static inline void context_tracking_task_switch(struct task_struct *prev,

						struct task_struct *next) { }

#endif /* !CONFIG_CONTEXT_TRACKING */

include/linux/context_tracking_state.h

+1 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ struct context_tracking { 
	 * may be further optimized using static keys.

	 */

	bool active;

	int recursion;

	enum ctx_state {

		CONTEXT_KERNEL = 0,

		CONTEXT_USER,

include/linux/sched.h

+3 −0
Original line number Diff line number Diff line
@@ -2599,6 +2599,9 @@ static inline unsigned long wait_task_inactive(struct task_struct *p, 
}

#endif



#define tasklist_empty() \

	list_empty(&init_task.tasks)



#define next_task(p) \

	list_entry_rcu((p)->tasks.next, struct task_struct, tasks)

include/linux/tick.h

+7 −0
Original line number Diff line number Diff line
@@ -134,6 +134,12 @@ static inline bool tick_nohz_full_cpu(int cpu) 
	return cpumask_test_cpu(cpu, tick_nohz_full_mask);

}



static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask)

{

	if (tick_nohz_full_enabled())

		cpumask_or(mask, mask, tick_nohz_full_mask);

}



extern void __tick_nohz_full_check(void);

extern void tick_nohz_full_kick(void);

extern void tick_nohz_full_kick_cpu(int cpu);
@@ -142,6 +148,7 @@ extern void __tick_nohz_task_switch(struct task_struct *tsk); 
#else

static inline bool tick_nohz_full_enabled(void) { return false; }

static inline bool tick_nohz_full_cpu(int cpu) { return false; }

static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { }

static inline void __tick_nohz_full_check(void) { }

static inline void tick_nohz_full_kick_cpu(int cpu) { }

static inline void tick_nohz_full_kick(void) { }

kernel/context_tracking.c

+45 −22
Original line number Diff line number Diff line
@@ -30,12 +30,23 @@ EXPORT_SYMBOL_GPL(context_tracking_enabled); 
DEFINE_PER_CPU(struct context_tracking, context_tracking);

EXPORT_SYMBOL_GPL(context_tracking);



void context_tracking_cpu_set(int cpu)

static bool context_tracking_recursion_enter(void)

{

	if (!per_cpu(context_tracking.active, cpu)) {

		per_cpu(context_tracking.active, cpu) = true;

		static_key_slow_inc(&context_tracking_enabled);

	int recursion;



	recursion = __this_cpu_inc_return(context_tracking.recursion);

	if (recursion == 1)

		return true;



	WARN_ONCE((recursion < 1), "Invalid context tracking recursion value %d\n", recursion);

	__this_cpu_dec(context_tracking.recursion);



	return false;

}



static void context_tracking_recursion_exit(void)

{

	__this_cpu_dec(context_tracking.recursion);

}



/**
@@ -75,6 +86,9 @@ void context_tracking_enter(enum ctx_state state) 
	WARN_ON_ONCE(!current->mm);



	local_irq_save(flags);

	if (!context_tracking_recursion_enter())

		goto out_irq_restore;



	if ( __this_cpu_read(context_tracking.state) != state) {

		if (__this_cpu_read(context_tracking.active)) {

			/*
@@ -105,6 +119,8 @@ void context_tracking_enter(enum ctx_state state) 
		 */

		__this_cpu_write(context_tracking.state, state);

	}

	context_tracking_recursion_exit();

out_irq_restore:

	local_irq_restore(flags);

}

NOKPROBE_SYMBOL(context_tracking_enter);
@@ -139,6 +155,9 @@ void context_tracking_exit(enum ctx_state state) 
		return;



	local_irq_save(flags);

	if (!context_tracking_recursion_enter())

		goto out_irq_restore;



	if (__this_cpu_read(context_tracking.state) == state) {

		if (__this_cpu_read(context_tracking.active)) {

			/*
@@ -153,6 +172,8 @@ void context_tracking_exit(enum ctx_state state) 
		}

		__this_cpu_write(context_tracking.state, CONTEXT_KERNEL);

	}

	context_tracking_recursion_exit();

out_irq_restore:

	local_irq_restore(flags);

}

NOKPROBE_SYMBOL(context_tracking_exit);
@@ -164,24 +185,26 @@ void context_tracking_user_exit(void) 
}

NOKPROBE_SYMBOL(context_tracking_user_exit);



/**

 * __context_tracking_task_switch - context switch the syscall callbacks

 * @prev: the task that is being switched out

 * @next: the task that is being switched in

 *

 * The context tracking uses the syscall slow path to implement its user-kernel

 * boundaries probes on syscalls. This way it doesn't impact the syscall fast

 * path on CPUs that don't do context tracking.

 *

 * But we need to clear the flag on the previous task because it may later

 * migrate to some CPU that doesn't do the context tracking. As such the TIF

 * flag may not be desired there.

 */

void __context_tracking_task_switch(struct task_struct *prev,

				    struct task_struct *next)

void __init context_tracking_cpu_set(int cpu)

{

	clear_tsk_thread_flag(prev, TIF_NOHZ);

	set_tsk_thread_flag(next, TIF_NOHZ);

	static __initdata bool initialized = false;



	if (!per_cpu(context_tracking.active, cpu)) {

		per_cpu(context_tracking.active, cpu) = true;

		static_key_slow_inc(&context_tracking_enabled);

	}



	if (initialized)

		return;



	/*

	 * Set TIF_NOHZ to init/0 and let it propagate to all tasks through fork

	 * This assumes that init is the only task at this early boot stage.

	 */

	set_tsk_thread_flag(&init_task, TIF_NOHZ);

	WARN_ON_ONCE(!tasklist_empty());



	initialized = true;

}



#ifdef CONFIG_CONTEXT_TRACKING_FORCE