Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e6253970 authored by Oleg Nesterov's avatar Oleg Nesterov Committed by Ingo Molnar
Browse files

stop_machine: Remove stop_cpus_lock and lg_double_lock/unlock() 



stop_two_cpus() and stop_cpus() use stop_cpus_lock to avoid the deadlock,
we need to ensure that the stopper functions can't be queued "backwards"
from one another. This doesn't look nice; if we use lglock then we do not
really need stopper->lock, cpu_stop_queue_work() could use lg_local_lock()
under local_irq_save().

OTOH it would be even better to avoid lglock in stop_machine.c and remove
lg_double_lock(). This patch adds "bool stop_cpus_in_progress" set/cleared
by queue_stop_cpus_work(), and changes cpu_stop_queue_two_works() to busy
wait until it is cleared.

queue_stop_cpus_work() sets stop_cpus_in_progress = T lockless, but after
it queues a work on CPU1 it must be visible to stop_two_cpus(CPU1, CPU2)
which checks it under the same lock. And since stop_two_cpus() holds the
2nd lock too, queue_stop_cpus_work() can not clear stop_cpus_in_progress
if it is also going to queue a work on CPU2, it needs to take that 2nd
lock to do this.

Signed-off-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20151121181148.GA433@redhat.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 87709e28

include/linux/lglock.h

+0 −5
Original line number Diff line number Diff line
@@ -52,15 +52,10 @@ struct lglock { 
	static struct lglock name = { .lock = &name ## _lock }



void lg_lock_init(struct lglock *lg, char *name);



void lg_local_lock(struct lglock *lg);

void lg_local_unlock(struct lglock *lg);

void lg_local_lock_cpu(struct lglock *lg, int cpu);

void lg_local_unlock_cpu(struct lglock *lg, int cpu);



void lg_double_lock(struct lglock *lg, int cpu1, int cpu2);

void lg_double_unlock(struct lglock *lg, int cpu1, int cpu2);



void lg_global_lock(struct lglock *lg);

void lg_global_unlock(struct lglock *lg);

kernel/locking/lglock.c

+0 −22
Original line number Diff line number Diff line
@@ -60,28 +60,6 @@ void lg_local_unlock_cpu(struct lglock *lg, int cpu) 
}

EXPORT_SYMBOL(lg_local_unlock_cpu);



void lg_double_lock(struct lglock *lg, int cpu1, int cpu2)

{

	BUG_ON(cpu1 == cpu2);



	/* lock in cpu order, just like lg_global_lock */

	if (cpu2 < cpu1)

		swap(cpu1, cpu2);



	preempt_disable();

	lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);

	arch_spin_lock(per_cpu_ptr(lg->lock, cpu1));

	arch_spin_lock(per_cpu_ptr(lg->lock, cpu2));

}



void lg_double_unlock(struct lglock *lg, int cpu1, int cpu2)

{

	lock_release(&lg->lock_dep_map, 1, _RET_IP_);

	arch_spin_unlock(per_cpu_ptr(lg->lock, cpu1));

	arch_spin_unlock(per_cpu_ptr(lg->lock, cpu2));

	preempt_enable();

}



void lg_global_lock(struct lglock *lg)

{

	int i;

kernel/stop_machine.c

+26 −16
Original line number Diff line number Diff line
@@ -20,7 +20,6 @@ 
#include <linux/kallsyms.h>

#include <linux/smpboot.h>

#include <linux/atomic.h>

#include <linux/lglock.h>

#include <linux/nmi.h>



/*
@@ -47,13 +46,9 @@ struct cpu_stopper { 
static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);

static bool stop_machine_initialized = false;



/*

 * Avoids a race between stop_two_cpus and global stop_cpus, where

 * the stoppers could get queued up in reverse order, leading to

 * system deadlock. Using an lglock means stop_two_cpus remains

 * relatively cheap.

 */

DEFINE_STATIC_LGLOCK(stop_cpus_lock);

/* static data for stop_cpus */

static DEFINE_MUTEX(stop_cpus_mutex);

static bool stop_cpus_in_progress;



static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)

{
@@ -230,14 +225,26 @@ static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1, 
	struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);

	struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);

	int err;



	lg_double_lock(&stop_cpus_lock, cpu1, cpu2);

retry:

	spin_lock_irq(&stopper1->lock);

	spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);



	err = -ENOENT;

	if (!stopper1->enabled || !stopper2->enabled)

		goto unlock;

	/*

	 * Ensure that if we race with __stop_cpus() the stoppers won't get

	 * queued up in reverse order leading to system deadlock.

	 *

	 * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has

	 * queued a work on cpu1 but not on cpu2, we hold both locks.

	 *

	 * It can be falsely true but it is safe to spin until it is cleared,

	 * queue_stop_cpus_work() does everything under preempt_disable().

	 */

	err = -EDEADLK;

	if (unlikely(stop_cpus_in_progress))

			goto unlock;



	err = 0;

	__cpu_stop_queue_work(stopper1, work1);
@@ -245,8 +252,12 @@ static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1, 
unlock:

	spin_unlock(&stopper2->lock);

	spin_unlock_irq(&stopper1->lock);

	lg_double_unlock(&stop_cpus_lock, cpu1, cpu2);



	if (unlikely(err == -EDEADLK)) {

		while (stop_cpus_in_progress)

			cpu_relax();

		goto retry;

	}

	return err;

}

/**
@@ -316,9 +327,6 @@ bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, 
	return cpu_stop_queue_work(cpu, work_buf);

}



/* static data for stop_cpus */

static DEFINE_MUTEX(stop_cpus_mutex);



static bool queue_stop_cpus_work(const struct cpumask *cpumask,

				 cpu_stop_fn_t fn, void *arg,

				 struct cpu_stop_done *done)
@@ -332,7 +340,8 @@ static bool queue_stop_cpus_work(const struct cpumask *cpumask, 
	 * preempted by a stopper which might wait for other stoppers

	 * to enter @fn which can lead to deadlock.

	 */

	lg_global_lock(&stop_cpus_lock);

	preempt_disable();

	stop_cpus_in_progress = true;

	for_each_cpu(cpu, cpumask) {

		work = &per_cpu(cpu_stopper.stop_work, cpu);

		work->fn = fn;
@@ -341,7 +350,8 @@ static bool queue_stop_cpus_work(const struct cpumask *cpumask, 
		if (cpu_stop_queue_work(cpu, work))

			queued = true;

	}

	lg_global_unlock(&stop_cpus_lock);

	stop_cpus_in_progress = false;

	preempt_enable();



	return queued;

}