Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6e275637 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

generic-ipi: remove CSD_FLAG_WAIT 



Oleg noticed that we don't strictly need CSD_FLAG_WAIT, rework
the code so that we can use CSD_FLAG_LOCK for both purposes.

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Jens Axboe <jens.axboe@oracle.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 8969a5ed

block/blk-softirq.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -64,7 +64,7 @@ static int raise_blk_irq(int cpu, struct request *rq) 
		data->info = rq;

		data->info = rq;

		data->flags = 0;

		data->flags = 0;





		__smp_call_function_single(cpu, data);

		__smp_call_function_single(cpu, data, 0);

		return 0;

		return 0;

	}

	}

include/linux/smp.h

+2 −1
Original line number Original line Diff line number Diff line
@@ -82,7 +82,8 @@ smp_call_function_mask(cpumask_t mask, void(*func)(void *info), void *info, 
	return 0;

	return 0;

}

}





void __smp_call_function_single(int cpuid, struct call_single_data *data);

void __smp_call_function_single(int cpuid, struct call_single_data *data,

				int wait);





/*

/*

 * Generic and arch helpers

 * Generic and arch helpers

kernel/sched.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -1093,7 +1093,7 @@ static void hrtick_start(struct rq *rq, u64 delay) 
	if (rq == this_rq()) {

	if (rq == this_rq()) {

		hrtimer_restart(timer);

		hrtimer_restart(timer);

	} else if (!rq->hrtick_csd_pending) {

	} else if (!rq->hrtick_csd_pending) {

		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);

		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);

		rq->hrtick_csd_pending = 1;

		rq->hrtick_csd_pending = 1;

	}

	}

}

}

kernel/smp.c

+23 −67
Original line number Original line Diff line number Diff line
@@ -23,8 +23,7 @@ static struct { 
};

};





enum {

enum {

	CSD_FLAG_WAIT		= 0x01,

	CSD_FLAG_LOCK		= 0x01,

	CSD_FLAG_LOCK		= 0x02,

};

};





struct call_function_data {

struct call_function_data {
@@ -94,31 +93,6 @@ static int __cpuinit init_call_single_data(void) 
}

}

early_initcall(init_call_single_data);

early_initcall(init_call_single_data);





/*

 * csd_wait/csd_complete are used for synchronous ipi calls

 */

static void csd_wait_prepare(struct call_single_data *data)

{

	data->flags |= CSD_FLAG_WAIT;

}



static void csd_complete(struct call_single_data *data)

{

	if (data->flags & CSD_FLAG_WAIT) {

		/*

		 * ensure we're all done before saying we are

		 */

		smp_mb();

		data->flags &= ~CSD_FLAG_WAIT;

	}

}



static void csd_wait(struct call_single_data *data)

{

	while (data->flags & CSD_FLAG_WAIT)

		cpu_relax();

}



/*

/*

 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources

 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources

 *

 *
@@ -126,10 +100,15 @@ static void csd_wait(struct call_single_data *data) 
 * function call. For multi-cpu calls its even more interesting as we'll have

 * function call. For multi-cpu calls its even more interesting as we'll have

 * to ensure no other cpu is observing our csd.

 * to ensure no other cpu is observing our csd.

 */

 */

static void csd_lock(struct call_single_data *data)

static void csd_lock_wait(struct call_single_data *data)

{

{

	while (data->flags & CSD_FLAG_LOCK)

	while (data->flags & CSD_FLAG_LOCK)

		cpu_relax();

		cpu_relax();

}



static void csd_lock(struct call_single_data *data)

{

	csd_lock_wait(data);

	data->flags = CSD_FLAG_LOCK;

	data->flags = CSD_FLAG_LOCK;





	/*

	/*
@@ -155,11 +134,12 @@ static void csd_unlock(struct call_single_data *data) 
 * Insert a previously allocated call_single_data element for execution

 * Insert a previously allocated call_single_data element for execution

 * on the given CPU. data must already have ->func, ->info, and ->flags set.

 * on the given CPU. data must already have ->func, ->info, and ->flags set.

 */

 */

static void generic_exec_single(int cpu, struct call_single_data *data)

static

void generic_exec_single(int cpu, struct call_single_data *data, int wait)

{

{

	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);

	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);

	int wait = data->flags & CSD_FLAG_WAIT, ipi;

	unsigned long flags;

	unsigned long flags;

	int ipi;





	spin_lock_irqsave(&dst->lock, flags);

	spin_lock_irqsave(&dst->lock, flags);

	ipi = list_empty(&dst->list);

	ipi = list_empty(&dst->list);
@@ -182,7 +162,7 @@ static void generic_exec_single(int cpu, struct call_single_data *data) 
		arch_send_call_function_single_ipi(cpu);

		arch_send_call_function_single_ipi(cpu);





	if (wait)

	if (wait)

		csd_wait(data);

		csd_lock_wait(data);

}

}





/*

/*
@@ -232,7 +212,6 @@ void generic_smp_call_function_interrupt(void) 
		if (refs)

		if (refs)

			continue;

			continue;





		csd_complete(&data->csd);

		csd_unlock(&data->csd);

		csd_unlock(&data->csd);

	}

	}
@@ -270,9 +249,6 @@ void generic_smp_call_function_single_interrupt(void) 




		data->func(data->info);

		data->func(data->info);





		if (data_flags & CSD_FLAG_WAIT)

			csd_complete(data);



		/*

		/*

		 * Unlocked CSDs are valid through generic_exec_single()

		 * Unlocked CSDs are valid through generic_exec_single()

		 */

		 */
@@ -313,36 +289,16 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, 
		func(info);

		func(info);

		local_irq_restore(flags);

		local_irq_restore(flags);

	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {

	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {

		struct call_single_data *data;

		struct call_single_data *data = &d;





		if (!wait) {

		if (!wait)

			/*

			 * We are calling a function on a single CPU

			 * and we are not going to wait for it to finish.

			 * We use a per cpu data to pass the information to

			 * that CPU. Since all callers of this code will

			 * use the same data, we must synchronize the

			 * callers to prevent a new caller from corrupting

			 * the data before the callee can access it.

			 *

			 * The CSD_FLAG_LOCK is used to let us know when

			 * the IPI handler is done with the data.

			 * The first caller will set it, and the callee

			 * will clear it. The next caller must wait for

			 * it to clear before we set it again. This

			 * will make sure the callee is done with the

			 * data before a new caller will use it.

			 */

			data = &__get_cpu_var(csd_data);

			data = &__get_cpu_var(csd_data);



		csd_lock(data);

		csd_lock(data);

		} else {

			data = &d;

			csd_wait_prepare(data);

		}





		data->func = func;

		data->func = func;

		data->info = info;

		data->info = info;

		generic_exec_single(cpu, data);

		generic_exec_single(cpu, data, wait);

	} else {

	} else {

		err = -ENXIO;	/* CPU not online */

		err = -ENXIO;	/* CPU not online */

	}

	}
@@ -362,12 +318,15 @@ EXPORT_SYMBOL(smp_call_function_single); 
 * instance.

 * instance.

 *

 *

 */

 */

void __smp_call_function_single(int cpu, struct call_single_data *data)

void __smp_call_function_single(int cpu, struct call_single_data *data,

				int wait)

{

{

	csd_lock(data);



	/* Can deadlock when called with interrupts disabled */

	/* Can deadlock when called with interrupts disabled */

	WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());

	WARN_ON(wait && irqs_disabled());





	generic_exec_single(cpu, data);

	generic_exec_single(cpu, data, wait);

}

}





/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */

/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
@@ -425,9 +384,6 @@ void smp_call_function_many(const struct cpumask *mask, 
	csd_lock(&data->csd);

	csd_lock(&data->csd);





	spin_lock_irqsave(&data->lock, flags);

	spin_lock_irqsave(&data->lock, flags);

	if (wait)

		csd_wait_prepare(&data->csd);



	data->csd.func = func;

	data->csd.func = func;

	data->csd.info = info;

	data->csd.info = info;

	cpumask_and(data->cpumask, mask, cpu_online_mask);

	cpumask_and(data->cpumask, mask, cpu_online_mask);
@@ -456,7 +412,7 @@ void smp_call_function_many(const struct cpumask *mask, 




	/* optionally wait for the CPUs to complete */

	/* optionally wait for the CPUs to complete */

	if (wait)

	if (wait)

		csd_wait(&data->csd);

		csd_lock_wait(&data->csd);

}

}

EXPORT_SYMBOL(smp_call_function_many);

EXPORT_SYMBOL(smp_call_function_many);

kernel/softirq.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -496,7 +496,7 @@ static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softir 
		cp->flags = 0;

		cp->flags = 0;

		cp->priv = softirq;

		cp->priv = softirq;





		__smp_call_function_single(cpu, cp);

		__smp_call_function_single(cpu, cp, 0);

		return 0;

		return 0;

	}

	}

	return 1;

	return 1;