Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fb1c8f93 authored by Ingo Molnar's avatar Ingo Molnar Committed by Linus Torvalds
Browse files

[PATCH] spinlock consolidation 



This patch (written by me and also containing many suggestions of Arjan van
de Ven) does a major cleanup of the spinlock code.  It does the following
things:

 - consolidates and enhances the spinlock/rwlock debugging code

 - simplifies the asm/spinlock.h files

 - encapsulates the raw spinlock type and moves generic spinlock
   features (such as ->break_lock) into the generic code.

 - cleans up the spinlock code hierarchy to get rid of the spaghetti.

Most notably there's now only a single variant of the debugging code,
located in lib/spinlock_debug.c.  (previously we had one SMP debugging
variant per architecture, plus a separate generic one for UP builds)

Also, i've enhanced the rwlock debugging facility, it will now track
write-owners.  There is new spinlock-owner/CPU-tracking on SMP builds too.
All locks have lockup detection now, which will work for both soft and hard
spin/rwlock lockups.

The arch-level include files now only contain the minimally necessary
subset of the spinlock code - all the rest that can be generalized now
lives in the generic headers:

 include/asm-i386/spinlock_types.h       |   16
 include/asm-x86_64/spinlock_types.h     |   16

I have also split up the various spinlock variants into separate files,
making it easier to see which does what. The new layout is:

   SMP                         |  UP
   ----------------------------|-----------------------------------
   asm/spinlock_types_smp.h    |  linux/spinlock_types_up.h
   linux/spinlock_types.h      |  linux/spinlock_types.h
   asm/spinlock_smp.h          |  linux/spinlock_up.h
   linux/spinlock_api_smp.h    |  linux/spinlock_api_up.h
   linux/spinlock.h            |  linux/spinlock.h

/*
 * here's the role of the various spinlock/rwlock related include files:
 *
 * on SMP builds:
 *
 *  asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the
 *                        initializers
 *
 *  linux/spinlock_types.h:
 *                        defines the generic type and initializers
 *
 *  asm/spinlock.h:       contains the __raw_spin_*()/etc. lowlevel
 *                        implementations, mostly inline assembly code
 *
 *   (also included on UP-debug builds:)
 *
 *  linux/spinlock_api_smp.h:
 *                        contains the prototypes for the _spin_*() APIs.
 *
 *  linux/spinlock.h:     builds the final spin_*() APIs.
 *
 * on UP builds:
 *
 *  linux/spinlock_type_up.h:
 *                        contains the generic, simplified UP spinlock type.
 *                        (which is an empty structure on non-debug builds)
 *
 *  linux/spinlock_types.h:
 *                        defines the generic type and initializers
 *
 *  linux/spinlock_up.h:
 *                        contains the __raw_spin_*()/etc. version of UP
 *                        builds. (which are NOPs on non-debug, non-preempt
 *                        builds)
 *
 *   (included on UP-non-debug builds:)
 *
 *  linux/spinlock_api_up.h:
 *                        builds the _spin_*() APIs.
 *
 *  linux/spinlock.h:     builds the final spin_*() APIs.
 */

All SMP and UP architectures are converted by this patch.

arm, i386, ia64, ppc, ppc64, s390/s390x, x64 was build-tested via
crosscompilers.  m32r, mips, sh, sparc, have not been tested yet, but should
be mostly fine.

From: Grant Grundler <grundler@parisc-linux.org>

  Booted and lightly tested on a500-44 (64-bit, SMP kernel, dual CPU).
  Builds 32-bit SMP kernel (not booted or tested).  I did not try to build
  non-SMP kernels.  That should be trivial to fix up later if necessary.

  I converted bit ops atomic_hash lock to raw_spinlock_t.  Doing so avoids
  some ugly nesting of linux/*.h and asm/*.h files.  Those particular locks
  are well tested and contained entirely inside arch specific code.  I do NOT
  expect any new issues to arise with them.

 If someone does ever need to use debug/metrics with them, then they will
  need to unravel this hairball between spinlocks, atomic ops, and bit ops
  that exist only because parisc has exactly one atomic instruction: LDCW
  (load and clear word).

From: "Luck, Tony" <tony.luck@intel.com>

   ia64 fix

Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarArjan van de Ven <arjanv@infradead.org>
Signed-off-by: default avatarGrant Grundler <grundler@parisc-linux.org>
Cc: Matthew Wilcox <willy@debian.org>
Signed-off-by: default avatarHirokazu Takata <takata@linux-m32r.org>
Signed-off-by: default avatarMikael Pettersson <mikpe@csd.uu.se>
Signed-off-by: default avatarBenoit Boissinot <benoit.boissinot@ens-lyon.org>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 4327edf6

arch/alpha/kernel/alpha_ksyms.c

+0 −9
Original line number Diff line number Diff line
@@ -185,15 +185,6 @@ EXPORT_SYMBOL(smp_num_cpus); 
EXPORT_SYMBOL(smp_call_function);

EXPORT_SYMBOL(smp_call_function_on_cpu);

EXPORT_SYMBOL(_atomic_dec_and_lock);

#ifdef CONFIG_DEBUG_SPINLOCK

EXPORT_SYMBOL(_raw_spin_unlock);

EXPORT_SYMBOL(debug_spin_lock);

EXPORT_SYMBOL(debug_spin_trylock);

#endif

#ifdef CONFIG_DEBUG_RWLOCK

EXPORT_SYMBOL(_raw_write_lock);

EXPORT_SYMBOL(_raw_read_lock);

#endif

EXPORT_SYMBOL(cpu_present_mask);

#endif /* CONFIG_SMP */

arch/alpha/kernel/smp.c

+0 −172
Original line number Diff line number Diff line
@@ -989,175 +989,3 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page, 


	preempt_enable();

}
 


#ifdef CONFIG_DEBUG_SPINLOCK

void

_raw_spin_unlock(spinlock_t * lock)

{

	mb();

	lock->lock = 0;



	lock->on_cpu = -1;

	lock->previous = NULL;

	lock->task = NULL;

	lock->base_file = "none";

	lock->line_no = 0;

}



void

debug_spin_lock(spinlock_t * lock, const char *base_file, int line_no)

{

	long tmp;

	long stuck;

	void *inline_pc = __builtin_return_address(0);

	unsigned long started = jiffies;

	int printed = 0;

	int cpu = smp_processor_id();



	stuck = 1L << 30;

 try_again:



	/* Use sub-sections to put the actual loop at the end

	   of this object file's text section so as to perfect

	   branch prediction.  */

	__asm__ __volatile__(

	"1:	ldl_l	%0,%1\n"

	"	subq	%2,1,%2\n"

	"	blbs	%0,2f\n"

	"	or	%0,1,%0\n"

	"	stl_c	%0,%1\n"

	"	beq	%0,3f\n"

	"4:	mb\n"

	".subsection 2\n"

	"2:	ldl	%0,%1\n"

	"	subq	%2,1,%2\n"

	"3:	blt	%2,4b\n"

	"	blbs	%0,2b\n"

	"	br	1b\n"

	".previous"

	: "=r" (tmp), "=m" (lock->lock), "=r" (stuck)

	: "m" (lock->lock), "2" (stuck) : "memory");



	if (stuck < 0) {

		printk(KERN_WARNING

		       "%s:%d spinlock stuck in %s at %p(%d)"

		       " owner %s at %p(%d) %s:%d\n",

		       base_file, line_no,

		       current->comm, inline_pc, cpu,

		       lock->task->comm, lock->previous,

		       lock->on_cpu, lock->base_file, lock->line_no);

		stuck = 1L << 36;

		printed = 1;

		goto try_again;

	}



	/* Exiting.  Got the lock.  */

	lock->on_cpu = cpu;

	lock->previous = inline_pc;

	lock->task = current;

	lock->base_file = base_file;

	lock->line_no = line_no;



	if (printed) {

		printk(KERN_WARNING

		       "%s:%d spinlock grabbed in %s at %p(%d) %ld ticks\n",

		       base_file, line_no, current->comm, inline_pc,

		       cpu, jiffies - started);

	}

}



int

debug_spin_trylock(spinlock_t * lock, const char *base_file, int line_no)

{

	int ret;

	if ((ret = !test_and_set_bit(0, lock))) {

		lock->on_cpu = smp_processor_id();

		lock->previous = __builtin_return_address(0);

		lock->task = current;

	} else {

		lock->base_file = base_file;

		lock->line_no = line_no;

	}

	return ret;

}

#endif /* CONFIG_DEBUG_SPINLOCK */



#ifdef CONFIG_DEBUG_RWLOCK

void _raw_write_lock(rwlock_t * lock)

{

	long regx, regy;

	int stuck_lock, stuck_reader;

	void *inline_pc = __builtin_return_address(0);



 try_again:



	stuck_lock = 1<<30;

	stuck_reader = 1<<30;



	__asm__ __volatile__(

	"1:	ldl_l	%1,%0\n"

	"	blbs	%1,6f\n"

	"	blt	%1,8f\n"

	"	mov	1,%1\n"

	"	stl_c	%1,%0\n"

	"	beq	%1,6f\n"

	"4:	mb\n"

	".subsection 2\n"

	"6:	blt	%3,4b	# debug\n"

	"	subl	%3,1,%3	# debug\n"

	"	ldl	%1,%0\n"

	"	blbs	%1,6b\n"

	"8:	blt	%4,4b	# debug\n"

	"	subl	%4,1,%4	# debug\n"

	"	ldl	%1,%0\n"

	"	blt	%1,8b\n"

	"	br	1b\n"

	".previous"

	: "=m" (*(volatile int *)lock), "=&r" (regx), "=&r" (regy),

	  "=&r" (stuck_lock), "=&r" (stuck_reader)

	: "m" (*(volatile int *)lock), "3" (stuck_lock), "4" (stuck_reader) : "memory");



	if (stuck_lock < 0) {

		printk(KERN_WARNING "write_lock stuck at %p\n", inline_pc);

		goto try_again;

	}

	if (stuck_reader < 0) {

		printk(KERN_WARNING "write_lock stuck on readers at %p\n",

		       inline_pc);

		goto try_again;

	}

}



void _raw_read_lock(rwlock_t * lock)

{

	long regx;

	int stuck_lock;

	void *inline_pc = __builtin_return_address(0);



 try_again:



	stuck_lock = 1<<30;



	__asm__ __volatile__(

	"1:	ldl_l	%1,%0;"

	"	blbs	%1,6f;"

	"	subl	%1,2,%1;"

	"	stl_c	%1,%0;"

	"	beq	%1,6f;"

	"4:	mb\n"

	".subsection 2\n"

	"6:	ldl	%1,%0;"

	"	blt	%2,4b	# debug\n"

	"	subl	%2,1,%2	# debug\n"

	"	blbs	%1,6b;"

	"	br	1b\n"

	".previous"

	: "=m" (*(volatile int *)lock), "=&r" (regx), "=&r" (stuck_lock)

	: "m" (*(volatile int *)lock), "2" (stuck_lock) : "memory");



	if (stuck_lock < 0) {

		printk(KERN_WARNING "read_lock stuck at %p\n", inline_pc);

		goto try_again;

	}

}

#endif /* CONFIG_DEBUG_RWLOCK */

arch/ia64/kernel/mca.c

+1 −10
Original line number Diff line number Diff line
@@ -491,12 +491,7 @@ init_handler_platform (pal_min_state_area_t *ms, 
	unw_init_from_interruption(&info, current, pt, sw);

	ia64_do_show_stack(&info, NULL);



#ifdef CONFIG_SMP

	/* read_trylock() would be handy... */

	if (!tasklist_lock.write_lock)

		read_lock(&tasklist_lock);

#endif

	{

	if (read_trylock(&tasklist_lock)) {

		struct task_struct *g, *t;

		do_each_thread (g, t) {

			if (t == current)
@@ -506,10 +501,6 @@ init_handler_platform (pal_min_state_area_t *ms, 
			show_stack(t, NULL);

		} while_each_thread (g, t);

	}

#ifdef CONFIG_SMP

	if (!tasklist_lock.write_lock)

		read_unlock(&tasklist_lock);

#endif



	printk("\nINIT dump complete.  Please reboot now.\n");

	while (1);			/* hang city if no debugger */

arch/m32r/kernel/smp.c

+12 −36
Original line number Diff line number Diff line
@@ -892,7 +892,6 @@ unsigned long send_IPI_mask_phys(cpumask_t physid_mask, int ipi_num, 
	int try)

{

	spinlock_t *ipilock;

	unsigned long flags = 0;

	volatile unsigned long *ipicr_addr;

	unsigned long ipicr_val;

	unsigned long my_physid_mask;
@@ -916,50 +915,27 @@ unsigned long send_IPI_mask_phys(cpumask_t physid_mask, int ipi_num, 
	 * write IPICRi (send IPIi)

	 * unlock ipi_lock[i]

	 */

	spin_lock(ipilock);

	__asm__ __volatile__ (

		";; LOCK ipi_lock[i]		\n\t"

		";; CHECK IPICRi == 0		\n\t"

		".fillinsn			\n"

		"1:				\n\t"

		"mvfc	%1, psw 		\n\t"

		"clrpsw	#0x40 -> nop		\n\t"

		DCACHE_CLEAR("r4", "r5", "%2")

		"lock	r4, @%2			\n\t"

		"addi	r4, #-1			\n\t"

		"unlock	r4, @%2			\n\t"

		"mvtc	%1, psw			\n\t"

		"bnez	r4, 2f			\n\t"

		LOCK_SECTION_START(".balign 4 \n\t")

		".fillinsn			\n"

		"2:				\n\t"

		"ld	r4, @%2			\n\t"

		"blez	r4, 2b			\n\t"

		"ld	%0, @%1			\n\t"

		"and	%0, %4			\n\t"

		"beqz	%0, 2f			\n\t"

		"bnez	%3, 3f			\n\t"

		"bra	1b			\n\t"

		LOCK_SECTION_END

		";; CHECK IPICRi == 0		\n\t"

		".fillinsn			\n"

		"3:				\n\t"

		"ld	%0, @%3			\n\t"

		"and	%0, %6			\n\t"

		"beqz	%0, 4f			\n\t"

		"bnez	%5, 5f			\n\t"

		"bra	3b			\n\t"

		";; WRITE IPICRi (send IPIi)	\n\t"

		".fillinsn			\n"

		"4:				\n\t"

		"st	%4, @%3			\n\t"

		";; UNLOCK ipi_lock[i]		\n\t"

		"2:				\n\t"

		"st	%2, @%1			\n\t"

		".fillinsn			\n"

		"5:				\n\t"

		"ldi	r4, #1			\n\t"

		"st	r4, @%2			\n\t"

		"3:				\n\t"

		: "=&r"(ipicr_val)

		: "r"(flags), "r"(&ipilock->slock), "r"(ipicr_addr),

		  "r"(mask), "r"(try), "r"(my_physid_mask)

		: "memory", "r4"

#ifdef CONFIG_CHIP_M32700_TS1

		, "r5"

#endif	/* CONFIG_CHIP_M32700_TS1 */

		: "r"(ipicr_addr), "r"(mask), "r"(try), "r"(my_physid_mask)

		: "memory"

	);

	spin_unlock(ipilock);



	return ipicr_val;

}

arch/mips/lib/dec_and_lock.c

+0 −8
Original line number Diff line number Diff line
@@ -20,14 +20,7 @@ 
 * has a cmpxchg, and where atomic->value is an int holding

 * the value of the atomic (i.e. the high bits aren't used

 * for a lock or anything like that).

 *

 * N.B. ATOMIC_DEC_AND_LOCK gets defined in include/linux/spinlock.h

 * if spinlocks are empty and thus atomic_dec_and_lock is defined

 * to be atomic_dec_and_test - in that case we don't need it

 * defined here as well.

 */



#ifndef ATOMIC_DEC_AND_LOCK

int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)

{

	int counter;
@@ -52,4 +45,3 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) 
}



EXPORT_SYMBOL(_atomic_dec_and_lock);
 
#endif /* ATOMIC_DEC_AND_LOCK */