Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 816434ec authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-spinlocks-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 spinlock changes from Ingo Molnar:
 "The biggest change here are paravirtualized ticket spinlocks (PV
  spinlocks), which bring a nice speedup on various benchmarks.

  The KVM host side will come to you via the KVM tree"

* 'x86-spinlocks-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/kvm/guest: Fix sparse warning: "symbol 'klock_waiting' was not declared as static"
  kvm: Paravirtual ticketlocks support for linux guests running on KVM hypervisor
  kvm guest: Add configuration support to enable debug information for KVM Guests
  kvm uapi: Add KICK_CPU and PV_UNHALT definition to uapi
  xen, pvticketlock: Allow interrupts to be enabled while blocking
  x86, ticketlock: Add slowpath logic
  jump_label: Split jumplabel ratelimit
  x86, pvticketlock: When paravirtualizing ticket locks, increment by 2
  x86, pvticketlock: Use callee-save for lock_spinning
  xen, pvticketlocks: Add xen_nopvspin parameter to disable xen pv ticketlocks
  xen, pvticketlock: Xen implementation for PV ticket locks
  xen: Defer spinlock setup until boot CPU setup
  x86, ticketlock: Collapse a layer of functions
  x86, ticketlock: Don't inline _spin_unlock when using paravirt spinlocks
  x86, spinlock: Replace pv spinlocks with pv ticketlocks
parents f357a820 36bd6213

arch/x86/Kconfig

+10 −0
Original line number Original line Diff line number Diff line
@@ -632,6 +632,7 @@ config PARAVIRT_DEBUG 
config PARAVIRT_SPINLOCKS

config PARAVIRT_SPINLOCKS

	bool "Paravirtualization layer for spinlocks"

	bool "Paravirtualization layer for spinlocks"

	depends on PARAVIRT && SMP

	depends on PARAVIRT && SMP

	select UNINLINE_SPIN_UNLOCK

	---help---

	---help---

	  Paravirtualized spinlocks allow a pvops backend to replace the

	  Paravirtualized spinlocks allow a pvops backend to replace the

	  spinlock implementation with something virtualization-friendly

	  spinlock implementation with something virtualization-friendly
@@ -656,6 +657,15 @@ config KVM_GUEST 
	  underlying device model, the host provides the guest with

	  underlying device model, the host provides the guest with

	  timing infrastructure such as time of day, and system time

	  timing infrastructure such as time of day, and system time





config KVM_DEBUG_FS

	bool "Enable debug information for KVM Guests in debugfs"

	depends on KVM_GUEST && DEBUG_FS

	default n

	---help---

	  This option enables collection of various statistics for KVM guest.

	  Statistics are displayed in debugfs filesystem. Enabling this option

	  may incur significant overhead.



source "arch/x86/lguest/Kconfig"

source "arch/x86/lguest/Kconfig"





config PARAVIRT_TIME_ACCOUNTING

config PARAVIRT_TIME_ACCOUNTING

arch/x86/include/asm/kvm_para.h

+12 −2
Original line number Original line Diff line number Diff line
@@ -112,10 +112,20 @@ void kvm_async_pf_task_wait(u32 token); 
void kvm_async_pf_task_wake(u32 token);

void kvm_async_pf_task_wake(u32 token);

u32 kvm_read_and_reset_pf_reason(void);

u32 kvm_read_and_reset_pf_reason(void);

extern void kvm_disable_steal_time(void);

extern void kvm_disable_steal_time(void);

#else



#ifdef CONFIG_PARAVIRT_SPINLOCKS

void __init kvm_spinlock_init(void);

#else /* !CONFIG_PARAVIRT_SPINLOCKS */

static inline void kvm_spinlock_init(void)

{

}

#endif /* CONFIG_PARAVIRT_SPINLOCKS */



#else /* CONFIG_KVM_GUEST */

#define kvm_guest_init() do {} while (0)

#define kvm_guest_init() do {} while (0)

#define kvm_async_pf_task_wait(T) do {} while(0)

#define kvm_async_pf_task_wait(T) do {} while(0)

#define kvm_async_pf_task_wake(T) do {} while(0)

#define kvm_async_pf_task_wake(T) do {} while(0)



static inline u32 kvm_read_and_reset_pf_reason(void)

static inline u32 kvm_read_and_reset_pf_reason(void)

{

{

	return 0;

	return 0;

arch/x86/include/asm/paravirt.h

+6 −26
Original line number Original line Diff line number Diff line
@@ -712,36 +712,16 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 




#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)

#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)





static inline int arch_spin_is_locked(struct arch_spinlock *lock)

static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,

							__ticket_t ticket)

{

{

	return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);

	PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket);

}

}





static inline int arch_spin_is_contended(struct arch_spinlock *lock)

static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,

							__ticket_t ticket)

{

{

	return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);

	PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);

}

#define arch_spin_is_contended	arch_spin_is_contended



static __always_inline void arch_spin_lock(struct arch_spinlock *lock)

{

	PVOP_VCALL1(pv_lock_ops.spin_lock, lock);

}



static __always_inline void arch_spin_lock_flags(struct arch_spinlock *lock,

						  unsigned long flags)

{

	PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);

}



static __always_inline int arch_spin_trylock(struct arch_spinlock *lock)

{

	return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);

}



static __always_inline void arch_spin_unlock(struct arch_spinlock *lock)

{

	PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);

}

}





#endif

#endif

arch/x86/include/asm/paravirt_types.h

+8 −6
Original line number Original line Diff line number Diff line
@@ -327,13 +327,15 @@ struct pv_mmu_ops { 
};

};





struct arch_spinlock;

struct arch_spinlock;

#ifdef CONFIG_SMP

#include <asm/spinlock_types.h>

#else

typedef u16 __ticket_t;

#endif



struct pv_lock_ops {

struct pv_lock_ops {

	int (*spin_is_locked)(struct arch_spinlock *lock);

	struct paravirt_callee_save lock_spinning;

	int (*spin_is_contended)(struct arch_spinlock *lock);

	void (*unlock_kick)(struct arch_spinlock *lock, __ticket_t ticket);

	void (*spin_lock)(struct arch_spinlock *lock);

	void (*spin_lock_flags)(struct arch_spinlock *lock, unsigned long flags);

	int (*spin_trylock)(struct arch_spinlock *lock);

	void (*spin_unlock)(struct arch_spinlock *lock);

};

};





/* This contains all the paravirt structures: we get a convenient

/* This contains all the paravirt structures: we get a convenient

arch/x86/include/asm/spinlock.h

+87 −41
Original line number Original line Diff line number Diff line 
#ifndef _ASM_X86_SPINLOCK_H

#ifndef _ASM_X86_SPINLOCK_H

#define _ASM_X86_SPINLOCK_H

#define _ASM_X86_SPINLOCK_H





#include <linux/jump_label.h>

#include <linux/atomic.h>

#include <linux/atomic.h>

#include <asm/page.h>

#include <asm/page.h>

#include <asm/processor.h>

#include <asm/processor.h>

#include <linux/compiler.h>

#include <linux/compiler.h>

#include <asm/paravirt.h>

#include <asm/paravirt.h>

#include <asm/bitops.h>



/*

/*

 * Your basic SMP spinlocks, allowing only a single CPU anywhere

 * Your basic SMP spinlocks, allowing only a single CPU anywhere

 *

 *
@@ -34,6 +37,31 @@ 
# define UNLOCK_LOCK_PREFIX

# define UNLOCK_LOCK_PREFIX

#endif

#endif





/* How long a lock should spin before we consider blocking */

#define SPIN_THRESHOLD	(1 << 15)



extern struct static_key paravirt_ticketlocks_enabled;

static __always_inline bool static_key_false(struct static_key *key);



#ifdef CONFIG_PARAVIRT_SPINLOCKS



static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)

{

	set_bit(0, (volatile unsigned long *)&lock->tickets.tail);

}



#else  /* !CONFIG_PARAVIRT_SPINLOCKS */

static __always_inline void __ticket_lock_spinning(arch_spinlock_t *lock,

							__ticket_t ticket)

{

}

static inline void __ticket_unlock_kick(arch_spinlock_t *lock,

							__ticket_t ticket)

{

}



#endif /* CONFIG_PARAVIRT_SPINLOCKS */



static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)

static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)

{

{

	return lock.tickets.head == lock.tickets.tail;

	return lock.tickets.head == lock.tickets.tail;
@@ -52,81 +80,101 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) 
 * in the high part, because a wide xadd increment of the low part would carry

 * in the high part, because a wide xadd increment of the low part would carry

 * up and contaminate the high part.

 * up and contaminate the high part.

 */

 */

static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)

static __always_inline void arch_spin_lock(arch_spinlock_t *lock)

{

{

	register struct __raw_tickets inc = { .tail = 1 };

	register struct __raw_tickets inc = { .tail = TICKET_LOCK_INC };





	inc = xadd(&lock->tickets, inc);

	inc = xadd(&lock->tickets, inc);

	if (likely(inc.head == inc.tail))

		goto out;





	inc.tail &= ~TICKET_SLOWPATH_FLAG;

	for (;;) {

	for (;;) {

		if (inc.head == inc.tail)

		unsigned count = SPIN_THRESHOLD;

			break;



		do {

			if (ACCESS_ONCE(lock->tickets.head) == inc.tail)

				goto out;

			cpu_relax();

			cpu_relax();

		inc.head = ACCESS_ONCE(lock->tickets.head);

		} while (--count);

		__ticket_lock_spinning(lock, inc.tail);

	}

	}

	barrier();		/* make sure nothing creeps before the lock is taken */

out:	barrier();	/* make sure nothing creeps before the lock is taken */

}

}





static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)

static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)

{

{

	arch_spinlock_t old, new;

	arch_spinlock_t old, new;





	old.tickets = ACCESS_ONCE(lock->tickets);

	old.tickets = ACCESS_ONCE(lock->tickets);

	if (old.tickets.head != old.tickets.tail)

	if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG))

		return 0;

		return 0;





	new.head_tail = old.head_tail + (1 << TICKET_SHIFT);

	new.head_tail = old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT);





	/* cmpxchg is a full barrier, so nothing can move before it */

	/* cmpxchg is a full barrier, so nothing can move before it */

	return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;

	return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;

}

}





static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)

static inline void __ticket_unlock_slowpath(arch_spinlock_t *lock,

					    arch_spinlock_t old)

{

{

	__add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);

	arch_spinlock_t new;

}





static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)

	BUILD_BUG_ON(((__ticket_t)NR_CPUS) != NR_CPUS);

{

	struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);





	return tmp.tail != tmp.head;

	/* Perform the unlock on the "before" copy */

	old.tickets.head += TICKET_LOCK_INC;



	/* Clear the slowpath flag */

	new.head_tail = old.head_tail & ~(TICKET_SLOWPATH_FLAG << TICKET_SHIFT);



	/*

	 * If the lock is uncontended, clear the flag - use cmpxchg in

	 * case it changes behind our back though.

	 */

	if (new.tickets.head != new.tickets.tail ||

	    cmpxchg(&lock->head_tail, old.head_tail,

					new.head_tail) != old.head_tail) {

		/*

		 * Lock still has someone queued for it, so wake up an

		 * appropriate waiter.

		 */

		__ticket_unlock_kick(lock, old.tickets.head);

	}

}

}





static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)

static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)

{

{

	struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);

	if (TICKET_SLOWPATH_FLAG &&

	    static_key_false(&paravirt_ticketlocks_enabled)) {

		arch_spinlock_t prev;





	return (__ticket_t)(tmp.tail - tmp.head) > 1;

		prev = *lock;

}

		add_smp(&lock->tickets.head, TICKET_LOCK_INC);





#ifndef CONFIG_PARAVIRT_SPINLOCKS

		/* add_smp() is a full mb() */





static inline int arch_spin_is_locked(arch_spinlock_t *lock)

		if (unlikely(lock->tickets.tail & TICKET_SLOWPATH_FLAG))

{

			__ticket_unlock_slowpath(lock, prev);

	return __ticket_spin_is_locked(lock);

	} else

		__add(&lock->tickets.head, TICKET_LOCK_INC, UNLOCK_LOCK_PREFIX);

}

}





static inline int arch_spin_is_contended(arch_spinlock_t *lock)

static inline int arch_spin_is_locked(arch_spinlock_t *lock)

{

{

	return __ticket_spin_is_contended(lock);

	struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);

}

#define arch_spin_is_contended	arch_spin_is_contended





static __always_inline void arch_spin_lock(arch_spinlock_t *lock)

	return tmp.tail != tmp.head;

{

	__ticket_spin_lock(lock);

}

}





static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)

static inline int arch_spin_is_contended(arch_spinlock_t *lock)

{

{

	return __ticket_spin_trylock(lock);

	struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);

}





static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)

	return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;

{

	__ticket_spin_unlock(lock);

}

}

#define arch_spin_is_contended	arch_spin_is_contended





static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,

static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,

						  unsigned long flags)

						  unsigned long flags)
@@ -134,8 +182,6 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock, 
	arch_spin_lock(lock);

	arch_spin_lock(lock);

}

}





#endif	/* CONFIG_PARAVIRT_SPINLOCKS */



static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)

static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)

{

{

	while (arch_spin_is_locked(lock))

	while (arch_spin_is_locked(lock))