Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

/*

 * Prevent the compiler from merging or refetching reads or writes. The

 * compiler is also forbidden from reordering successive instances of

 * READ_ONCE, WRITE_ONCE and ACCESS_ONCE (see below), but only when the

 * compiler is aware of some particular ordering.  One way to make the

 * compiler aware of ordering is to put the two invocations of READ_ONCE,

 * WRITE_ONCE or ACCESS_ONCE() in different C statements.

 * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some

 * particular ordering. One way to make the compiler aware of ordering is to

 * put the two invocations of READ_ONCE or WRITE_ONCE in different C

 * statements.

 *

 * In contrast to ACCESS_ONCE these two macros will also work on aggregate

 * data types like structs or unions. If the size of the accessed data

 * type exceeds the word size of the machine (e.g., 32 bits or 64 bits)

 * READ_ONCE() and WRITE_ONCE() will fall back to memcpy(). There's at

 * least two memcpy()s: one for the __builtin_memcpy() and then one for

 * the macro doing the copy of variable - '__u' allocated on the stack.

 * These two macros will also work on aggregate data types like structs or

 * unions. If the size of the accessed data type exceeds the word size of

 * the machine (e.g., 32 bits or 64 bits) READ_ONCE() and WRITE_ONCE() will

 * fall back to memcpy(). There's at least two memcpy()s: one for the

 * __builtin_memcpy() and then one for the macro doing the copy of variable

 * - '__u' allocated on the stack.

 *

 * Their two major use cases are: (1) Mediating communication between

 * process-level code and irq/NMI handlers, all running on the same CPU,

	compiletime_assert(__native_word(t),				\

		"Need native word sized stores/loads for atomicity.")

/*

 * Prevent the compiler from merging or refetching accesses.  The compiler

 * is also forbidden from reordering successive instances of ACCESS_ONCE(),

 * but only when the compiler is aware of some particular ordering.  One way

 * to make the compiler aware of ordering is to put the two invocations of

 * ACCESS_ONCE() in different C statements.

 *

 * ACCESS_ONCE will only work on scalar types. For union types, ACCESS_ONCE

 * on a union member will work as long as the size of the member matches the

 * size of the union and the size is smaller than word size.

 *

 * The major use cases of ACCESS_ONCE used to be (1) Mediating communication

 * between process-level code and irq/NMI handlers, all running on the same CPU,

 * and (2) Ensuring that the compiler does not  fold, spindle, or otherwise

 * mutilate accesses that either do not require ordering or that interact

 * with an explicit memory barrier or atomic instruction that provides the

 * required ordering.

 *

 * If possible use READ_ONCE()/WRITE_ONCE() instead.

 */

#define __ACCESS_ONCE(x) ({ \

	 __maybe_unused typeof(x) __var = (__force typeof(x)) 0; \

	(volatile typeof(x) *)&(x); })

#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x))

#endif /* __LINUX_COMPILER_H */

 */

#include <linux/wait.h>

#ifdef CONFIG_LOCKDEP_COMPLETIONS

#include <linux/lockdep.h>

#endif

/*

 * struct completion - structure used to maintain state for a "completion"

struct completion {

	unsigned int done;

	wait_queue_head_t wait;

#ifdef CONFIG_LOCKDEP_COMPLETIONS

	struct lockdep_map_cross map;

#endif

};

#ifdef CONFIG_LOCKDEP_COMPLETIONS

static inline void complete_acquire(struct completion *x)

{

	lock_acquire_exclusive((struct lockdep_map *)&x->map, 0, 0, NULL, _RET_IP_);

}

static inline void complete_release(struct completion *x)

{

	lock_release((struct lockdep_map *)&x->map, 0, _RET_IP_);

}

static inline void complete_release_commit(struct completion *x)

{

	lock_commit_crosslock((struct lockdep_map *)&x->map);

}

#define init_completion_map(x, m)					\

do {									\

	lockdep_init_map_crosslock((struct lockdep_map *)&(x)->map,	\

			(m)->name, (m)->key, 0);				\

	__init_completion(x);						\

} while (0)

#define init_completion(x)						\

do {									\

	static struct lock_class_key __key;				\

	lockdep_init_map_crosslock((struct lockdep_map *)&(x)->map,	\

			"(completion)" #x,				\

			&__key, 0);					\

	__init_completion(x);						\

} while (0)

#else

#define init_completion_map(x, m) __init_completion(x)

#define init_completion(x) __init_completion(x)

static inline void complete_acquire(struct completion *x) {}

static inline void complete_release(struct completion *x) {}

static inline void complete_release_commit(struct completion *x) {}

#endif

#ifdef CONFIG_LOCKDEP_COMPLETIONS

#define COMPLETION_INITIALIZER(work) \

	{ 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait), \

	STATIC_CROSS_LOCKDEP_MAP_INIT("(completion)" #work, &(work)) }

#else

#define COMPLETION_INITIALIZER(work) \

	{ 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) }

#endif

#define COMPLETION_INITIALIZER_ONSTACK_MAP(work, map) \

	(*({ init_completion_map(&(work), &(map)); &(work); }))

	int				cpu;

	unsigned long			ip;

#endif

#ifdef CONFIG_LOCKDEP_CROSSRELEASE

	/*

	 * Whether it's a crosslock.

	 */

	int				cross;

#endif

};

static inline void lockdep_copy_map(struct lockdep_map *to,

	unsigned int hardirqs_off:1;

	unsigned int references:12;					/* 32 bits */

	unsigned int pin_count;

#ifdef CONFIG_LOCKDEP_CROSSRELEASE

	/*

	 * Generation id.

	 *

	 * A value of cross_gen_id will be stored when holding this,

	 * which is globally increased whenever each crosslock is held.

	 */

	unsigned int gen_id;

#endif

};

#ifdef CONFIG_LOCKDEP_CROSSRELEASE

#define MAX_XHLOCK_TRACE_ENTRIES 5

/*

 * This is for keeping locks waiting for commit so that true dependencies

 * can be added at commit step.

 */

struct hist_lock {

	/*

	 * Id for each entry in the ring buffer. This is used to

	 * decide whether the ring buffer was overwritten or not.

	 *

	 * For example,

	 *

	 *           |<----------- hist_lock ring buffer size ------->|

	 *           pppppppppppppppppppppiiiiiiiiiiiiiiiiiiiiiiiiiiiii

	 * wrapped > iiiiiiiiiiiiiiiiiiiiiiiiiii.......................

	 *

	 *           where 'p' represents an acquisition in process

	 *           context, 'i' represents an acquisition in irq

	 *           context.

	 *

	 * In this example, the ring buffer was overwritten by

	 * acquisitions in irq context, that should be detected on

	 * rollback or commit.

	 */

	unsigned int hist_id;

	/*

	 * Seperate stack_trace data. This will be used at commit step.

	 */

	struct stack_trace	trace;

	unsigned long		trace_entries[MAX_XHLOCK_TRACE_ENTRIES];

	/*

	 * Seperate hlock instance. This will be used at commit step.

	 *

	 * TODO: Use a smaller data structure containing only necessary

	 * data. However, we should make lockdep code able to handle the

	 * smaller one first.

	 */

	struct held_lock	hlock;

};

/*

 * To initialize a lock as crosslock, lockdep_init_map_crosslock() should

 * be called instead of lockdep_init_map().

 */

struct cross_lock {

	/*

	 * When more than one acquisition of crosslocks are overlapped,

	 * we have to perform commit for them based on cross_gen_id of

	 * the first acquisition, which allows us to add more true

	 * dependencies.

	 *

	 * Moreover, when no acquisition of a crosslock is in progress,

	 * we should not perform commit because the lock might not exist

	 * any more, which might cause incorrect memory access. So we

	 * have to track the number of acquisitions of a crosslock.

	 */

	int nr_acquire;

	/*

	 * Seperate hlock instance. This will be used at commit step.

	 *

	 * TODO: Use a smaller data structure containing only necessary

	 * data. However, we should make lockdep code able to handle the

	 * smaller one first.

	 */

	struct held_lock	hlock;

};

struct lockdep_map_cross {

	struct lockdep_map map;

	struct cross_lock xlock;

};

#endif

/*

 * Initialization, self-test and debugging-output methods:

 */

	XHLOCK_CTX_NR,

};

#ifdef CONFIG_LOCKDEP_CROSSRELEASE

extern void lockdep_init_map_crosslock(struct lockdep_map *lock,

				       const char *name,

				       struct lock_class_key *key,

				       int subclass);

extern void lock_commit_crosslock(struct lockdep_map *lock);

/*

 * What we essencially have to initialize is 'nr_acquire'. Other members

 * will be initialized in add_xlock().

 */

#define STATIC_CROSS_LOCK_INIT() \

	{ .nr_acquire = 0,}

#define STATIC_CROSS_LOCKDEP_MAP_INIT(_name, _key) \

	{ .map.name = (_name), .map.key = (void *)(_key), \

	  .map.cross = 1, .xlock = STATIC_CROSS_LOCK_INIT(), }

/*

 * To initialize a lockdep_map statically use this macro.

 * Note that _name must not be NULL.

 */

#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \

	{ .name = (_name), .key = (void *)(_key), .cross = 0, }

extern void crossrelease_hist_start(enum xhlock_context_t c);

extern void crossrelease_hist_end(enum xhlock_context_t c);

extern void lockdep_invariant_state(bool force);

extern void lockdep_init_task(struct task_struct *task);

extern void lockdep_free_task(struct task_struct *task);

#else /* !CROSSRELEASE */

#define lockdep_init_map_crosslock(m, n, k, s) do {} while (0)

/*

 * To initialize a lockdep_map statically use this macro.

static inline void lockdep_invariant_state(bool force) {}

static inline void lockdep_init_task(struct task_struct *task) {}

static inline void lockdep_free_task(struct task_struct *task) {}

#endif /* CROSSRELEASE */

#ifdef CONFIG_LOCK_STAT

 */

typedef struct {

	arch_rwlock_t raw_lock;

#ifdef CONFIG_GENERIC_LOCKBREAK

	unsigned int break_lock;

#endif

#ifdef CONFIG_DEBUG_SPINLOCK

	unsigned int magic, owner_cpu;

	void *owner;