Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 98ec21a0 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'sched-hrtimers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler updates from Thomas Gleixner:
 "This series of scheduler updates depends on sched/core and timers/core
  branches, which are already in your tree:

   - Scheduler balancing overhaul to plug a hard to trigger race which
     causes an oops in the balancer (Peter Zijlstra)

   - Lockdep updates which are related to the balancing updates (Peter
     Zijlstra)"

* 'sched-hrtimers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched,lockdep: Employ lock pinning
  lockdep: Implement lock pinning
  lockdep: Simplify lock_release()
  sched: Streamline the task migration locking a little
  sched: Move code around
  sched,dl: Fix sched class hopping CBS hole
  sched, dl: Convert switched_{from, to}_dl() / prio_changed_dl() to balance callbacks
  sched,dl: Remove return value from pull_dl_task()
  sched, rt: Convert switched_{from, to}_rt() / prio_changed_rt() to balance callbacks
  sched,rt: Remove return value from pull_rt_task()
  sched: Allow balance callbacks for check_class_changed()
  sched: Use replace normalize_task() with __sched_setscheduler()
  sched: Replace post_schedule with a balance callback list
parents a2629483 cbce1a68

include/linux/lockdep.h

+10 −0
Original line number Diff line number Diff line
@@ -255,6 +255,7 @@ struct held_lock { 
	unsigned int check:1;       /* see lock_acquire() comment */

	unsigned int hardirqs_off:1;

	unsigned int references:12;					/* 32 bits */

	unsigned int pin_count;

};



/*
@@ -354,6 +355,9 @@ extern void lockdep_set_current_reclaim_state(gfp_t gfp_mask); 
extern void lockdep_clear_current_reclaim_state(void);

extern void lockdep_trace_alloc(gfp_t mask);



extern void lock_pin_lock(struct lockdep_map *lock);

extern void lock_unpin_lock(struct lockdep_map *lock);



# define INIT_LOCKDEP				.lockdep_recursion = 0, .lockdep_reclaim_gfp = 0,



#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)
@@ -368,6 +372,9 @@ extern void lockdep_trace_alloc(gfp_t mask); 


#define lockdep_recursing(tsk)	((tsk)->lockdep_recursion)



#define lockdep_pin_lock(l)		lock_pin_lock(&(l)->dep_map)

#define lockdep_unpin_lock(l)	lock_unpin_lock(&(l)->dep_map)



#else /* !CONFIG_LOCKDEP */



static inline void lockdep_off(void)
@@ -420,6 +427,9 @@ struct lock_class_key { }; 


#define lockdep_recursing(tsk)			(0)



#define lockdep_pin_lock(l)				do { (void)(l); } while (0)

#define lockdep_unpin_lock(l)			do { (void)(l); } while (0)



#endif /* !LOCKDEP */



#ifdef CONFIG_LOCK_STAT

kernel/locking/lockdep.c

+87 −90
Original line number Diff line number Diff line
@@ -3157,6 +3157,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, 
	hlock->waittime_stamp = 0;

	hlock->holdtime_stamp = lockstat_clock();

#endif

	hlock->pin_count = 0;



	if (check && !mark_irqflags(curr, hlock))

		return 0;
@@ -3260,26 +3261,6 @@ print_unlock_imbalance_bug(struct task_struct *curr, struct lockdep_map *lock, 
	return 0;

}



/*

 * Common debugging checks for both nested and non-nested unlock:

 */

static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,

			unsigned long ip)

{

	if (unlikely(!debug_locks))

		return 0;

	/*

	 * Lockdep should run with IRQs disabled, recursion, head-ache, etc..

	 */

	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))

		return 0;



	if (curr->lockdep_depth <= 0)

		return print_unlock_imbalance_bug(curr, lock, ip);



	return 1;

}



static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)

{

	if (hlock->instance == lock)
@@ -3376,31 +3357,35 @@ __lock_set_class(struct lockdep_map *lock, const char *name, 
}



/*

 * Remove the lock to the list of currently held locks in a

 * potentially non-nested (out of order) manner. This is a

 * relatively rare operation, as all the unlock APIs default

 * to nested mode (which uses lock_release()):

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()).

 *

 * @nested is an hysterical artifact, needs a tree wide cleanup.

 */

static int

lock_release_non_nested(struct task_struct *curr,

			struct lockdep_map *lock, unsigned long ip)

__lock_release(struct lockdep_map *lock, int nested, unsigned long ip)

{

	struct task_struct *curr = current;

	struct held_lock *hlock, *prev_hlock;

	unsigned int depth;

	int i;



	/*

	 * Check whether the lock exists in the current stack

	 * of held locks:

	 */

	if (unlikely(!debug_locks))

		return 0;



	depth = curr->lockdep_depth;

	/*

	 * So we're all set to release this lock.. wait what lock? We don't

	 * own any locks, you've been drinking again?

	 */

	if (DEBUG_LOCKS_WARN_ON(!depth))

		return 0;

	if (DEBUG_LOCKS_WARN_ON(depth <= 0))

		 return print_unlock_imbalance_bug(curr, lock, ip);



	/*

	 * Check whether the lock exists in the current stack

	 * of held locks:

	 */

	prev_hlock = NULL;

	for (i = depth-1; i >= 0; i--) {

		hlock = curr->held_locks + i;
@@ -3419,6 +3404,8 @@ lock_release_non_nested(struct task_struct *curr, 
	if (hlock->instance == lock)

		lock_release_holdtime(hlock);



	WARN(hlock->pin_count, "releasing a pinned lock\n");



	if (hlock->references) {

		hlock->references--;

		if (hlock->references) {
@@ -3456,91 +3443,66 @@ lock_release_non_nested(struct task_struct *curr, 
	 */

	if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))

		return 0;



	return 1;

}



/*

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()). This is done for unlocks that nest

 * perfectly. (i.e. the current top of the lock-stack is unlocked)

 */

static int lock_release_nested(struct task_struct *curr,

			       struct lockdep_map *lock, unsigned long ip)

static int __lock_is_held(struct lockdep_map *lock)

{

	struct held_lock *hlock;

	unsigned int depth;

	struct task_struct *curr = current;

	int i;



	/*

	 * Pop off the top of the lock stack:

	 */

	depth = curr->lockdep_depth - 1;

	hlock = curr->held_locks + depth;

	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;



	/*

	 * Is the unlock non-nested:

	 */

	if (hlock->instance != lock || hlock->references)

		return lock_release_non_nested(curr, lock, ip);

	curr->lockdep_depth--;

		if (match_held_lock(hlock, lock))

			return 1;

	}



	/*

	 * No more locks, but somehow we've got hash left over, who left it?

	 */

	if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))

	return 0;



	curr->curr_chain_key = hlock->prev_chain_key;



	lock_release_holdtime(hlock);



#ifdef CONFIG_DEBUG_LOCKDEP

	hlock->prev_chain_key = 0;

	hlock->class_idx = 0;

	hlock->acquire_ip = 0;

	hlock->irq_context = 0;

#endif

	return 1;

}



/*

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()). This is done for unlocks that nest

 * perfectly. (i.e. the current top of the lock-stack is unlocked)

 */

static void

__lock_release(struct lockdep_map *lock, int nested, unsigned long ip)

static void __lock_pin_lock(struct lockdep_map *lock)

{

	struct task_struct *curr = current;

	int i;



	if (!check_unlock(curr, lock, ip))

	if (unlikely(!debug_locks))

		return;



	if (nested) {

		if (!lock_release_nested(curr, lock, ip))

			return;

	} else {

		if (!lock_release_non_nested(curr, lock, ip))

	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;



		if (match_held_lock(hlock, lock)) {

			hlock->pin_count++;

			return;

		}

	}



	check_chain_key(curr);

	WARN(1, "pinning an unheld lock\n");

}



static int __lock_is_held(struct lockdep_map *lock)

static void __lock_unpin_lock(struct lockdep_map *lock)

{

	struct task_struct *curr = current;

	int i;



	if (unlikely(!debug_locks))

		return;



	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;



		if (match_held_lock(hlock, lock))

			return 1;

		if (match_held_lock(hlock, lock)) {

			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))

				return;



			hlock->pin_count--;

			return;

		}

	}



	return 0;

	WARN(1, "unpinning an unheld lock\n");

}



/*
@@ -3639,7 +3601,8 @@ void lock_release(struct lockdep_map *lock, int nested, 
	check_flags(flags);

	current->lockdep_recursion = 1;

	trace_lock_release(lock, ip);

	__lock_release(lock, nested, ip);

	if (__lock_release(lock, nested, ip))

		check_chain_key(current);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}
@@ -3665,6 +3628,40 @@ int lock_is_held(struct lockdep_map *lock) 
}

EXPORT_SYMBOL_GPL(lock_is_held);



void lock_pin_lock(struct lockdep_map *lock)

{

	unsigned long flags;



	if (unlikely(current->lockdep_recursion))

		return;



	raw_local_irq_save(flags);

	check_flags(flags);



	current->lockdep_recursion = 1;

	__lock_pin_lock(lock);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(lock_pin_lock);



void lock_unpin_lock(struct lockdep_map *lock)

{

	unsigned long flags;



	if (unlikely(current->lockdep_recursion))

		return;



	raw_local_irq_save(flags);

	check_flags(flags);



	current->lockdep_recursion = 1;

	__lock_unpin_lock(lock);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(lock_unpin_lock);



void lockdep_set_current_reclaim_state(gfp_t gfp_mask)

{

	current->lockdep_reclaim_gfp = gfp_mask;