Merge branch 'sched-hrtimers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

	unsigned int check:1;       /* see lock_acquire() comment */

	unsigned int hardirqs_off:1;

	unsigned int references:12;					/* 32 bits */

	unsigned int pin_count;

};

/*

extern void lockdep_clear_current_reclaim_state(void);

extern void lockdep_trace_alloc(gfp_t mask);

extern void lock_pin_lock(struct lockdep_map *lock);

extern void lock_unpin_lock(struct lockdep_map *lock);

# define INIT_LOCKDEP				.lockdep_recursion = 0, .lockdep_reclaim_gfp = 0,

#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)

#define lockdep_recursing(tsk)	((tsk)->lockdep_recursion)

#define lockdep_pin_lock(l)		lock_pin_lock(&(l)->dep_map)

#define lockdep_unpin_lock(l)	lock_unpin_lock(&(l)->dep_map)

#else /* !CONFIG_LOCKDEP */

static inline void lockdep_off(void)

#define lockdep_recursing(tsk)			(0)

#define lockdep_pin_lock(l)				do { (void)(l); } while (0)

#define lockdep_unpin_lock(l)			do { (void)(l); } while (0)

#endif /* !LOCKDEP */

#ifdef CONFIG_LOCK_STAT

	hlock->waittime_stamp = 0;

	hlock->holdtime_stamp = lockstat_clock();

#endif

	hlock->pin_count = 0;

	if (check && !mark_irqflags(curr, hlock))

		return 0;

	return 0;

}

/*

 * Common debugging checks for both nested and non-nested unlock:

 */

static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,

			unsigned long ip)

{

	if (unlikely(!debug_locks))

		return 0;

	/*

	 * Lockdep should run with IRQs disabled, recursion, head-ache, etc..

	 */

	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))

		return 0;

	if (curr->lockdep_depth <= 0)

		return print_unlock_imbalance_bug(curr, lock, ip);

	return 1;

}

static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)

{

	if (hlock->instance == lock)

}

/*

 * Remove the lock to the list of currently held locks in a

 * potentially non-nested (out of order) manner. This is a

 * relatively rare operation, as all the unlock APIs default

 * to nested mode (which uses lock_release()):

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()).

 *

 * @nested is an hysterical artifact, needs a tree wide cleanup.

 */

static int

lock_release_non_nested(struct task_struct *curr,

			struct lockdep_map *lock, unsigned long ip)

__lock_release(struct lockdep_map *lock, int nested, unsigned long ip)

{

	struct task_struct *curr = current;

	struct held_lock *hlock, *prev_hlock;

	unsigned int depth;

	int i;

	/*

	 * Check whether the lock exists in the current stack

	 * of held locks:

	 */

	if (unlikely(!debug_locks))

		return 0;

	depth = curr->lockdep_depth;

	/*

	 * So we're all set to release this lock.. wait what lock? We don't

	 * own any locks, you've been drinking again?

	 */

	if (DEBUG_LOCKS_WARN_ON(!depth))

		return 0;

	if (DEBUG_LOCKS_WARN_ON(depth <= 0))

		 return print_unlock_imbalance_bug(curr, lock, ip);

	/*

	 * Check whether the lock exists in the current stack

	 * of held locks:

	 */

	prev_hlock = NULL;

	for (i = depth-1; i >= 0; i--) {

		hlock = curr->held_locks + i;

	if (hlock->instance == lock)

		lock_release_holdtime(hlock);

	WARN(hlock->pin_count, "releasing a pinned lock\n");

	if (hlock->references) {

		hlock->references--;

		if (hlock->references) {

	 */

	if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))

		return 0;

	return 1;

}

/*

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()). This is done for unlocks that nest

 * perfectly. (i.e. the current top of the lock-stack is unlocked)

 */

static int lock_release_nested(struct task_struct *curr,

			       struct lockdep_map *lock, unsigned long ip)

static int __lock_is_held(struct lockdep_map *lock)

{

	struct held_lock *hlock;

	unsigned int depth;

	struct task_struct *curr = current;

	int i;

	/*

	 * Pop off the top of the lock stack:

	 */

	depth = curr->lockdep_depth - 1;

	hlock = curr->held_locks + depth;

	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;

	/*

	 * Is the unlock non-nested:

	 */

	if (hlock->instance != lock || hlock->references)

		return lock_release_non_nested(curr, lock, ip);

	curr->lockdep_depth--;

		if (match_held_lock(hlock, lock))

			return 1;

	}

	/*

	 * No more locks, but somehow we've got hash left over, who left it?

	 */

	if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))

	return 0;

	curr->curr_chain_key = hlock->prev_chain_key;

	lock_release_holdtime(hlock);

#ifdef CONFIG_DEBUG_LOCKDEP

	hlock->prev_chain_key = 0;

	hlock->class_idx = 0;

	hlock->acquire_ip = 0;

	hlock->irq_context = 0;

#endif

	return 1;

}

/*

 * Remove the lock to the list of currently held locks - this gets

 * called on mutex_unlock()/spin_unlock*() (or on a failed

 * mutex_lock_interruptible()). This is done for unlocks that nest

 * perfectly. (i.e. the current top of the lock-stack is unlocked)

 */

static void

__lock_release(struct lockdep_map *lock, int nested, unsigned long ip)

static void __lock_pin_lock(struct lockdep_map *lock)

{

	struct task_struct *curr = current;

	int i;

	if (!check_unlock(curr, lock, ip))

	if (unlikely(!debug_locks))

		return;

	if (nested) {

		if (!lock_release_nested(curr, lock, ip))

			return;

	} else {

		if (!lock_release_non_nested(curr, lock, ip))

	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;

		if (match_held_lock(hlock, lock)) {

			hlock->pin_count++;

			return;

		}

	}

	check_chain_key(curr);

	WARN(1, "pinning an unheld lock\n");

}

static int __lock_is_held(struct lockdep_map *lock)

static void __lock_unpin_lock(struct lockdep_map *lock)

{

	struct task_struct *curr = current;

	int i;

	if (unlikely(!debug_locks))

		return;

	for (i = 0; i < curr->lockdep_depth; i++) {

		struct held_lock *hlock = curr->held_locks + i;

		if (match_held_lock(hlock, lock))

			return 1;

		if (match_held_lock(hlock, lock)) {

			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))

				return;

			hlock->pin_count--;

			return;

		}

	}

	return 0;

	WARN(1, "unpinning an unheld lock\n");

}

/*

	check_flags(flags);

	current->lockdep_recursion = 1;

	trace_lock_release(lock, ip);

	__lock_release(lock, nested, ip);

	if (__lock_release(lock, nested, ip))

		check_chain_key(current);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}

}

EXPORT_SYMBOL_GPL(lock_is_held);

void lock_pin_lock(struct lockdep_map *lock)

{

	unsigned long flags;

	if (unlikely(current->lockdep_recursion))

		return;

	raw_local_irq_save(flags);

	check_flags(flags);

	current->lockdep_recursion = 1;

	__lock_pin_lock(lock);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(lock_pin_lock);

void lock_unpin_lock(struct lockdep_map *lock)

{

	unsigned long flags;

	if (unlikely(current->lockdep_recursion))

		return;

	raw_local_irq_save(flags);

	check_flags(flags);

	current->lockdep_recursion = 1;

	__lock_unpin_lock(lock);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(lock_unpin_lock);

void lockdep_set_current_reclaim_state(gfp_t gfp_mask)

{

	current->lockdep_reclaim_gfp = gfp_mask;

	return dl_task(prev);

}

static inline void set_post_schedule(struct rq *rq)

static DEFINE_PER_CPU(struct callback_head, dl_push_head);

static DEFINE_PER_CPU(struct callback_head, dl_pull_head);

static void push_dl_tasks(struct rq *);

static void pull_dl_task(struct rq *);

static inline void queue_push_tasks(struct rq *rq)

{

	rq->post_schedule = has_pushable_dl_tasks(rq);

	if (!has_pushable_dl_tasks(rq))

		return;

	queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks);

}

static inline void queue_pull_task(struct rq *rq)

{

	queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task);

}

static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq);

static void dl_task_offline_migration(struct rq *rq, struct task_struct *p)

static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p)

{

	struct rq *later_rq = NULL;

	bool fallback = false;

		double_lock_balance(rq, later_rq);

	}

	/*

	 * By now the task is replenished and enqueued; migrate it.

	 */

	deactivate_task(rq, p, 0);

	set_task_cpu(p, later_rq->cpu);

	activate_task(later_rq, p, ENQUEUE_REPLENISH);

	activate_task(later_rq, p, 0);

	if (!fallback)

		resched_curr(later_rq);

	double_unlock_balance(rq, later_rq);

	double_unlock_balance(later_rq, rq);

	return later_rq;

}

#else

	return false;

}

static inline int pull_dl_task(struct rq *rq)

static inline void pull_dl_task(struct rq *rq)

{

}

static inline void queue_push_tasks(struct rq *rq)

{

	return 0;

}

static inline void set_post_schedule(struct rq *rq)

static inline void queue_pull_task(struct rq *rq)

{

}

#endif /* CONFIG_SMP */

 * actually started or not (i.e., the replenishment instant is in

 * the future or in the past).

 */

static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted)

static int start_dl_timer(struct task_struct *p)

{

	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);

	struct rq *rq = rq_of_dl_rq(dl_rq);

	struct sched_dl_entity *dl_se = &p->dl;

	struct hrtimer *timer = &dl_se->dl_timer;

	struct rq *rq = task_rq(p);

	ktime_t now, act;

	s64 delta;

	if (boosted)

		return 0;

	lockdep_assert_held(&rq->lock);

	/*

	 * We want the timer to fire at the deadline, but considering

	 * that it is actually coming from rq->clock and not from

	 * hrtimer's time base reading.

	 */

	act = ns_to_ktime(dl_se->deadline);

	now = hrtimer_cb_get_time(&dl_se->dl_timer);

	now = hrtimer_cb_get_time(timer);

	delta = ktime_to_ns(now) - rq_clock(rq);

	act = ktime_add_ns(act, delta);

	if (ktime_us_delta(act, now) < 0)

		return 0;

	hrtimer_start(&dl_se->dl_timer, act, HRTIMER_MODE_ABS);

	/*

	 * !enqueued will guarantee another callback; even if one is already in

	 * progress. This ensures a balanced {get,put}_task_struct().

	 *

	 * The race against __run_timer() clearing the enqueued state is

	 * harmless because we're holding task_rq()->lock, therefore the timer

	 * expiring after we've done the check will wait on its task_rq_lock()

	 * and observe our state.

	 */

	if (!hrtimer_is_queued(timer)) {

		get_task_struct(p);

		hrtimer_start(timer, act, HRTIMER_MODE_ABS);

	}

	return 1;

}

	rq = task_rq_lock(p, &flags);

	/*

	 * We need to take care of several possible races here:

	 *

	 *   - the task might have changed its scheduling policy

	 *     to something different than SCHED_DEADLINE

	 *   - the task might have changed its reservation parameters

	 *     (through sched_setattr())

	 *   - the task might have been boosted by someone else and

	 *     might be in the boosting/deboosting path

	 * The task might have changed its scheduling policy to something

	 * different than SCHED_DEADLINE (through switched_fromd_dl()).

	 */

	if (!dl_task(p)) {

		__dl_clear_params(p);

		goto unlock;

	}

	/*

	 * This is possible if switched_from_dl() raced against a running

	 * callback that took the above !dl_task() path and we've since then

	 * switched back into SCHED_DEADLINE.

	 *

	 * In all this cases we bail out, as the task is already

	 * in the runqueue or is going to be enqueued back anyway.

	 * There's nothing to do except drop our task reference.

	 */

	if (!dl_task(p) || dl_se->dl_new ||

	    dl_se->dl_boosted || !dl_se->dl_throttled)

	if (dl_se->dl_new)

		goto unlock;

	sched_clock_tick();

	update_rq_clock(rq);

	/*

	 * The task might have been boosted by someone else and might be in the

	 * boosting/deboosting path, its not throttled.

	 */

	if (dl_se->dl_boosted)

		goto unlock;

#ifdef CONFIG_SMP

	/*

	 * If we find that the rq the task was on is no longer

	 * available, we need to select a new rq.

	 * Spurious timer due to start_dl_timer() race; or we already received

	 * a replenishment from rt_mutex_setprio().

	 */

	if (unlikely(!rq->online)) {

		dl_task_offline_migration(rq, p);

	if (!dl_se->dl_throttled)

		goto unlock;

	}

#endif

	sched_clock_tick();

	update_rq_clock(rq);

	/*

	 * If the throttle happened during sched-out; like:

		check_preempt_curr_dl(rq, p, 0);

	else

		resched_curr(rq);

#ifdef CONFIG_SMP

	/*

	 * Queueing this task back might have overloaded rq,

	 * check if we need to kick someone away.

	 * Perform balancing operations here; after the replenishments.  We

	 * cannot drop rq->lock before this, otherwise the assertion in

	 * start_dl_timer() about not missing updates is not true.

	 *

	 * If we find that the rq the task was on is no longer available, we

	 * need to select a new rq.

	 *

	 * XXX figure out if select_task_rq_dl() deals with offline cpus.

	 */

	if (unlikely(!rq->online))

		rq = dl_task_offline_migration(rq, p);

	/*

	 * Queueing this task back might have overloaded rq, check if we need

	 * to kick someone away.

	 */

	if (has_pushable_dl_tasks(rq))

		push_dl_task(rq);

#endif

unlock:

	task_rq_unlock(rq, p, &flags);

	/*

	 * This can free the task_struct, including this hrtimer, do not touch

	 * anything related to that after this.

	 */

	put_task_struct(p);

	return HRTIMER_NORESTART;

}

	if (dl_runtime_exceeded(dl_se)) {

		dl_se->dl_throttled = 1;

		__dequeue_task_dl(rq, curr, 0);

		if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted)))

		if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr)))

			enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH);

		if (!is_leftmost(curr, &rq->dl))

	resched_curr(rq);

}

static int pull_dl_task(struct rq *this_rq);

#endif /* CONFIG_SMP */

/*

	dl_rq = &rq->dl;

	if (need_pull_dl_task(rq, prev)) {

		/*

		 * This is OK, because current is on_cpu, which avoids it being

		 * picked for load-balance and preemption/IRQs are still

		 * disabled avoiding further scheduler activity on it and we're

		 * being very careful to re-start the picking loop.

		 */

		lockdep_unpin_lock(&rq->lock);

		pull_dl_task(rq);

		lockdep_pin_lock(&rq->lock);

		/*

		 * pull_rt_task() can drop (and re-acquire) rq->lock; this

		 * means a stop task can slip in, in which case we need to

	if (hrtick_enabled(rq))

		start_hrtick_dl(rq, p);

	set_post_schedule(rq);

	queue_push_tasks(rq);

	return p;

}

static void task_dead_dl(struct task_struct *p)

{

	struct hrtimer *timer = &p->dl.dl_timer;

	struct dl_bw *dl_b = dl_bw_of(task_cpu(p));

	/*

	/* XXX we should retain the bw until 0-lag */

	dl_b->total_bw -= p->dl.dl_bw;

	raw_spin_unlock_irq(&dl_b->lock);

	hrtimer_cancel(timer);

}

static void set_curr_task_dl(struct rq *rq)

		;

}

static int pull_dl_task(struct rq *this_rq)

static void pull_dl_task(struct rq *this_rq)

{

	int this_cpu = this_rq->cpu, ret = 0, cpu;

	int this_cpu = this_rq->cpu, cpu;

	struct task_struct *p;

	bool resched = false;

	struct rq *src_rq;

	u64 dmin = LONG_MAX;

	if (likely(!dl_overloaded(this_rq)))

		return 0;

		return;

	/*

	 * Match the barrier from dl_set_overloaded; this guarantees that if we

					   src_rq->curr->dl.deadline))

				goto skip;

			ret = 1;

			resched = true;

			deactivate_task(src_rq, p, 0);

			set_task_cpu(p, this_cpu);

		double_unlock_balance(this_rq, src_rq);

	}

	return ret;

}

static void post_schedule_dl(struct rq *rq)

{

	push_dl_tasks(rq);

	if (resched)

		resched_curr(this_rq);

}

/*

#endif /* CONFIG_SMP */

/*

 *  Ensure p's dl_timer is cancelled. May drop rq->lock for a while.

 */

static void cancel_dl_timer(struct rq *rq, struct task_struct *p)

static void switched_from_dl(struct rq *rq, struct task_struct *p)

{

	struct hrtimer *dl_timer = &p->dl.dl_timer;

	/* Nobody will change task's class if pi_lock is held */

	lockdep_assert_held(&p->pi_lock);

	if (hrtimer_active(dl_timer)) {

		int ret = hrtimer_try_to_cancel(dl_timer);

		if (unlikely(ret == -1)) {

	/*

			 * Note, p may migrate OR new deadline tasks

			 * may appear in rq when we are unlocking it.

			 * A caller of us must be fine with that.

	 * Start the deadline timer; if we switch back to dl before this we'll

	 * continue consuming our current CBS slice. If we stay outside of

	 * SCHED_DEADLINE until the deadline passes, the timer will reset the

	 * task.

	 */

			raw_spin_unlock(&rq->lock);

			hrtimer_cancel(dl_timer);

			raw_spin_lock(&rq->lock);

		}

	}

}