sched/core: Introduce 'struct rq_flags'

/*

 * __task_rq_lock - lock the rq @p resides on.

 */

struct rq *__task_rq_lock(struct task_struct *p)

struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)

	__acquires(rq->lock)

{

	struct rq *rq;

/*

 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.

 */

struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)

struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)

	__acquires(p->pi_lock)

	__acquires(rq->lock)

{

	struct rq *rq;

	for (;;) {

		raw_spin_lock_irqsave(&p->pi_lock, *flags);

		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);

		rq = task_rq(p);

		raw_spin_lock(&rq->lock);

		/*

			return rq;

		}

		raw_spin_unlock(&rq->lock);

		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);

		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);

		while (unlikely(task_on_rq_migrating(p)))

			cpu_relax();

static int __set_cpus_allowed_ptr(struct task_struct *p,

				  const struct cpumask *new_mask, bool check)

{

	unsigned long flags;

	struct rq *rq;

	unsigned int dest_cpu;

	struct rq_flags rf;

	struct rq *rq;

	int ret = 0;

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	/*

	 * Must re-check here, to close a race against __kthread_bind(),

	if (task_running(rq, p) || p->state == TASK_WAKING) {

		struct migration_arg arg = { p, dest_cpu };

		/* Need help from migration thread: drop lock and wait. */

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);

		tlb_migrate_finish(p->mm);

		return 0;

		lockdep_pin_lock(&rq->lock);

	}

out:

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

	return ret;

}

 */

unsigned long wait_task_inactive(struct task_struct *p, long match_state)

{

	unsigned long flags;

	int running, queued;

	struct rq_flags rf;

	unsigned long ncsw;

	struct rq *rq;

		 * lock now, to be *sure*. If we're wrong, we'll

		 * just go back and repeat.

		 */

		rq = task_rq_lock(p, &flags);

		rq = task_rq_lock(p, &rf);

		trace_sched_wait_task(p);

		running = task_running(rq, p);

		queued = task_on_rq_queued(p);

		ncsw = 0;

		if (!match_state || p->state == match_state)

			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		/*

		 * If it changed from the expected state, bail out now.

 */

static int ttwu_remote(struct task_struct *p, int wake_flags)

{

	struct rq_flags rf;

	struct rq *rq;

	int ret = 0;

	rq = __task_rq_lock(p);

	rq = __task_rq_lock(p, &rf);

	if (task_on_rq_queued(p)) {

		/* check_preempt_curr() may use rq clock */

		update_rq_clock(rq);

		ttwu_do_wakeup(rq, p, wake_flags);

		ret = 1;

	}

	__task_rq_unlock(rq);

	__task_rq_unlock(rq, &rf);

	return ret;

}

 */

void wake_up_new_task(struct task_struct *p)

{

	unsigned long flags;

	struct rq_flags rf;

	struct rq *rq;

	raw_spin_lock_irqsave(&p->pi_lock, flags);

	/* Initialize new task's runnable average */

	init_entity_runnable_average(&p->se);

	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);

#ifdef CONFIG_SMP

	/*

	 * Fork balancing, do it here and not earlier because:

	/* Post initialize new task's util average when its cfs_rq is set */

	post_init_entity_util_avg(&p->se);

	rq = __task_rq_lock(p);

	rq = __task_rq_lock(p, &rf);

	activate_task(rq, p, 0);

	p->on_rq = TASK_ON_RQ_QUEUED;

	trace_sched_wakeup_new(p);

		lockdep_pin_lock(&rq->lock);

	}

#endif

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

}

#ifdef CONFIG_PREEMPT_NOTIFIERS

 */

unsigned long long task_sched_runtime(struct task_struct *p)

{

	unsigned long flags;

	struct rq_flags rf;

	struct rq *rq;

	u64 ns;

		return p->se.sum_exec_runtime;

#endif

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	/*

	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would

	 * project cycles that may never be accounted to this

		p->sched_class->update_curr(rq);

	}

	ns = p->se.sum_exec_runtime;

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

	return ns;

}

void rt_mutex_setprio(struct task_struct *p, int prio)

{

	int oldprio, queued, running, queue_flag = DEQUEUE_SAVE | DEQUEUE_MOVE;

	struct rq *rq;

	const struct sched_class *prev_class;

	struct rq_flags rf;

	struct rq *rq;

	BUG_ON(prio > MAX_PRIO);

	rq = __task_rq_lock(p);

	rq = __task_rq_lock(p, &rf);

	/*

	 * Idle task boosting is a nono in general. There is one

	check_class_changed(rq, p, prev_class, oldprio);

out_unlock:

	preempt_disable(); /* avoid rq from going away on us */

	__task_rq_unlock(rq);

	__task_rq_unlock(rq, &rf);

	balance_callback(rq);

	preempt_enable();

void set_user_nice(struct task_struct *p, long nice)

{

	int old_prio, delta, queued;

	unsigned long flags;

	struct rq_flags rf;

	struct rq *rq;

	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)

	 * We have to be careful, if called from sys_setpriority(),

	 * the task might be in the middle of scheduling on another CPU.

	 */

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	/*

	 * The RT priorities are set via sched_setscheduler(), but we still

	 * allow the 'normal' nice value to be set - but as expected

			resched_curr(rq);

	}

out_unlock:

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

}

EXPORT_SYMBOL(set_user_nice);

		      MAX_RT_PRIO - 1 - attr->sched_priority;

	int retval, oldprio, oldpolicy = -1, queued, running;

	int new_effective_prio, policy = attr->sched_policy;

	unsigned long flags;

	const struct sched_class *prev_class;

	struct rq *rq;

	struct rq_flags rf;

	int reset_on_fork;

	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;

	struct rq *rq;

	/* may grab non-irq protected spin_locks */

	BUG_ON(in_interrupt());

	 * To be able to change p->policy safely, the appropriate

	 * runqueue lock must be held.

	 */

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	/*

	 * Changing the policy of the stop threads its a very bad idea

	 */

	if (p == rq->stop) {

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		return -EINVAL;

	}

			goto change;

		p->sched_reset_on_fork = reset_on_fork;

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		return 0;

	}

change:

		if (rt_bandwidth_enabled() && rt_policy(policy) &&

				task_group(p)->rt_bandwidth.rt_runtime == 0 &&

				!task_group_is_autogroup(task_group(p))) {

			task_rq_unlock(rq, p, &flags);

			task_rq_unlock(rq, p, &rf);

			return -EPERM;

		}

#endif

			 */

			if (!cpumask_subset(span, &p->cpus_allowed) ||

			    rq->rd->dl_bw.bw == 0) {

				task_rq_unlock(rq, p, &flags);

				task_rq_unlock(rq, p, &rf);

				return -EPERM;

			}

		}

	/* recheck policy now with rq lock held */

	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {

		policy = oldpolicy = -1;

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		goto recheck;

	}

	 * is available.

	 */

	if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {

		task_rq_unlock(rq, p, &flags);

		task_rq_unlock(rq, p, &rf);

		return -EBUSY;

	}

	check_class_changed(rq, p, prev_class, oldprio);

	preempt_disable(); /* avoid rq from going away on us */

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

	if (pi)

		rt_mutex_adjust_pi(p);

{

	struct task_struct *p;

	unsigned int time_slice;

	unsigned long flags;

	struct rq_flags rf;

	struct timespec t;

	struct rq *rq;

	int retval;

	struct timespec t;

	if (pid < 0)

		return -EINVAL;

	if (retval)

		goto out_unlock;

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	time_slice = 0;

	if (p->sched_class->get_rr_interval)

		time_slice = p->sched_class->get_rr_interval(rq, p);

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

	rcu_read_unlock();

	jiffies_to_timespec(time_slice, &t);

 */

void sched_setnuma(struct task_struct *p, int nid)

{

	struct rq *rq;

	unsigned long flags;

	bool queued, running;

	struct rq_flags rf;

	struct rq *rq;

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	queued = task_on_rq_queued(p);

	running = task_current(rq, p);

		p->sched_class->set_curr_task(rq);

	if (queued)

		enqueue_task(rq, p, ENQUEUE_RESTORE);

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

}

#endif /* CONFIG_NUMA_BALANCING */

{

	struct task_group *tg;

	int queued, running;

	unsigned long flags;

	struct rq_flags rf;

	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

	rq = task_rq_lock(tsk, &rf);

	running = task_current(rq, tsk);

	queued = task_on_rq_queued(tsk);

	if (queued)

		enqueue_task(rq, tsk, ENQUEUE_RESTORE | ENQUEUE_MOVE);

	task_rq_unlock(rq, tsk, &flags);

	task_rq_unlock(rq, tsk, &rf);

}

#endif /* CONFIG_CGROUP_SCHED */

						     struct sched_dl_entity,

						     dl_timer);

	struct task_struct *p = dl_task_of(dl_se);

	unsigned long flags;

	struct rq_flags rf;

	struct rq *rq;

	rq = task_rq_lock(p, &flags);

	rq = task_rq_lock(p, &rf);

	/*

	 * The task might have changed its scheduling policy to something

#endif

unlock:

	task_rq_unlock(rq, p, &flags);

	task_rq_unlock(rq, p, &rf);

	/*

	 * This can free the task_struct, including this hrtimer, do not touch

static inline void sched_avg_update(struct rq *rq) { }

#endif

struct rq *__task_rq_lock(struct task_struct *p)

struct rq_flags {

	unsigned long flags;

};

struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)

	__acquires(rq->lock);

struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)

struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)

	__acquires(p->pi_lock)

	__acquires(rq->lock);

static inline void __task_rq_unlock(struct rq *rq)

static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)

	__releases(rq->lock)

{

	lockdep_unpin_lock(&rq->lock);

}

static inline void

task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)

task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)

	__releases(rq->lock)

	__releases(p->pi_lock)

{

	lockdep_unpin_lock(&rq->lock);

	raw_spin_unlock(&rq->lock);

	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);

	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);

}

#ifdef CONFIG_SMP