sched/core: Add rq->lock wrappers

/* CPUs with isolated domains */

cpumask_var_t cpu_isolated_map;

/*

 * this_rq_lock - lock this runqueue and disable interrupts.

 */

static struct rq *this_rq_lock(void)

	__acquires(rq->lock)

{

	struct rq *rq;

	local_irq_disable();

	rq = this_rq();

	raw_spin_lock(&rq->lock);

	return rq;

}

/*

 * __task_rq_lock - lock the rq @p resides on.

 */

static enum hrtimer_restart hrtick(struct hrtimer *timer)

{

	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	struct rq_flags rf;

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

	raw_spin_lock(&rq->lock);

	rq_lock(rq, &rf);

	update_rq_clock(rq);

	rq->curr->sched_class->task_tick(rq, rq->curr, 1);

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

	return HRTIMER_NORESTART;

}

static void __hrtick_start(void *arg)

{

	struct rq *rq = arg;

	struct rq_flags rf;

	raw_spin_lock(&rq->lock);

	rq_lock(rq, &rf);

	__hrtick_restart(rq);

	rq->hrtick_csd_pending = 0;

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

}

/*

 *

 * Returns (locked) new rq. Old rq's lock is released.

 */

static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new_cpu)

static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,

				   struct task_struct *p, int new_cpu)

{

	lockdep_assert_held(&rq->lock);

	p->on_rq = TASK_ON_RQ_MIGRATING;

	dequeue_task(rq, p, 0);

	set_task_cpu(p, new_cpu);

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, rf);

	rq = cpu_rq(new_cpu);

	raw_spin_lock(&rq->lock);

	rq_lock(rq, rf);

	BUG_ON(task_cpu(p) != new_cpu);

	enqueue_task(rq, p, 0);

	p->on_rq = TASK_ON_RQ_QUEUED;

 * So we race with normal scheduler movements, but that's OK, as long

 * as the task is no longer on this CPU.

 */

static struct rq *__migrate_task(struct rq *rq, struct task_struct *p, int dest_cpu)

static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,

				 struct task_struct *p, int dest_cpu)

{

	if (unlikely(!cpu_active(dest_cpu)))

		return rq;

	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))

		return rq;

	rq = move_queued_task(rq, p, dest_cpu);

	rq = move_queued_task(rq, rf, p, dest_cpu);

	return rq;

}

	struct migration_arg *arg = data;

	struct task_struct *p = arg->task;

	struct rq *rq = this_rq();

	struct rq_flags rf;

	/*

	 * The original target CPU might have gone down and we might

	sched_ttwu_pending();

	raw_spin_lock(&p->pi_lock);

	raw_spin_lock(&rq->lock);

	rq_lock(rq, &rf);

	/*

	 * If task_rq(p) != rq, it cannot be migrated here, because we're

	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because

	 */

	if (task_rq(p) == rq) {

		if (task_on_rq_queued(p))

			rq = __migrate_task(rq, p, arg->dest_cpu);

			rq = __migrate_task(rq, &rf, p, arg->dest_cpu);

		else

			p->wake_cpu = arg->dest_cpu;

	}

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

	raw_spin_unlock(&p->pi_lock);

	local_irq_enable();

		 * OK, since we're going to drop the lock immediately

		 * afterwards anyway.

		 */

		rq_unpin_lock(rq, &rf);

		rq = move_queued_task(rq, p, dest_cpu);

		rq_repin_lock(rq, &rf);

		rq = move_queued_task(rq, &rf, p, dest_cpu);

	}

out:

	task_rq_unlock(rq, p, &rf);

{

	if (task_on_rq_queued(p)) {

		struct rq *src_rq, *dst_rq;

		struct rq_flags srf, drf;

		src_rq = task_rq(p);

		dst_rq = cpu_rq(cpu);

		rq_pin_lock(src_rq, &srf);

		rq_pin_lock(dst_rq, &drf);

		p->on_rq = TASK_ON_RQ_MIGRATING;

		deactivate_task(src_rq, p, 0);

		set_task_cpu(p, cpu);

		activate_task(dst_rq, p, 0);

		p->on_rq = TASK_ON_RQ_QUEUED;

		check_preempt_curr(dst_rq, p, 0);

		rq_unpin_lock(dst_rq, &drf);

		rq_unpin_lock(src_rq, &srf);

	} else {

		/*

		 * Task isn't running anymore; make it appear like we migrated

	struct rq *rq = this_rq();

	struct llist_node *llist = llist_del_all(&rq->wake_list);

	struct task_struct *p;

	unsigned long flags;

	struct rq_flags rf;

	if (!llist)

		return;

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_pin_lock(rq, &rf);

	rq_lock_irqsave(rq, &rf);

	while (llist) {

		int wake_flags = 0;

		ttwu_do_activate(rq, p, wake_flags, &rf);

	}

	rq_unpin_lock(rq, &rf);

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

}

void scheduler_ipi(void)

void wake_up_if_idle(int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;

	struct rq_flags rf;

	rcu_read_lock();

	if (set_nr_if_polling(rq->idle)) {

		trace_sched_wake_idle_without_ipi(cpu);

	} else {

		raw_spin_lock_irqsave(&rq->lock, flags);

		rq_lock_irqsave(rq, &rf);

		if (is_idle_task(rq->curr))

			smp_send_reschedule(cpu);

		/* Else CPU is not idle, do nothing here: */

		raw_spin_unlock_irqrestore(&rq->lock, flags);

		rq_unlock_irqrestore(rq, &rf);

	}

out:

	}

#endif

	raw_spin_lock(&rq->lock);

	rq_pin_lock(rq, &rf);

	rq_lock(rq, &rf);

	ttwu_do_activate(rq, p, wake_flags, &rf);

	rq_unpin_lock(rq, &rf);

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

}

/*

		 * disabled avoiding further scheduler activity on it and we've

		 * not yet picked a replacement task.

		 */

		rq_unpin_lock(rq, rf);

		raw_spin_unlock(&rq->lock);

		rq_unlock(rq, rf);

		raw_spin_lock(&p->pi_lock);

		raw_spin_lock(&rq->lock);

		rq_repin_lock(rq, rf);

		rq_relock(rq, rf);

	}

	if (!(p->state & TASK_NORMAL))

{

	struct callback_head *head, *next;

	void (*func)(struct rq *rq);

	unsigned long flags;

	struct rq_flags rf;

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_lock_irqsave(rq, &rf);

	head = rq->balance_callback;

	rq->balance_callback = NULL;

	while (head) {

		func(rq);

	}

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

}

static inline void balance_callback(struct rq *rq)

	int cpu = smp_processor_id();

	struct rq *rq = cpu_rq(cpu);

	struct task_struct *curr = rq->curr;

	struct rq_flags rf;

	sched_clock_tick();

	raw_spin_lock(&rq->lock);

	rq_lock(rq, &rf);

	update_rq_clock(rq);

	curr->sched_class->task_tick(rq, curr, 0);

	cpu_load_update_active(rq);

	calc_global_load_tick(rq);

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

	perf_event_task_tick();

	 * done by the caller to avoid the race with signal_wake_up().

	 */

	smp_mb__before_spinlock();

	raw_spin_lock(&rq->lock);

	rq_pin_lock(rq, &rf);

	rq_lock(rq, &rf);

	/* Promote REQ to ACT */

	rq->clock_update_flags <<= 1;

		rq = context_switch(rq, prev, next, &rf);

	} else {

		rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);

		rq_unpin_lock(rq, &rf);

		raw_spin_unlock_irq(&rq->lock);

		rq_unlock_irq(rq, &rf);

	}

	balance_callback(rq);

 */

SYSCALL_DEFINE0(sched_yield)

{

	struct rq *rq = this_rq_lock();

	struct rq_flags rf;

	struct rq *rq;

	local_irq_disable();

	rq = this_rq();

	rq_lock(rq, &rf);

	schedstat_inc(rq->yld_count);

	current->sched_class->yield_task(rq);

	 * Since we are going to call schedule() anyway, there's

	 * no need to preempt or enable interrupts:

	 */

	__release(rq->lock);

	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);

	do_raw_spin_unlock(&rq->lock);

	preempt_disable();

	rq_unlock(rq, &rf);

	sched_preempt_enable_no_resched();

	schedule();

 * there's no concurrency possible, we hold the required locks anyway

 * because of lock validation efforts.

 */

static void migrate_tasks(struct rq *dead_rq)

static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)

{

	struct rq *rq = dead_rq;

	struct task_struct *next, *stop = rq->stop;

	struct rq_flags rf;

	struct rq_flags orf = *rf;

	int dest_cpu;

	/*

	 * class method both need to have an up-to-date

	 * value of rq->clock[_task]

	 */

	rq_pin_lock(rq, &rf);

	update_rq_clock(rq);

	rq_unpin_lock(rq, &rf);

	for (;;) {

		/*

		/*

		 * pick_next_task() assumes pinned rq->lock:

		 */

		rq_repin_lock(rq, &rf);

		next = pick_next_task(rq, &fake_task, &rf);

		next = pick_next_task(rq, &fake_task, rf);

		BUG_ON(!next);

		next->sched_class->put_prev_task(rq, next);

		 * because !cpu_active at this point, which means load-balance

		 * will not interfere. Also, stop-machine.

		 */

		rq_unpin_lock(rq, &rf);

		raw_spin_unlock(&rq->lock);

		rq_unlock(rq, rf);

		raw_spin_lock(&next->pi_lock);

		raw_spin_lock(&rq->lock);

		rq_relock(rq, rf);

		/*

		 * Since we're inside stop-machine, _nothing_ should have

		/* Find suitable destination for @next, with force if needed. */

		dest_cpu = select_fallback_rq(dead_rq->cpu, next);

		rq = __migrate_task(rq, next, dest_cpu);

		rq = __migrate_task(rq, rf, next, dest_cpu);

		if (rq != dead_rq) {

			raw_spin_unlock(&rq->lock);

			rq_unlock(rq, rf);

			rq = dead_rq;

			raw_spin_lock(&rq->lock);

			*rf = orf;

			rq_relock(rq, rf);

		}

		raw_spin_unlock(&next->pi_lock);

	}

int sched_cpu_activate(unsigned int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;

	struct rq_flags rf;

	set_cpu_active(cpu, true);

	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the

	 *    domains.

	 */

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_lock_irqsave(rq, &rf);

	if (rq->rd) {

		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));

		set_rq_online(rq);

	}

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

	update_max_interval();

int sched_cpu_dying(unsigned int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;

	struct rq_flags rf;

	/* Handle pending wakeups and then migrate everything off */

	sched_ttwu_pending();

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_lock_irqsave(rq, &rf);

	if (rq->rd) {

		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));

		set_rq_offline(rq);

	}

	migrate_tasks(rq);

	migrate_tasks(rq, &rf);

	BUG_ON(rq->nr_running != 1);

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

	calc_load_migrate(rq);

	update_max_interval();

	nohz_balance_exit_idle(cpu);

	for_each_online_cpu(i) {

		struct cfs_rq *cfs_rq = tg->cfs_rq[i];

		struct rq *rq = cfs_rq->rq;

		struct rq_flags rf;

		raw_spin_lock_irq(&rq->lock);

		rq_lock_irq(rq, &rf);

		cfs_rq->runtime_enabled = runtime_enabled;

		cfs_rq->runtime_remaining = 0;

		if (cfs_rq->throttled)

			unthrottle_cfs_rq(cfs_rq);

		raw_spin_unlock_irq(&rq->lock);

		rq_unlock_irq(rq, &rf);

	}

	if (runtime_was_enabled && !runtime_enabled)

		cfs_bandwidth_usage_dec();

	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,

				throttled_list) {

		struct rq *rq = rq_of(cfs_rq);

		struct rq_flags rf;

		raw_spin_lock(&rq->lock);

		rq_lock(rq, &rf);

		if (!cfs_rq_throttled(cfs_rq))

			goto next;

			unthrottle_cfs_rq(cfs_rq);

next:

		raw_spin_unlock(&rq->lock);

		rq_unlock(rq, &rf);

		if (!remaining)

			break;

	unsigned long curr_jiffies = READ_ONCE(jiffies);

	struct rq *this_rq = this_rq();

	unsigned long load;

	struct rq_flags rf;

	if (curr_jiffies == this_rq->last_load_update_tick)

		return;

	load = weighted_cpuload(cpu_of(this_rq));

	raw_spin_lock(&this_rq->lock);

	rq_lock(this_rq, &rf);

	update_rq_clock(this_rq);

	cpu_load_update_nohz(this_rq, curr_jiffies, load);

	raw_spin_unlock(&this_rq->lock);

	rq_unlock(this_rq, &rf);

}

#else /* !CONFIG_NO_HZ_COMMON */

static inline void cpu_load_update_nohz(struct rq *this_rq,

 */

static void attach_one_task(struct rq *rq, struct task_struct *p)

{

	raw_spin_lock(&rq->lock);

	struct rq_flags rf;

	rq_lock(rq, &rf);

	attach_task(rq, p);

	raw_spin_unlock(&rq->lock);

	rq_unlock(rq, &rf);

}

/*

{

	struct list_head *tasks = &env->tasks;

	struct task_struct *p;

	struct rq_flags rf;

	raw_spin_lock(&env->dst_rq->lock);

	rq_lock(env->dst_rq, &rf);

	while (!list_empty(tasks)) {

		p = list_first_entry(tasks, struct task_struct, se.group_node);

		attach_task(env->dst_rq, p);

	}

	raw_spin_unlock(&env->dst_rq->lock);

	rq_unlock(env->dst_rq, &rf);

}

#ifdef CONFIG_FAIR_GROUP_SCHED

{

	struct rq *rq = cpu_rq(cpu);

	struct cfs_rq *cfs_rq;

	unsigned long flags;

	struct rq_flags rf;

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_lock_irqsave(rq, &rf);

	update_rq_clock(rq);

	/*

		if (cfs_rq->tg->se[cpu])

			update_load_avg(cfs_rq->tg->se[cpu], 0);

	}

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

}

/*

{

	struct rq *rq = cpu_rq(cpu);

	struct cfs_rq *cfs_rq = &rq->cfs;

	unsigned long flags;

	struct rq_flags rf;

	raw_spin_lock_irqsave(&rq->lock, flags);

	rq_lock_irqsave(rq, &rf);

	update_rq_clock(rq);

	update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq, true);

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	rq_unlock_irqrestore(rq, &rf);

}

static unsigned long task_h_load(struct task_struct *p)

	struct sched_domain *sd_parent = sd->parent;

	struct sched_group *group;

	struct rq *busiest;

	unsigned long flags;

	struct rq_flags rf;

	struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);

	struct lb_env env = {

		env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);

more_balance:

		raw_spin_lock_irqsave(&busiest->lock, flags);

		rq_lock_irqsave(busiest, &rf);

		update_rq_clock(busiest);

		/*

		 * See task_rq_lock() family for the details.

		 */

		raw_spin_unlock(&busiest->lock);

		rq_unlock(busiest, &rf);

		if (cur_ld_moved) {

			attach_tasks(&env);

			ld_moved += cur_ld_moved;

		}