Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 371fd7e7 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched: Add enqueue/dequeue flags 



In order to reduce the dependency on TASK_WAKING rework the enqueue
interface to support a proper flags field.

Replace the int wakeup, bool head arguments with an int flags argument
and create the following flags:

  ENQUEUE_WAKEUP - the enqueue is a wakeup of a sleeping task,
  ENQUEUE_WAKING - the enqueue has relative vruntime due to
                   having sched_class::task_waking() called,
  ENQUEUE_HEAD - the waking task should be places on the head
                 of the priority queue (where appropriate).

For symmetry also convert sched_class::dequeue() to a flags scheme.

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent cc87f76a

include/linux/sched.h

+8 −3
Original line number Diff line number Diff line
@@ -1032,12 +1032,17 @@ struct sched_domain; 
#define WF_SYNC		0x01		/* waker goes to sleep after wakup */

#define WF_FORK		0x02		/* child wakeup after fork */



#define ENQUEUE_WAKEUP		1

#define ENQUEUE_WAKING		2

#define ENQUEUE_HEAD		4



#define DEQUEUE_SLEEP		1



struct sched_class {

	const struct sched_class *next;



	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup,

			      bool head);

	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);

	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);

	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);

	void (*yield_task) (struct rq *rq);



	void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);

kernel/sched.c

+17 −15
Original line number Diff line number Diff line
@@ -1877,44 +1877,43 @@ static void update_avg(u64 *avg, u64 sample) 
	*avg += diff >> 3;

}



static void

enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, bool head)

static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)

{

	update_rq_clock(rq);

	sched_info_queued(p);

	p->sched_class->enqueue_task(rq, p, wakeup, head);

	p->sched_class->enqueue_task(rq, p, flags);

	p->se.on_rq = 1;

}



static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)

static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)

{

	update_rq_clock(rq);

	sched_info_dequeued(p);

	p->sched_class->dequeue_task(rq, p, sleep);

	p->sched_class->dequeue_task(rq, p, flags);

	p->se.on_rq = 0;

}



/*

 * activate_task - move a task to the runqueue.

 */

static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)

static void activate_task(struct rq *rq, struct task_struct *p, int flags)

{

	if (task_contributes_to_load(p))

		rq->nr_uninterruptible--;



	enqueue_task(rq, p, wakeup, false);

	enqueue_task(rq, p, flags);

	inc_nr_running(rq);

}



/*

 * deactivate_task - remove a task from the runqueue.

 */

static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)

static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)

{

	if (task_contributes_to_load(p))

		rq->nr_uninterruptible++;



	dequeue_task(rq, p, sleep);

	dequeue_task(rq, p, flags);

	dec_nr_running(rq);

}
@@ -2353,6 +2352,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, 
{

	int cpu, orig_cpu, this_cpu, success = 0;

	unsigned long flags;

	unsigned long en_flags = ENQUEUE_WAKEUP;

	struct rq *rq;



	this_cpu = get_cpu();
@@ -2386,8 +2386,10 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, 
	}

	p->state = TASK_WAKING;



	if (p->sched_class->task_waking)

	if (p->sched_class->task_waking) {

		p->sched_class->task_waking(rq, p);

		en_flags |= ENQUEUE_WAKING;

	}



	cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags);

	if (cpu != orig_cpu)
@@ -2432,7 +2434,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, 
		schedstat_inc(p, se.statistics.nr_wakeups_local);

	else

		schedstat_inc(p, se.statistics.nr_wakeups_remote);

	activate_task(rq, p, 1);

	activate_task(rq, p, en_flags);

	success = 1;



out_running:
@@ -3623,7 +3625,7 @@ asmlinkage void __sched schedule(void) 
		if (unlikely(signal_pending_state(prev->state, prev)))

			prev->state = TASK_RUNNING;

		else

			deactivate_task(rq, prev, 1);

			deactivate_task(rq, prev, DEQUEUE_SLEEP);

		switch_count = &prev->nvcsw;

	}
@@ -4193,7 +4195,7 @@ void rt_mutex_setprio(struct task_struct *p, int prio) 
	if (running)

		p->sched_class->set_curr_task(rq);

	if (on_rq) {

		enqueue_task(rq, p, 0, oldprio < prio);

		enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);



		check_class_changed(rq, p, prev_class, oldprio, running);

	}
@@ -4236,7 +4238,7 @@ void set_user_nice(struct task_struct *p, long nice) 
	delta = p->prio - old_prio;



	if (on_rq) {

		enqueue_task(rq, p, 0, false);

		enqueue_task(rq, p, 0);

		/*

		 * If the task increased its priority or is running and

		 * lowered its priority, then reschedule its CPU:
@@ -8180,7 +8182,7 @@ void sched_move_task(struct task_struct *tsk) 
	if (unlikely(running))

		tsk->sched_class->set_curr_task(rq);

	if (on_rq)

		enqueue_task(rq, tsk, 0, false);

		enqueue_task(rq, tsk, 0);



	task_rq_unlock(rq, &flags);

}

kernel/sched_fair.c

+8 −17
Original line number Diff line number Diff line
@@ -757,9 +757,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) 
	se->vruntime = vruntime;

}



#define ENQUEUE_WAKEUP	1

#define ENQUEUE_MIGRATE 2



static void

enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)

{
@@ -767,7 +764,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 
	 * Update the normalized vruntime before updating min_vruntime

	 * through callig update_curr().

	 */

	if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATE))

	if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING))

		se->vruntime += cfs_rq->min_vruntime;



	/*
@@ -803,7 +800,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 
}



static void

dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)

dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)

{

	/*

	 * Update run-time statistics of the 'current'.
@@ -811,7 +808,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) 
	update_curr(cfs_rq);



	update_stats_dequeue(cfs_rq, se);

	if (sleep) {

	if (flags & DEQUEUE_SLEEP) {

#ifdef CONFIG_SCHEDSTATS

		if (entity_is_task(se)) {

			struct task_struct *tsk = task_of(se);
@@ -836,7 +833,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) 
	 * update can refer to the ->curr item and we need to reflect this

	 * movement in our normalized position.

	 */

	if (!sleep)

	if (!(flags & DEQUEUE_SLEEP))

		se->vruntime -= cfs_rq->min_vruntime;

}
@@ -1045,16 +1042,10 @@ static inline void hrtick_update(struct rq *rq) 
 * then put the task into the rbtree:

 */

static void

enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, bool head)

enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)

{

	struct cfs_rq *cfs_rq;

	struct sched_entity *se = &p->se;

	int flags = 0;



	if (wakeup)

		flags |= ENQUEUE_WAKEUP;

	if (p->state == TASK_WAKING)

		flags |= ENQUEUE_MIGRATE;



	for_each_sched_entity(se) {

		if (se->on_rq)
@@ -1072,18 +1063,18 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, bool head) 
 * decreased. We remove the task from the rbtree and

 * update the fair scheduling stats:

 */

static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep)

static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)

{

	struct cfs_rq *cfs_rq;

	struct sched_entity *se = &p->se;



	for_each_sched_entity(se) {

		cfs_rq = cfs_rq_of(se);

		dequeue_entity(cfs_rq, se, sleep);

		dequeue_entity(cfs_rq, se, flags);

		/* Don't dequeue parent if it has other entities besides us */

		if (cfs_rq->load.weight)

			break;

		sleep = 1;

		flags |= DEQUEUE_SLEEP;

	}



	hrtick_update(rq);

kernel/sched_idletask.c

+1 −1
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ static struct task_struct *pick_next_task_idle(struct rq *rq) 
 * message if some code attempts to do it:

 */

static void

dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep)

dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags)

{

	raw_spin_unlock_irq(&rq->lock);

	printk(KERN_ERR "bad: scheduling from the idle thread!\n");

kernel/sched_rt.c

+4 −4
Original line number Diff line number Diff line
@@ -888,20 +888,20 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se) 
 * Adding/removing a task to/from a priority array:

 */

static void

enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, bool head)

enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)

{

	struct sched_rt_entity *rt_se = &p->rt;



	if (wakeup)

	if (flags & ENQUEUE_WAKEUP)

		rt_se->timeout = 0;



	enqueue_rt_entity(rt_se, head);

	enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);



	if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)

		enqueue_pushable_task(rq, p);

}



static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)

static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)

{

	struct sched_rt_entity *rt_se = &p->rt;