Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 098fb9db authored by Ingo Molnar's avatar Ingo Molnar
Browse files

sched: clean up wakeup balancing, move wake_affine() 



split out the affine-wakeup bits.

No code changed:

kernel/sched.o:

   text	   data	    bss	    dec	    hex	filename
  42521	   2858	    232	  45611	   b22b	sched.o.before
  42521	   2858	    232	  45611	   b22b	sched.o.after

md5:
   9d76738f1272aa82f0b7affd2f51df6b  sched.o.before.asm
   09b31c44e9aff8666f72773dc433e2df  sched.o.after.asm

(the md5's changed because stack slots changed and some registers
get scheduled by gcc in a different order - but otherwise the before
and after assembly is instruction for instruction equivalent.)

Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent f920bb6f

kernel/sched_fair.c

+75 −59
Original line number Diff line number Diff line
@@ -980,12 +980,59 @@ static inline int wake_idle(int cpu, struct task_struct *p) 
#endif



#ifdef CONFIG_SMP



static int

wake_affine(struct rq *rq, struct sched_domain *this_sd, struct task_struct *p,

	    int cpu, int this_cpu, int sync, int idx,

	    unsigned long load, unsigned long this_load,

	    unsigned int imbalance)

{

	unsigned long tl = this_load;

	unsigned long tl_per_task;



	if (!(this_sd->flags & SD_WAKE_AFFINE))

		return 0;



	/*

	 * Attract cache-cold tasks on sync wakeups:

	 */

	if (sync && !task_hot(p, rq->clock, this_sd))

		return 1;



	schedstat_inc(p, se.nr_wakeups_affine_attempts);

	tl_per_task = cpu_avg_load_per_task(this_cpu);



	/*

	 * If sync wakeup then subtract the (maximum possible)

	 * effect of the currently running task from the load

	 * of the current CPU:

	 */

	if (sync)

		tl -= current->se.load.weight;



	if ((tl <= load && tl + target_load(cpu, idx) <= tl_per_task) ||

			100*(tl + p->se.load.weight) <= imbalance*load) {

		/*

		 * This domain has SD_WAKE_AFFINE and

		 * p is cache cold in this domain, and

		 * there is no bad imbalance.

		 */

		schedstat_inc(this_sd, ttwu_move_affine);

		schedstat_inc(p, se.nr_wakeups_affine);



		return 1;

	}

	return 0;

}



static int select_task_rq_fair(struct task_struct *p, int sync)

{

	int cpu, this_cpu;

	struct rq *rq;

	struct sched_domain *sd, *this_sd = NULL;

	int new_cpu;

	unsigned long load, this_load;

	int cpu, this_cpu, new_cpu;

	unsigned int imbalance;

	struct rq *rq;

	int idx;



	cpu      = task_cpu(p);

	rq       = task_rq(p);
@@ -1008,10 +1055,10 @@ static int select_task_rq_fair(struct task_struct *p, int sync) 
	/*

	 * Check for affine wakeup and passive balancing possibilities.

	 */

	if (this_sd) {

		int idx = this_sd->wake_idx;

		unsigned int imbalance;

		unsigned long load, this_load;

	if (!this_sd)

		goto out_keep_cpu;



	idx = this_sd->wake_idx;



	imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;
@@ -1020,40 +1067,9 @@ static int select_task_rq_fair(struct task_struct *p, int sync) 


	new_cpu = this_cpu; /* Wake to this CPU if we can */



		if (this_sd->flags & SD_WAKE_AFFINE) {

			unsigned long tl = this_load;

			unsigned long tl_per_task;



			/*

			 * Attract cache-cold tasks on sync wakeups:

			 */

			if (sync && !task_hot(p, rq->clock, this_sd))

				goto out_set_cpu;



			schedstat_inc(p, se.nr_wakeups_affine_attempts);

			tl_per_task = cpu_avg_load_per_task(this_cpu);



			/*

			 * If sync wakeup then subtract the (maximum possible)

			 * effect of the currently running task from the load

			 * of the current CPU:

			 */

			if (sync)

				tl -= current->se.load.weight;



			if ((tl <= load &&

				tl + target_load(cpu, idx) <= tl_per_task) ||

			       100*(tl + p->se.load.weight) <= imbalance*load) {

				/*

				 * This domain has SD_WAKE_AFFINE and

				 * p is cache cold in this domain, and

				 * there is no bad imbalance.

				 */

				schedstat_inc(this_sd, ttwu_move_affine);

				schedstat_inc(p, se.nr_wakeups_affine);

	if (wake_affine(rq, this_sd, p, cpu, this_cpu, sync, idx,

				     load, this_load, imbalance))

		goto out_set_cpu;

			}

		}



	/*

	 * Start passive balancing when half the imbalance_pct
@@ -1066,8 +1082,8 @@ static int select_task_rq_fair(struct task_struct *p, int sync) 
			goto out_set_cpu;

		}

	}

	}



out_keep_cpu:

	new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */

out_set_cpu:

	return wake_idle(new_cpu, p);