Loading kernel/sched/fair.c +40 −30 Original line number Diff line number Diff line Loading @@ -6878,37 +6878,30 @@ static bool is_packing_eligible(struct task_struct *p, int target_cpu, return (estimated_capacity <= capacity_curr_of(target_cpu)); } static inline bool skip_sg(struct task_struct *p, struct sched_group *sg, static int start_cpu(struct task_struct *p, bool boosted, struct cpumask *rtg_target) { int fcpu = group_first_cpu(sg); /* Are all CPUs isolated in this group? */ if (!sg->group_weight) return true; /* * Don't skip a group if a task affinity allows it * to run only on that group. */ if (cpumask_subset(tsk_cpus_allowed(p), sched_group_cpus(sg))) return false; struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int start_cpu = -1; if (!task_fits_max(p, fcpu)) return true; if (boosted) return rd->max_cap_orig_cpu; if (rtg_target && !cpumask_test_cpu(fcpu, rtg_target)) return true; /* A task always fits on its rtg_target */ if (rtg_target) { int rtg_target_cpu = cpumask_first_and(rtg_target, cpu_online_mask); return false; if (rtg_target_cpu < nr_cpu_ids) return rtg_target_cpu; } static int start_cpu(bool boosted) { struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int start_cpu; start_cpu = boosted ? rd->max_cap_orig_cpu : rd->min_cap_orig_cpu; /* Where the task should land based on its demand */ if (rd->min_cap_orig_cpu != -1 && task_fits_max(p, rd->min_cap_orig_cpu)) start_cpu = rd->min_cap_orig_cpu; else start_cpu = rd->max_cap_orig_cpu; return walt_start_cpu(start_cpu); } Loading @@ -6932,6 +6925,8 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, int best_idle_cpu = -1; int target_cpu = -1; int cpu, i; long spare_cap, most_spare_cap = 0; int most_spare_cap_cpu = -1; unsigned int active_cpus_count = 0; int isolated_candidate = -1; int prev_cpu = task_cpu(p); Loading @@ -6942,7 +6937,7 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, schedstat_inc(this_rq()->eas_stats.fbt_attempts); /* Find start CPU based on boost value */ cpu = start_cpu(boosted); cpu = start_cpu(p, boosted, fbt_env->rtg_target); if (cpu < 0) { schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_cpu); schedstat_inc(this_rq()->eas_stats.fbt_no_cpu); Loading @@ -6963,9 +6958,6 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, cpumask_t search_cpus; bool do_rotate = false, avoid_prev_cpu = false; if (skip_sg(p, sg, fbt_env->rtg_target)) continue; cpumask_copy(&search_cpus, tsk_cpus_allowed(p)); cpumask_and(&search_cpus, &search_cpus, sched_group_cpus(sg)); i = find_first_cpu_bit(p, &search_cpus, sg, &avoid_prev_cpu, Loading Loading @@ -7000,6 +6992,12 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, */ wake_util = cpu_util_wake(i, p); new_util = wake_util + task_util(p); spare_cap = capacity_orig_of(i) - wake_util; if (spare_cap > most_spare_cap) { most_spare_cap = spare_cap; most_spare_cap_cpu = i; } /* * Ensure minimum capacity to grant the required boost. Loading Loading @@ -7235,6 +7233,13 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, target_capacity = ULONG_MAX; } /* * if we have found a target cpu within a group, don't bother * checking other groups. */ if (target_capacity != ULONG_MAX) break; } while (sg = sg->next, sg != sd->groups); if (best_idle_cpu != -1 && !is_packing_eligible(p, target_cpu, fbt_env, Loading Loading @@ -7275,6 +7280,11 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, ? best_active_cpu : best_idle_cpu; if (target_cpu == -1 && most_spare_cap_cpu != -1 && /* ensure we use active cpu for active migration */ !(p->state == TASK_RUNNING && !idle_cpu(most_spare_cap_cpu))) target_cpu = most_spare_cap_cpu; if (target_cpu == -1 && cpu_isolated(prev_cpu) && isolated_candidate != -1) { target_cpu = isolated_candidate; Loading Loading
kernel/sched/fair.c +40 −30 Original line number Diff line number Diff line Loading @@ -6878,37 +6878,30 @@ static bool is_packing_eligible(struct task_struct *p, int target_cpu, return (estimated_capacity <= capacity_curr_of(target_cpu)); } static inline bool skip_sg(struct task_struct *p, struct sched_group *sg, static int start_cpu(struct task_struct *p, bool boosted, struct cpumask *rtg_target) { int fcpu = group_first_cpu(sg); /* Are all CPUs isolated in this group? */ if (!sg->group_weight) return true; /* * Don't skip a group if a task affinity allows it * to run only on that group. */ if (cpumask_subset(tsk_cpus_allowed(p), sched_group_cpus(sg))) return false; struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int start_cpu = -1; if (!task_fits_max(p, fcpu)) return true; if (boosted) return rd->max_cap_orig_cpu; if (rtg_target && !cpumask_test_cpu(fcpu, rtg_target)) return true; /* A task always fits on its rtg_target */ if (rtg_target) { int rtg_target_cpu = cpumask_first_and(rtg_target, cpu_online_mask); return false; if (rtg_target_cpu < nr_cpu_ids) return rtg_target_cpu; } static int start_cpu(bool boosted) { struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int start_cpu; start_cpu = boosted ? rd->max_cap_orig_cpu : rd->min_cap_orig_cpu; /* Where the task should land based on its demand */ if (rd->min_cap_orig_cpu != -1 && task_fits_max(p, rd->min_cap_orig_cpu)) start_cpu = rd->min_cap_orig_cpu; else start_cpu = rd->max_cap_orig_cpu; return walt_start_cpu(start_cpu); } Loading @@ -6932,6 +6925,8 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, int best_idle_cpu = -1; int target_cpu = -1; int cpu, i; long spare_cap, most_spare_cap = 0; int most_spare_cap_cpu = -1; unsigned int active_cpus_count = 0; int isolated_candidate = -1; int prev_cpu = task_cpu(p); Loading @@ -6942,7 +6937,7 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, schedstat_inc(this_rq()->eas_stats.fbt_attempts); /* Find start CPU based on boost value */ cpu = start_cpu(boosted); cpu = start_cpu(p, boosted, fbt_env->rtg_target); if (cpu < 0) { schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_cpu); schedstat_inc(this_rq()->eas_stats.fbt_no_cpu); Loading @@ -6963,9 +6958,6 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, cpumask_t search_cpus; bool do_rotate = false, avoid_prev_cpu = false; if (skip_sg(p, sg, fbt_env->rtg_target)) continue; cpumask_copy(&search_cpus, tsk_cpus_allowed(p)); cpumask_and(&search_cpus, &search_cpus, sched_group_cpus(sg)); i = find_first_cpu_bit(p, &search_cpus, sg, &avoid_prev_cpu, Loading Loading @@ -7000,6 +6992,12 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, */ wake_util = cpu_util_wake(i, p); new_util = wake_util + task_util(p); spare_cap = capacity_orig_of(i) - wake_util; if (spare_cap > most_spare_cap) { most_spare_cap = spare_cap; most_spare_cap_cpu = i; } /* * Ensure minimum capacity to grant the required boost. Loading Loading @@ -7235,6 +7233,13 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, target_capacity = ULONG_MAX; } /* * if we have found a target cpu within a group, don't bother * checking other groups. */ if (target_capacity != ULONG_MAX) break; } while (sg = sg->next, sg != sd->groups); if (best_idle_cpu != -1 && !is_packing_eligible(p, target_cpu, fbt_env, Loading Loading @@ -7275,6 +7280,11 @@ static inline int find_best_target(struct task_struct *p, int *backup_cpu, ? best_active_cpu : best_idle_cpu; if (target_cpu == -1 && most_spare_cap_cpu != -1 && /* ensure we use active cpu for active migration */ !(p->state == TASK_RUNNING && !idle_cpu(most_spare_cap_cpu))) target_cpu = most_spare_cap_cpu; if (target_cpu == -1 && cpu_isolated(prev_cpu) && isolated_candidate != -1) { target_cpu = isolated_candidate; Loading