@@ -5777,6 +5777,7 @@ struct lb_env {
int new_dst_cpu;
enum cpu_idle_type idle;
long imbalance;
+ long imbalance_tasks;
/* The set of CPUs under consideration for load-balancing */
struct cpumask *cpus;
@@ -6018,7 +6019,7 @@ static int detach_tasks(struct lb_env *env)
lockdep_assert_held(&env->src_rq->lock);
- if (env->imbalance <= 0)
+ if (env->imbalance <= 0 || env->imbalance_tasks == 0)
return 0;
while (!list_empty(tasks)) {
@@ -6072,9 +6073,9 @@ static int detach_tasks(struct lb_env *env)
/*
* We only want to steal up to the prescribed amount of
- * weighted load.
+ * weighted load or max number of tasks.
*/
- if (env->imbalance <= 0)
+ if (env->imbalance <= 0 || env->imbalance_tasks <= detached)
break;
continue;
@@ -6873,12 +6874,14 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
*/
static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
{
- unsigned long max_pull, load_above_capacity = ~0UL;
+ unsigned long max_pull;
struct sg_lb_stats *local, *busiest;
local = &sds->local_stat;
busiest = &sds->busiest_stat;
+ env->imbalance_tasks = ~0UL;
+
if (busiest->group_type == group_imbalanced) {
/*
* In the group_imb case we cannot rely on group-wide averages
@@ -6904,23 +6907,17 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
*/
if (busiest->group_type == group_overloaded &&
local->group_type == group_overloaded) {
- load_above_capacity = busiest->sum_nr_running *
- SCHED_LOAD_SCALE;
- if (load_above_capacity > busiest->group_capacity)
- load_above_capacity -= busiest->group_capacity;
- else
- load_above_capacity = ~0UL;
+ if (busiest->sum_nr_running > busiest->group_weight)
+ env->imbalance_tasks =
+ busiest->sum_nr_running - busiest->group_weight;
}
/*
* We're trying to get all the cpus to the average_load, so we don't
* want to push ourselves above the average load, nor do we wish to
- * reduce the max loaded cpu below the average load. At the same time,
- * we also don't want to reduce the group load below the group capacity
- * (so that we can implement power-savings policies etc). Thus we look
- * for the minimum possible imbalance.
+ * reduce the max loaded cpu below the average load.
*/
- max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity);
+ max_pull = busiest->avg_load - sds->avg_load;
/* How much load to actually move to equalise the imbalance */
env->imbalance = min(