Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e4ff6c07 authored by Srivatsa Vaddagiri's avatar Srivatsa Vaddagiri Committed by Syed Rameez Mustafa
Browse files

sched: window-stats: Avoid taking all cpu's rq->lock for long 



reset_all_window_stats() walks task-list with all cpu's rq->lock held,
which can cause spinlock timeouts if task-list is huge (and hence lead
to a spinlock bug report). Avoid this by walking task-list without
cpu's rq->lock held.

Change-Id: Id09afd8b730fa32c76cd3bff5da7c0cd7aeb8dfb
Signed-off-by: default avatarSrivatsa Vaddagiri <vatsa@codeaurora.org>
parent b432b691

kernel/sched/core.c

+67 −35
Original line number Diff line number Diff line
@@ -1401,6 +1401,9 @@ static void update_task_ravg(struct task_struct *p, struct rq *rq, 


	lockdep_assert_held(&rq->lock);



	if (!p->ravg.mark_start)

		goto done;



	update_sum = (event == PUT_PREV_TASK || event == TASK_UPDATE ||

			(sched_account_wait_time &&

			(event == PICK_NEXT_TASK || event == TASK_MIGRATE)));
@@ -1592,18 +1595,26 @@ static void init_cpu_efficiency(void) 
	min_possible_efficiency = min;

}



static void reset_task_stats(struct task_struct *p)

{

	int i;



	p->ravg.sum = 0;

	p->ravg.demand = 0;

	p->ravg.partial_demand = 0;

	p->ravg.flags &= ~(CURR_WINDOW_CONTRIB | PREV_WINDOW_CONTRIB);

	for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)

		p->ravg.sum_history[i] = 0;

	p->ravg.mark_start = 0;

}



static inline void mark_task_starting(struct task_struct *p)

{

	struct rq *rq = task_rq(p);

	u64 wallclock = sched_clock();



	if (sched_disable_window_stats)

		return;



	if (!rq->window_start) {

		p->ravg.partial_demand = 0;

		p->ravg.demand = 0;

		p->ravg.sum = 0;

	if (!rq->window_start || sched_disable_window_stats) {

		reset_task_stats(p);

		return;

	}
@@ -1667,16 +1678,15 @@ unsigned long sched_get_busy(int cpu) 
			  NSEC_PER_USEC);

}



static void reset_task_stats(struct task_struct *p)

static void reset_all_task_stats(void)

{

	int i;

	struct task_struct *g, *p;



	p->ravg.sum = 0;

	p->ravg.demand = 0;

	p->ravg.partial_demand = 0;

	p->ravg.flags &= ~(CURR_WINDOW_CONTRIB | PREV_WINDOW_CONTRIB);

	for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)

		p->ravg.sum_history[i] = 0;

	read_lock(&tasklist_lock);

	do_each_thread(g, p) {

		reset_task_stats(p);

	}  while_each_thread(g, p);

	read_unlock(&tasklist_lock);

}



/*
@@ -1699,10 +1709,12 @@ void sched_exit(struct task_struct *p) 
	unsigned long flags;

	int cpu = get_cpu();

	struct rq *rq = cpu_rq(cpu);

	u64 wallclock;



	raw_spin_lock_irqsave(&rq->lock, flags);

	/* rq->curr == p */

	update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0);

	wallclock = sched_clock();

	update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

	dequeue_task(rq, p, 0);

	if (!sched_disable_window_stats &&

			(p->ravg.flags & CURR_WINDOW_CONTRIB))
@@ -1713,6 +1725,7 @@ void sched_exit(struct task_struct *p) 
	BUG_ON((s64)rq->curr_runnable_sum < 0);

	BUG_ON((s64)rq->prev_runnable_sum < 0);

	reset_task_stats(p);

	p->ravg.mark_start = wallclock;

	p->ravg.flags |= DONT_ACCOUNT;

	enqueue_task(rq, p, 0);

	raw_spin_unlock_irqrestore(&rq->lock, flags);
@@ -1720,14 +1733,43 @@ void sched_exit(struct task_struct *p) 
	put_cpu();

}



/* Called with IRQs disabled */

static void disable_window_stats(void)

{

	unsigned long flags;

	int i;



	local_irq_save(flags);

	for_each_possible_cpu(i)

		raw_spin_lock(&cpu_rq(i)->lock);



	sched_disable_window_stats = 1;



	for_each_possible_cpu(i)

		raw_spin_unlock(&cpu_rq(i)->lock);



	local_irq_restore(flags);

}



/* Called with all cpu's rq->lock held */

static void enable_window_stats(void)

{

	sched_disable_window_stats = 0;



}



/* Called with IRQs enabled */

void reset_all_window_stats(u64 window_start, unsigned int window_size)

{

	int cpu;

	u64 wallclock;

	struct task_struct *g, *p;

	unsigned long flags;



	for_each_online_cpu(cpu) {

	disable_window_stats();



	reset_all_task_stats();



	local_irq_save(flags);



	for_each_possible_cpu(cpu) {

		struct rq *rq = cpu_rq(cpu);

		raw_spin_lock(&rq->lock);

	}
@@ -1737,16 +1779,9 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) 
		set_hmp_defaults();

	}



	wallclock = sched_clock();

	enable_window_stats();



	read_lock(&tasklist_lock);

	do_each_thread(g, p) {

		reset_task_stats(p);

		p->ravg.mark_start = wallclock;

	}  while_each_thread(g, p);

	read_unlock(&tasklist_lock);



	for_each_online_cpu(cpu) {

	for_each_possible_cpu(cpu) {

		struct rq *rq = cpu_rq(cpu);



		if (window_start)
@@ -1761,10 +1796,12 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) 
	sched_ravg_hist_size = sysctl_sched_ravg_hist_size;

	sched_freq_legacy_mode = sysctl_sched_freq_legacy_mode;



	for_each_online_cpu(cpu) {

	for_each_possible_cpu(cpu) {

		struct rq *rq = cpu_rq(cpu);

		raw_spin_unlock(&rq->lock);

	}



	local_irq_restore(flags);

}



void sched_set_io_is_busy(int val)
@@ -1776,7 +1813,6 @@ int sched_set_window(u64 window_start, unsigned int window_size) 
{

	u64 ws, now;

	int delta;

	unsigned long flags;



	if (sched_use_pelt ||

		 (window_size * TICK_NSEC <  MIN_SCHED_RAVG_WINDOW))
@@ -1784,8 +1820,6 @@ int sched_set_window(u64 window_start, unsigned int window_size) 


	update_alignment = 1;



	local_irq_save(flags);



	now = get_jiffies_64();

	if (time_after64(window_start, now)) {

		delta = window_start - now; /* how many jiffies ahead */
@@ -1802,8 +1836,6 @@ int sched_set_window(u64 window_start, unsigned int window_size) 


	reset_all_window_stats(ws, window_size);



	local_irq_restore(flags);



	return 0;

}

kernel/sched/fair.c

+0 −5
Original line number Diff line number Diff line
@@ -1913,7 +1913,6 @@ int sched_window_update_handler(struct ctl_table *table, int write, 
	int ret;

	unsigned int *data = (unsigned int *)table->data;

	unsigned int old_val;

	unsigned long flags;



	if (!sched_enable_hmp)

		return -EINVAL;
@@ -1932,12 +1931,8 @@ int sched_window_update_handler(struct ctl_table *table, int write, 
		goto done;

	}



	local_irq_save(flags);



	reset_all_window_stats(0, 0);



	local_irq_restore(flags);



done:

	mutex_unlock(&policy_mutex);