Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3f0abd80 authored by Shaohua Li's avatar Shaohua Li Committed by Jens Axboe
Browse files

blk-throttle: add downgrade logic 



When queue state machine is in LIMIT_MAX state, but a cgroup is below
its low limit for some time, the queue should be downgraded to lower
state as one cgroup's low limit isn't met.

Signed-off-by: default avatarShaohua Li <shli@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent c79892c5

block/blk-throttle.c

+156 −0
Original line number Diff line number Diff line
@@ -140,6 +140,13 @@ struct throtl_grp { 
	/* Number of bio's dispatched in current slice */

	unsigned int io_disp[2];



	unsigned long last_low_overflow_time[2];



	uint64_t last_bytes_disp[2];

	unsigned int last_io_disp[2];



	unsigned long last_check_time;



	/* When did we start a new slice */

	unsigned long slice_start[2];

	unsigned long slice_end[2];
@@ -159,6 +166,9 @@ struct throtl_data 
	struct work_struct dispatch_work;

	unsigned int limit_index;

	bool limit_valid[LIMIT_CNT];



	unsigned long low_upgrade_time;

	unsigned long low_downgrade_time;

};



static void throtl_pending_timer_fn(unsigned long arg);
@@ -898,6 +908,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) 
	/* Charge the bio to the group */

	tg->bytes_disp[rw] += bio->bi_iter.bi_size;

	tg->io_disp[rw]++;

	tg->last_bytes_disp[rw] += bio->bi_iter.bi_size;

	tg->last_io_disp[rw]++;



	/*

	 * BIO_THROTTLED is used to prevent the same bio to be throttled
@@ -1527,6 +1539,45 @@ static struct blkcg_policy blkcg_policy_throtl = { 
	.pd_free_fn		= throtl_pd_free,

};



static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg)

{

	unsigned long rtime = jiffies, wtime = jiffies;



	if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW])

		rtime = tg->last_low_overflow_time[READ];

	if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW])

		wtime = tg->last_low_overflow_time[WRITE];

	return min(rtime, wtime);

}



/* tg should not be an intermediate node */

static unsigned long tg_last_low_overflow_time(struct throtl_grp *tg)

{

	struct throtl_service_queue *parent_sq;

	struct throtl_grp *parent = tg;

	unsigned long ret = __tg_last_low_overflow_time(tg);



	while (true) {

		parent_sq = parent->service_queue.parent_sq;

		parent = sq_to_tg(parent_sq);

		if (!parent)

			break;



		/*

		 * The parent doesn't have low limit, it always reaches low

		 * limit. Its overflow time is useless for children

		 */

		if (!parent->bps[READ][LIMIT_LOW] &&

		    !parent->iops[READ][LIMIT_LOW] &&

		    !parent->bps[WRITE][LIMIT_LOW] &&

		    !parent->iops[WRITE][LIMIT_LOW])

			continue;

		if (time_after(__tg_last_low_overflow_time(parent), ret))

			ret = __tg_last_low_overflow_time(parent);

	}

	return ret;

}



static bool throtl_tg_can_upgrade(struct throtl_grp *tg)

{

	struct throtl_service_queue *sq = &tg->service_queue;
@@ -1570,6 +1621,9 @@ static bool throtl_can_upgrade(struct throtl_data *td, 
	if (td->limit_index != LIMIT_LOW)

		return false;



	if (time_before(jiffies, td->low_downgrade_time + throtl_slice))

		return false;



	rcu_read_lock();

	blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {

		struct throtl_grp *tg = blkg_to_tg(blkg);
@@ -1593,6 +1647,7 @@ static void throtl_upgrade_state(struct throtl_data *td) 
	struct blkcg_gq *blkg;



	td->limit_index = LIMIT_MAX;

	td->low_upgrade_time = jiffies;

	rcu_read_lock();

	blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {

		struct throtl_grp *tg = blkg_to_tg(blkg);
@@ -1608,6 +1663,99 @@ static void throtl_upgrade_state(struct throtl_data *td) 
	queue_work(kthrotld_workqueue, &td->dispatch_work);

}



static void throtl_downgrade_state(struct throtl_data *td, int new)

{

	td->limit_index = new;

	td->low_downgrade_time = jiffies;

}



static bool throtl_tg_can_downgrade(struct throtl_grp *tg)

{

	struct throtl_data *td = tg->td;

	unsigned long now = jiffies;



	/*

	 * If cgroup is below low limit, consider downgrade and throttle other

	 * cgroups

	 */

	if (time_after_eq(now, td->low_upgrade_time + throtl_slice) &&

	    time_after_eq(now, tg_last_low_overflow_time(tg) + throtl_slice))

		return true;

	return false;

}



static bool throtl_hierarchy_can_downgrade(struct throtl_grp *tg)

{

	while (true) {

		if (!throtl_tg_can_downgrade(tg))

			return false;

		tg = sq_to_tg(tg->service_queue.parent_sq);

		if (!tg || !tg_to_blkg(tg)->parent)

			break;

	}

	return true;

}



static void throtl_downgrade_check(struct throtl_grp *tg)

{

	uint64_t bps;

	unsigned int iops;

	unsigned long elapsed_time;

	unsigned long now = jiffies;



	if (tg->td->limit_index != LIMIT_MAX ||

	    !tg->td->limit_valid[LIMIT_LOW])

		return;

	if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))

		return;

	if (time_after(tg->last_check_time + throtl_slice, now))

		return;



	elapsed_time = now - tg->last_check_time;

	tg->last_check_time = now;



	if (time_before(now, tg_last_low_overflow_time(tg) + throtl_slice))

		return;



	if (tg->bps[READ][LIMIT_LOW]) {

		bps = tg->last_bytes_disp[READ] * HZ;

		do_div(bps, elapsed_time);

		if (bps >= tg->bps[READ][LIMIT_LOW])

			tg->last_low_overflow_time[READ] = now;

	}



	if (tg->bps[WRITE][LIMIT_LOW]) {

		bps = tg->last_bytes_disp[WRITE] * HZ;

		do_div(bps, elapsed_time);

		if (bps >= tg->bps[WRITE][LIMIT_LOW])

			tg->last_low_overflow_time[WRITE] = now;

	}



	if (tg->iops[READ][LIMIT_LOW]) {

		iops = tg->last_io_disp[READ] * HZ / elapsed_time;

		if (iops >= tg->iops[READ][LIMIT_LOW])

			tg->last_low_overflow_time[READ] = now;

	}



	if (tg->iops[WRITE][LIMIT_LOW]) {

		iops = tg->last_io_disp[WRITE] * HZ / elapsed_time;

		if (iops >= tg->iops[WRITE][LIMIT_LOW])

			tg->last_low_overflow_time[WRITE] = now;

	}



	/*

	 * If cgroup is below low limit, consider downgrade and throttle other

	 * cgroups

	 */

	if (throtl_hierarchy_can_downgrade(tg))

		throtl_downgrade_state(tg->td, LIMIT_LOW);



	tg->last_bytes_disp[READ] = 0;

	tg->last_bytes_disp[WRITE] = 0;

	tg->last_io_disp[READ] = 0;

	tg->last_io_disp[WRITE] = 0;

}



bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,

		    struct bio *bio)

{
@@ -1632,12 +1780,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, 


again:

	while (true) {

		if (tg->last_low_overflow_time[rw] == 0)

			tg->last_low_overflow_time[rw] = jiffies;

		throtl_downgrade_check(tg);

		/* throtl is FIFO - if bios are already queued, should queue */

		if (sq->nr_queued[rw])

			break;



		/* if above limits, break to queue */

		if (!tg_may_dispatch(tg, bio, NULL)) {

			tg->last_low_overflow_time[rw] = jiffies;

			if (throtl_can_upgrade(tg->td, tg)) {

				throtl_upgrade_state(tg->td);

				goto again;
@@ -1681,6 +1833,8 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, 
		   tg->io_disp[rw], tg_iops_limit(tg, rw),

		   sq->nr_queued[READ], sq->nr_queued[WRITE]);



	tg->last_low_overflow_time[rw] = jiffies;



	bio_associate_current(bio);

	tg->td->nr_queued[rw]++;

	throtl_add_bio_tg(bio, qn, tg);
@@ -1791,6 +1945,8 @@ int blk_throtl_init(struct request_queue *q) 


	td->limit_valid[LIMIT_MAX] = true;

	td->limit_index = LIMIT_MAX;

	td->low_upgrade_time = jiffies;

	td->low_downgrade_time = jiffies;

	/* activate policy */

	ret = blkcg_activate_policy(q, &blkcg_policy_throtl);

	if (ret)