Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 84f60324 authored by Josef Bacik's avatar Josef Bacik Committed by Jens Axboe
Browse files

block: add rq_qos_wait to rq_qos 



Originally when I split out the common code from blk-wbt into rq_qos I
left the wbt_wait() where it was and simply copied and modified it
slightly to work for io-latency.  However they are both basically the
same thing, and as time has gone on wbt_wait() has ended up much smarter
and kinder than it was when I copied it into io-latency, which means
io-latency has lost out on these improvements.

Since they are the same thing essentially except for a few minor things,
create rq_qos_wait() that replicates what wbt_wait() currently does with
callbacks that can be passed in for the snowflakes to do their own thing
as appropriate.

Signed-off-by: default avatarJosef Bacik <josef@toxicpanda.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 7754f669

block/blk-rq-qos.c

+86 −0
Original line number Original line Diff line number Diff line
@@ -176,6 +176,92 @@ void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle) 
	rq_depth_calc_max_depth(rqd);

	rq_depth_calc_max_depth(rqd);

}

}





struct rq_qos_wait_data {

	struct wait_queue_entry wq;

	struct task_struct *task;

	struct rq_wait *rqw;

	acquire_inflight_cb_t *cb;

	void *private_data;

	bool got_token;

};



static int rq_qos_wake_function(struct wait_queue_entry *curr,

				unsigned int mode, int wake_flags, void *key)

{

	struct rq_qos_wait_data *data = container_of(curr,

						     struct rq_qos_wait_data,

						     wq);



	/*

	 * If we fail to get a budget, return -1 to interrupt the wake up loop

	 * in __wake_up_common.

	 */

	if (!data->cb(data->rqw, data->private_data))

		return -1;



	data->got_token = true;

	list_del_init(&curr->entry);

	wake_up_process(data->task);

	return 1;

}



/**

 * rq_qos_wait - throttle on a rqw if we need to

 * @private_data - caller provided specific data

 * @acquire_inflight_cb - inc the rqw->inflight counter if we can

 * @cleanup_cb - the callback to cleanup in case we race with a waker

 *

 * This provides a uniform place for the rq_qos users to do their throttling.

 * Since you can end up with a lot of things sleeping at once, this manages the

 * waking up based on the resources available.  The acquire_inflight_cb should

 * inc the rqw->inflight if we have the ability to do so, or return false if not

 * and then we will sleep until the room becomes available.

 *

 * cleanup_cb is in case that we race with a waker and need to cleanup the

 * inflight count accordingly.

 */

void rq_qos_wait(struct rq_wait *rqw, void *private_data,

		 acquire_inflight_cb_t *acquire_inflight_cb,

		 cleanup_cb_t *cleanup_cb)

{

	struct rq_qos_wait_data data = {

		.wq = {

			.func	= rq_qos_wake_function,

			.entry	= LIST_HEAD_INIT(data.wq.entry),

		},

		.task = current,

		.rqw = rqw,

		.cb = acquire_inflight_cb,

		.private_data = private_data,

	};

	bool has_sleeper;



	has_sleeper = wq_has_sleeper(&rqw->wait);

	if (!has_sleeper && acquire_inflight_cb(rqw, private_data))

		return;



	prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);

	do {

		if (data.got_token)

			break;

		if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {

			finish_wait(&rqw->wait, &data.wq);



			/*

			 * We raced with wbt_wake_function() getting a token,

			 * which means we now have two. Put our local token

			 * and wake anyone else potentially waiting for one.

			 */

			if (data.got_token)

				cleanup_cb(rqw, private_data);

			break;

		}

		io_schedule();

		has_sleeper = false;

	} while (1);

	finish_wait(&rqw->wait, &data.wq);

}



void rq_qos_exit(struct request_queue *q)

void rq_qos_exit(struct request_queue *q)

{

{

	while (q->rq_qos) {

	while (q->rq_qos) {

block/blk-rq-qos.h

+6 −0
Original line number Original line Diff line number Diff line
@@ -93,6 +93,12 @@ static inline void rq_qos_del(struct request_queue *q, struct rq_qos *rqos) 
	}

	}

}

}





typedef bool (acquire_inflight_cb_t)(struct rq_wait *rqw, void *private_data);

typedef void (cleanup_cb_t)(struct rq_wait *rqw, void *private_data);



void rq_qos_wait(struct rq_wait *rqw, void *private_data,

		 acquire_inflight_cb_t *acquire_inflight_cb,

		 cleanup_cb_t *cleanup_cb);

bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit);

bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit);

void rq_depth_scale_up(struct rq_depth *rqd);

void rq_depth_scale_up(struct rq_depth *rqd);

void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle);

void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle);