Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bd166ef1 authored by Jens Axboe's avatar Jens Axboe
Browse files

blk-mq-sched: add framework for MQ capable IO schedulers 



This adds a set of hooks that intercepts the blk-mq path of
allocating/inserting/issuing/completing requests, allowing
us to develop a scheduler within that framework.

We reuse the existing elevator scheduler API on the registration
side, but augment that with the scheduler flagging support for
the blk-mq interfce, and with a separate set of ops hooks for MQ
devices.

We split driver and scheduler tags, so we can run the scheduling
independently of device queue depth.

Signed-off-by: default avatarJens Axboe <axboe@fb.com>
Reviewed-by: default avatarBart Van Assche <bart.vanassche@sandisk.com>
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
parent 2af8cbe3

block/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ 
			blk-flush.o blk-settings.o blk-ioc.o blk-map.o \

			blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \

			blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \

			blk-mq-sysfs.o blk-mq-cpumap.o ioctl.o \

			blk-mq-sysfs.o blk-mq-cpumap.o blk-mq-sched.o ioctl.o \

			genhd.o scsi_ioctl.o partition-generic.o ioprio.o \

			badblocks.o partitions/

block/blk-cgroup.c

+20 −4
Original line number Diff line number Diff line
@@ -1223,6 +1223,10 @@ int blkcg_activate_policy(struct request_queue *q, 
	if (blkcg_policy_enabled(q, pol))

		return 0;



	if (q->mq_ops) {

		blk_mq_freeze_queue(q);

		blk_mq_quiesce_queue(q);

	} else

		blk_queue_bypass_start(q);

pd_prealloc:

	if (!pd_prealloc) {
@@ -1261,6 +1265,9 @@ int blkcg_activate_policy(struct request_queue *q, 


	spin_unlock_irq(q->queue_lock);

out_bypass_end:

	if (q->mq_ops)

		blk_mq_unfreeze_queue(q);

	else

		blk_queue_bypass_end(q);

	if (pd_prealloc)

		pol->pd_free_fn(pd_prealloc);
@@ -1284,7 +1291,12 @@ void blkcg_deactivate_policy(struct request_queue *q, 
	if (!blkcg_policy_enabled(q, pol))

		return;



	if (q->mq_ops) {

		blk_mq_freeze_queue(q);

		blk_mq_quiesce_queue(q);

	} else

		blk_queue_bypass_start(q);



	spin_lock_irq(q->queue_lock);



	__clear_bit(pol->plid, q->blkcg_pols);
@@ -1304,6 +1316,10 @@ void blkcg_deactivate_policy(struct request_queue *q, 
	}



	spin_unlock_irq(q->queue_lock);



	if (q->mq_ops)

		blk_mq_unfreeze_queue(q);

	else

		blk_queue_bypass_end(q);

}

EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);

block/blk-core.c

+3 −1
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ 


#include "blk.h"

#include "blk-mq.h"

#include "blk-mq-sched.h"

#include "blk-wbt.h"



EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
@@ -134,6 +135,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) 
	rq->cmd = rq->__cmd;

	rq->cmd_len = BLK_MAX_CDB;

	rq->tag = -1;

	rq->internal_tag = -1;

	rq->start_time = jiffies;

	set_start_time_ns(rq);

	rq->part = NULL;
@@ -2127,7 +2129,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) 
	if (q->mq_ops) {

		if (blk_queue_io_stat(q))

			blk_account_io_start(rq, true);

		blk_mq_insert_request(rq, false, true, false);

		blk_mq_sched_insert_request(rq, false, true, false);

		return 0;

	}

block/blk-exec.c

+2 −1
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include <linux/sched/sysctl.h>



#include "blk.h"

#include "blk-mq-sched.h"



/*

 * for max sense size
@@ -65,7 +66,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, 
	 * be reused after dying flag is set

	 */

	if (q->mq_ops) {

		blk_mq_insert_request(rq, at_head, true, false);

		blk_mq_sched_insert_request(rq, at_head, true, false);

		return;

	}

block/blk-flush.c

+7 −5
Original line number Diff line number Diff line
@@ -74,6 +74,7 @@ 
#include "blk.h"

#include "blk-mq.h"

#include "blk-mq-tag.h"

#include "blk-mq-sched.h"



/* FLUSH/FUA sequences */

enum {
@@ -391,9 +392,10 @@ static void mq_flush_data_end_io(struct request *rq, int error) 
	 * the comment in flush_end_io().

	 */

	spin_lock_irqsave(&fq->mq_flush_lock, flags);

	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))

		blk_mq_run_hw_queue(hctx, true);

	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);

	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);



	blk_mq_run_hw_queue(hctx, true);

}



/**
@@ -453,9 +455,9 @@ void blk_insert_flush(struct request *rq) 
	 */

	if ((policy & REQ_FSEQ_DATA) &&

	    !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {

		if (q->mq_ops) {

			blk_mq_insert_request(rq, false, true, false);

		} else

		if (q->mq_ops)

			blk_mq_sched_insert_request(rq, false, true, false);

		else

			list_add_tail(&rq->queuelist, &q->queue_head);

		return;

	}