Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit aeb6fafb authored by Jens Axboe's avatar Jens Axboe Committed by Linus Torvalds
Browse files

block: Add flag for telling the IO schedulers NOT to anticipate more IO 



By default, CFQ will anticipate more IO from a given io context if the
previously completed IO was sync. This used to be fine, since the only
sync IO was reads and O_DIRECT writes. But with more "normal" sync writes
being used now, we don't want to anticipate for those.

Add a bio/request flag that informs the IO scheduler that this is a sync
request that we should not idle for. Introduce WRITE_ODIRECT specifically
for O_DIRECT writes, and make sure that the other sync writes set this
flag.

Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 644b2d99

block/blk-core.c

+2 −0
Original line number Original line Diff line number Diff line
@@ -1128,6 +1128,8 @@ void init_request_from_bio(struct request *req, struct bio *bio) 
		req->cmd_flags |= REQ_UNPLUG;

		req->cmd_flags |= REQ_UNPLUG;

	if (bio_rw_meta(bio))

	if (bio_rw_meta(bio))

		req->cmd_flags |= REQ_RW_META;

		req->cmd_flags |= REQ_RW_META;

	if (bio_noidle(bio))

		req->cmd_flags |= REQ_NOIDLE;





	req->errors = 0;

	req->errors = 0;

	req->hard_sector = req->sector = bio->bi_sector;

	req->hard_sector = req->sector = bio->bi_sector;

block/cfq-iosched.c

+3 −1
Original line number Original line Diff line number Diff line
@@ -1992,9 +1992,11 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) 
		}

		}

		if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq))

		if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq))

			cfq_slice_expired(cfqd, 1);

			cfq_slice_expired(cfqd, 1);

		else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list))

		else if (sync && !rq_noidle(rq) &&

			 RB_EMPTY_ROOT(&cfqq->sort_list)) {

			cfq_arm_slice_timer(cfqd);

			cfq_arm_slice_timer(cfqd);

		}

		}

	}





	if (!cfqd->rq_in_driver)

	if (!cfqd->rq_in_driver)

		cfq_schedule_dispatch(cfqd);

		cfq_schedule_dispatch(cfqd);

fs/direct-io.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -1126,7 +1126,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
	int acquire_i_mutex = 0;

	int acquire_i_mutex = 0;





	if (rw & WRITE)

	if (rw & WRITE)

		rw = WRITE_SYNC;

		rw = WRITE_ODIRECT;





	if (bdev)

	if (bdev)

		bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));

		bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));

include/linux/bio.h

+11 −8
Original line number Original line Diff line number Diff line
@@ -145,20 +145,21 @@ struct bio { 
 * bit 2 -- barrier

 * bit 2 -- barrier

 *	Insert a serialization point in the IO queue, forcing previously

 *	Insert a serialization point in the IO queue, forcing previously

 *	submitted IO to be completed before this one is issued.

 *	submitted IO to be completed before this one is issued.

 * bit 3 -- synchronous I/O hint: the block layer will unplug immediately

 * bit 3 -- synchronous I/O hint.

 *	Note that this does NOT indicate that the IO itself is sync, just

 * bit 4 -- Unplug the device immediately after submitting this bio.

 *	that the block layer will not postpone issue of this IO by plugging.

 * bit 5 -- metadata request

 * bit 4 -- metadata request

 *	Used for tracing to differentiate metadata and data IO. May also

 *	Used for tracing to differentiate metadata and data IO. May also

 *	get some preferential treatment in the IO scheduler

 *	get some preferential treatment in the IO scheduler

 * bit 5 -- discard sectors

 * bit 6 -- discard sectors

 *	Informs the lower level device that this range of sectors is no longer

 *	Informs the lower level device that this range of sectors is no longer

 *	used by the file system and may thus be freed by the device. Used

 *	used by the file system and may thus be freed by the device. Used

 *	for flash based storage.

 *	for flash based storage.

 * bit 6 -- fail fast device errors

 * bit 7 -- fail fast device errors

 * bit 7 -- fail fast transport errors

 * bit 8 -- fail fast transport errors

 * bit 8 -- fail fast driver errors

 * bit 9 -- fail fast driver errors

 *	Don't want driver retries for any fast fail whatever the reason.

 *	Don't want driver retries for any fast fail whatever the reason.

 * bit 10 -- Tell the IO scheduler not to wait for more requests after this

	one has been submitted, even if it is a SYNC request.

 */

 */

#define BIO_RW		0	/* Must match RW in req flags (blkdev.h) */

#define BIO_RW		0	/* Must match RW in req flags (blkdev.h) */

#define BIO_RW_AHEAD	1	/* Must match FAILFAST in req flags */

#define BIO_RW_AHEAD	1	/* Must match FAILFAST in req flags */
@@ -170,6 +171,7 @@ struct bio { 
#define BIO_RW_FAILFAST_DEV		7

#define BIO_RW_FAILFAST_DEV		7

#define BIO_RW_FAILFAST_TRANSPORT	8

#define BIO_RW_FAILFAST_TRANSPORT	8

#define BIO_RW_FAILFAST_DRIVER		9

#define BIO_RW_FAILFAST_DRIVER		9

#define BIO_RW_NOIDLE	10





#define bio_rw_flagged(bio, flag)	((bio)->bi_rw & (1 << (flag)))

#define bio_rw_flagged(bio, flag)	((bio)->bi_rw & (1 << (flag)))
@@ -188,6 +190,7 @@ struct bio { 
#define bio_rw_ahead(bio)	bio_rw_flagged(bio, BIO_RW_AHEAD)

#define bio_rw_ahead(bio)	bio_rw_flagged(bio, BIO_RW_AHEAD)

#define bio_rw_meta(bio)	bio_rw_flagged(bio, BIO_RW_META)

#define bio_rw_meta(bio)	bio_rw_flagged(bio, BIO_RW_META)

#define bio_discard(bio)	bio_rw_flagged(bio, BIO_RW_DISCARD)

#define bio_discard(bio)	bio_rw_flagged(bio, BIO_RW_DISCARD)

#define bio_noidle(bio)		bio_rw_flagged(bio, BIO_RW_NOIDLE)





/*

/*

 * upper 16 bits of bi_rw define the io priority of this bio

 * upper 16 bits of bi_rw define the io priority of this bio

include/linux/blkdev.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -118,6 +118,7 @@ enum rq_flag_bits { 
	__REQ_COPY_USER,	/* contains copies of user pages */

	__REQ_COPY_USER,	/* contains copies of user pages */

	__REQ_INTEGRITY,	/* integrity metadata has been remapped */

	__REQ_INTEGRITY,	/* integrity metadata has been remapped */

	__REQ_UNPLUG,		/* unplug queue on submission */

	__REQ_UNPLUG,		/* unplug queue on submission */

	__REQ_NOIDLE,		/* Don't anticipate more IO after this one */

	__REQ_NR_BITS,		/* stops here */

	__REQ_NR_BITS,		/* stops here */

};

};
@@ -145,6 +146,7 @@ enum rq_flag_bits { 
#define REQ_COPY_USER	(1 << __REQ_COPY_USER)

#define REQ_COPY_USER	(1 << __REQ_COPY_USER)

#define REQ_INTEGRITY	(1 << __REQ_INTEGRITY)

#define REQ_INTEGRITY	(1 << __REQ_INTEGRITY)

#define REQ_UNPLUG	(1 << __REQ_UNPLUG)

#define REQ_UNPLUG	(1 << __REQ_UNPLUG)

#define REQ_NOIDLE	(1 << __REQ_NOIDLE)





#define BLK_MAX_CDB	16

#define BLK_MAX_CDB	16
@@ -633,6 +635,7 @@ static inline bool rq_is_sync(struct request *rq) 
}

}





#define rq_is_meta(rq)		((rq)->cmd_flags & REQ_RW_META)

#define rq_is_meta(rq)		((rq)->cmd_flags & REQ_RW_META)

#define rq_noidle(rq)		((rq)->cmd_flags & REQ_NOIDLE)





static inline int blk_queue_full(struct request_queue *q, int sync)

static inline int blk_queue_full(struct request_queue *q, int sync)

{

{