Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ef295ecf authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: better op and flags encoding 



Now that we don't need the common flags to overflow outside the range
of a 32-bit type we can encode them the same way for both the bio and
request fields.  This in addition allows us to place the operation
first (and make some room for more ops while we're at it) and to
stop having to shift around the operation values.

In addition this allows passing around only one value in the block layer
instead of two (and eventuall also in the file systems, but we can do
that later) and thus clean up a lot of code.

Last but not least this allows decreasing the size of the cmd_flags
field in struct request to 32-bits.  Various functions passing this
value could also be updated, but I'd like to avoid the churn for now.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent e8064021

Documentation/block/biodoc.txt

+2 −2
Original line number Original line Diff line number Diff line
@@ -553,8 +553,8 @@ struct request { 
	struct request_list *rl;

	struct request_list *rl;

}

}

	

	

See the rq_flag_bits definitions for an explanation of the various flags

See the req_ops and req_flag_bits definitions for an explanation of the various

available. Some bits are used by the block layer or i/o scheduler.

flags available. Some bits are used by the block layer or i/o scheduler.

	

	

The behaviour of the various sector counts are almost the same as before,

The behaviour of the various sector counts are almost the same as before,

except that since we have multi-segment bios, current_nr_sectors refers

except that since we have multi-segment bios, current_nr_sectors refers

block/blk-core.c

+20 −40
Original line number Original line Diff line number Diff line
@@ -1056,8 +1056,7 @@ static struct io_context *rq_ioc(struct bio *bio) 
/**

/**

 * __get_request - get a free request

 * __get_request - get a free request

 * @rl: request list to allocate from

 * @rl: request list to allocate from

 * @op: REQ_OP_READ/REQ_OP_WRITE

 * @op: operation and flags

 * @op_flags: rq_flag_bits

 * @bio: bio to allocate request for (can be %NULL)

 * @bio: bio to allocate request for (can be %NULL)

 * @gfp_mask: allocation mask

 * @gfp_mask: allocation mask

 *

 *
@@ -1068,23 +1067,22 @@ static struct io_context *rq_ioc(struct bio *bio) 
 * Returns ERR_PTR on failure, with @q->queue_lock held.

 * Returns ERR_PTR on failure, with @q->queue_lock held.

 * Returns request pointer on success, with @q->queue_lock *not held*.

 * Returns request pointer on success, with @q->queue_lock *not held*.

 */

 */

static struct request *__get_request(struct request_list *rl, int op,

static struct request *__get_request(struct request_list *rl, unsigned int op,

				     int op_flags, struct bio *bio,

		struct bio *bio, gfp_t gfp_mask)

				     gfp_t gfp_mask)

{

{

	struct request_queue *q = rl->q;

	struct request_queue *q = rl->q;

	struct request *rq;

	struct request *rq;

	struct elevator_type *et = q->elevator->type;

	struct elevator_type *et = q->elevator->type;

	struct io_context *ioc = rq_ioc(bio);

	struct io_context *ioc = rq_ioc(bio);

	struct io_cq *icq = NULL;

	struct io_cq *icq = NULL;

	const bool is_sync = rw_is_sync(op, op_flags) != 0;

	const bool is_sync = op_is_sync(op);

	int may_queue;

	int may_queue;

	req_flags_t rq_flags = RQF_ALLOCED;

	req_flags_t rq_flags = RQF_ALLOCED;





	if (unlikely(blk_queue_dying(q)))

	if (unlikely(blk_queue_dying(q)))

		return ERR_PTR(-ENODEV);

		return ERR_PTR(-ENODEV);





	may_queue = elv_may_queue(q, op, op_flags);

	may_queue = elv_may_queue(q, op);

	if (may_queue == ELV_MQUEUE_NO)

	if (may_queue == ELV_MQUEUE_NO)

		goto rq_starved;

		goto rq_starved;
@@ -1154,7 +1152,7 @@ static struct request *__get_request(struct request_list *rl, int op, 




	blk_rq_init(q, rq);

	blk_rq_init(q, rq);

	blk_rq_set_rl(rq, rl);

	blk_rq_set_rl(rq, rl);

	req_set_op_attrs(rq, op, op_flags);

	rq->cmd_flags = op;

	rq->rq_flags = rq_flags;

	rq->rq_flags = rq_flags;





	/* init elvpriv */

	/* init elvpriv */
@@ -1232,8 +1230,7 @@ static struct request *__get_request(struct request_list *rl, int op, 
/**

/**

 * get_request - get a free request

 * get_request - get a free request

 * @q: request_queue to allocate request from

 * @q: request_queue to allocate request from

 * @op: REQ_OP_READ/REQ_OP_WRITE

 * @op: operation and flags

 * @op_flags: rq_flag_bits

 * @bio: bio to allocate request for (can be %NULL)

 * @bio: bio to allocate request for (can be %NULL)

 * @gfp_mask: allocation mask

 * @gfp_mask: allocation mask

 *

 *
@@ -1244,18 +1241,17 @@ static struct request *__get_request(struct request_list *rl, int op, 
 * Returns ERR_PTR on failure, with @q->queue_lock held.

 * Returns ERR_PTR on failure, with @q->queue_lock held.

 * Returns request pointer on success, with @q->queue_lock *not held*.

 * Returns request pointer on success, with @q->queue_lock *not held*.

 */

 */

static struct request *get_request(struct request_queue *q, int op,

static struct request *get_request(struct request_queue *q, unsigned int op,

				   int op_flags, struct bio *bio,

		struct bio *bio, gfp_t gfp_mask)

				   gfp_t gfp_mask)

{

{

	const bool is_sync = rw_is_sync(op, op_flags) != 0;

	const bool is_sync = op_is_sync(op);

	DEFINE_WAIT(wait);

	DEFINE_WAIT(wait);

	struct request_list *rl;

	struct request_list *rl;

	struct request *rq;

	struct request *rq;





	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */

	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */

retry:

retry:

	rq = __get_request(rl, op, op_flags, bio, gfp_mask);

	rq = __get_request(rl, op, bio, gfp_mask);

	if (!IS_ERR(rq))

	if (!IS_ERR(rq))

		return rq;

		return rq;
@@ -1297,7 +1293,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, 
	create_io_context(gfp_mask, q->node);

	create_io_context(gfp_mask, q->node);





	spin_lock_irq(q->queue_lock);

	spin_lock_irq(q->queue_lock);

	rq = get_request(q, rw, 0, NULL, gfp_mask);

	rq = get_request(q, rw, NULL, gfp_mask);

	if (IS_ERR(rq)) {

	if (IS_ERR(rq)) {

		spin_unlock_irq(q->queue_lock);

		spin_unlock_irq(q->queue_lock);

		return rq;

		return rq;
@@ -1446,7 +1442,7 @@ void __blk_put_request(struct request_queue *q, struct request *req) 
	 */

	 */

	if (rq_flags & RQF_ALLOCED) {

	if (rq_flags & RQF_ALLOCED) {

		struct request_list *rl = blk_rq_rl(req);

		struct request_list *rl = blk_rq_rl(req);

		bool sync = rw_is_sync(req_op(req), req->cmd_flags);

		bool sync = op_is_sync(req->cmd_flags);





		BUG_ON(!list_empty(&req->queuelist));

		BUG_ON(!list_empty(&req->queuelist));

		BUG_ON(ELV_ON_HASH(req));

		BUG_ON(ELV_ON_HASH(req));
@@ -1652,8 +1648,6 @@ unsigned int blk_plug_queued_count(struct request_queue *q) 
void init_request_from_bio(struct request *req, struct bio *bio)

void init_request_from_bio(struct request *req, struct bio *bio)

{

{

	req->cmd_type = REQ_TYPE_FS;

	req->cmd_type = REQ_TYPE_FS;



	req->cmd_flags |= bio->bi_opf & REQ_COMMON_MASK;

	if (bio->bi_opf & REQ_RAHEAD)

	if (bio->bi_opf & REQ_RAHEAD)

		req->cmd_flags |= REQ_FAILFAST_MASK;

		req->cmd_flags |= REQ_FAILFAST_MASK;
@@ -1665,9 +1659,8 @@ void init_request_from_bio(struct request *req, struct bio *bio) 




static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)

static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)

{

{

	const bool sync = !!(bio->bi_opf & REQ_SYNC);

	struct blk_plug *plug;

	struct blk_plug *plug;

	int el_ret, rw_flags = 0, where = ELEVATOR_INSERT_SORT;

	int el_ret, where = ELEVATOR_INSERT_SORT;

	struct request *req;

	struct request *req;

	unsigned int request_count = 0;

	unsigned int request_count = 0;
@@ -1722,24 +1715,11 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) 
	}

	}





get_rq:

get_rq:

	/*

	 * This sync check and mask will be re-done in init_request_from_bio(),

	 * but we need to set it earlier to expose the sync flag to the

	 * rq allocator and io schedulers.

	 */

	if (sync)

		rw_flags |= REQ_SYNC;



	/*

	 * Add in META/PRIO flags, if set, before we get to the IO scheduler

	 */

	rw_flags |= (bio->bi_opf & (REQ_META | REQ_PRIO));



	/*

	/*

	 * Grab a free request. This is might sleep but can not fail.

	 * Grab a free request. This is might sleep but can not fail.

	 * Returns with the queue unlocked.

	 * Returns with the queue unlocked.

	 */

	 */

	req = get_request(q, bio_data_dir(bio), rw_flags, bio, GFP_NOIO);

	req = get_request(q, bio->bi_opf, bio, GFP_NOIO);

	if (IS_ERR(req)) {

	if (IS_ERR(req)) {

		bio->bi_error = PTR_ERR(req);

		bio->bi_error = PTR_ERR(req);

		bio_endio(bio);

		bio_endio(bio);
@@ -2946,8 +2926,6 @@ EXPORT_SYMBOL_GPL(__blk_end_request_err); 
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,

void blk_rq_bio_prep(struct request_queue *q, struct request *rq,

		     struct bio *bio)

		     struct bio *bio)

{

{

	req_set_op(rq, bio_op(bio));



	if (bio_has_data(bio))

	if (bio_has_data(bio))

		rq->nr_phys_segments = bio_phys_segments(q, bio);

		rq->nr_phys_segments = bio_phys_segments(q, bio);
@@ -3031,8 +3009,7 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 
static void __blk_rq_prep_clone(struct request *dst, struct request *src)

static void __blk_rq_prep_clone(struct request *dst, struct request *src)

{

{

	dst->cpu = src->cpu;

	dst->cpu = src->cpu;

	req_set_op_attrs(dst, req_op(src),

	dst->cmd_flags = src->cmd_flags | REQ_NOMERGE;

			 (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE);

	dst->cmd_type = src->cmd_type;

	dst->cmd_type = src->cmd_type;

	dst->__sector = blk_rq_pos(src);

	dst->__sector = blk_rq_pos(src);

	dst->__data_len = blk_rq_bytes(src);

	dst->__data_len = blk_rq_bytes(src);
@@ -3537,8 +3514,11 @@ EXPORT_SYMBOL(blk_set_runtime_active); 




int __init blk_dev_init(void)

int __init blk_dev_init(void)

{

{

	BUILD_BUG_ON(__REQ_NR_BITS > 8 *

	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));

	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *

			FIELD_SIZEOF(struct request, cmd_flags));

			FIELD_SIZEOF(struct request, cmd_flags));

	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *

			FIELD_SIZEOF(struct bio, bi_opf));





	/* used for unplugging and affects IO latency/throughput - HIGHPRI */

	/* used for unplugging and affects IO latency/throughput - HIGHPRI */

	kblockd_workqueue = alloc_workqueue("kblockd",

	kblockd_workqueue = alloc_workqueue("kblockd",

block/blk-flush.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -330,7 +330,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) 
	}

	}





	flush_rq->cmd_type = REQ_TYPE_FS;

	flush_rq->cmd_type = REQ_TYPE_FS;

	req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH);

	flush_rq->cmd_flags = REQ_OP_FLUSH | WRITE_FLUSH;

	flush_rq->rq_flags |= RQF_FLUSH_SEQ;

	flush_rq->rq_flags |= RQF_FLUSH_SEQ;

	flush_rq->rq_disk = first_rq->rq_disk;

	flush_rq->rq_disk = first_rq->rq_disk;

	flush_rq->end_io = flush_end_io;

	flush_rq->end_io = flush_end_io;

block/blk-lib.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -29,7 +29,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 
	struct request_queue *q = bdev_get_queue(bdev);

	struct request_queue *q = bdev_get_queue(bdev);

	struct bio *bio = *biop;

	struct bio *bio = *biop;

	unsigned int granularity;

	unsigned int granularity;

	enum req_op op;

	unsigned int op;

	int alignment;

	int alignment;

	sector_t bs_mask;

	sector_t bs_mask;

block/blk-map.c

+2 −0
Original line number Original line Diff line number Diff line
@@ -16,6 +16,8 @@ 
int blk_rq_append_bio(struct request *rq, struct bio *bio)

int blk_rq_append_bio(struct request *rq, struct bio *bio)

{

{

	if (!rq->bio) {

	if (!rq->bio) {

		rq->cmd_flags &= REQ_OP_MASK;

		rq->cmd_flags |= (bio->bi_opf & REQ_OP_MASK);

		blk_rq_bio_prep(rq->q, rq, bio);

		blk_rq_bio_prep(rq->q, rq, bio);

	} else {

	} else {

		if (!ll_back_merge_fn(rq->q, rq, bio))

		if (!ll_back_merge_fn(rq->q, rq, bio))