Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 544ccc8d authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe
Browse files

block: get rid of struct blk_issue_stat 



struct blk_issue_stat squashes three things into one u64:

- The time the driver started working on a request
- The original size of the request (for the io.low controller)
- Flags for writeback throttling

It turns out that on x86_64, we have a 4 byte hole in struct request
which we can fill with the non-timestamp fields from blk_issue_stat,
simplifying things quite a bit.

Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5238dcf4

block/blk-core.c

+4 −1
Original line number Diff line number Diff line
@@ -2991,7 +2991,10 @@ void blk_start_request(struct request *req) 
	blk_dequeue_request(req);



	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {

		blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req));

		req->io_start_time_ns = ktime_get_ns();

#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

		req->throtl_size = blk_rq_sectors(req);

#endif

		req->rq_flags |= RQF_STATS;

		wbt_issue(req->q->rq_wb, req);

	}

block/blk-mq.c

+6 −2
Original line number Diff line number Diff line
@@ -310,6 +310,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 
	rq->rq_disk = NULL;

	rq->part = NULL;

	rq->start_time = jiffies;

	rq->io_start_time_ns = 0;

	rq->nr_phys_segments = 0;

#if defined(CONFIG_BLK_DEV_INTEGRITY)

	rq->nr_integrity_segments = 0;
@@ -329,7 +330,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 
#ifdef CONFIG_BLK_CGROUP

	rq->rl = NULL;

	set_start_time_ns(rq);

	rq->io_start_time_ns = 0;

	rq->cgroup_io_start_time_ns = 0;

#endif



	data->ctx->rq_dispatched[op_is_sync(op)]++;
@@ -669,7 +670,10 @@ void blk_mq_start_request(struct request *rq) 
	trace_block_rq_issue(q, rq);



	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {

		blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));

		rq->io_start_time_ns = ktime_get_ns();

#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

		rq->throtl_size = blk_rq_sectors(rq);

#endif

		rq->rq_flags |= RQF_STATS;

		wbt_issue(q->rq_wb, rq);

	}

block/blk-stat.c

+2 −5
Original line number Diff line number Diff line
@@ -55,11 +55,8 @@ void blk_stat_add(struct request *rq) 
	int bucket;

	u64 now, value;



	now = __blk_stat_time(ktime_to_ns(ktime_get()));

	if (now < blk_stat_time(&rq->issue_stat))

		return;



	value = now - blk_stat_time(&rq->issue_stat);

	now = ktime_get_ns();

	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;



	blk_throtl_stat_add(rq, value);

block/blk-stat.h

+0 −43
Original line number Diff line number Diff line
@@ -8,21 +8,6 @@ 
#include <linux/rcupdate.h>

#include <linux/timer.h>



/*

 * from upper:

 * 4 bits: reserved for other usage

 * 12 bits: size

 * 48 bits: time

 */

#define BLK_STAT_RES_BITS	4

#define BLK_STAT_SIZE_BITS	12

#define BLK_STAT_RES_SHIFT	(64 - BLK_STAT_RES_BITS)

#define BLK_STAT_SIZE_SHIFT	(BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)

#define BLK_STAT_TIME_MASK	((1ULL << BLK_STAT_SIZE_SHIFT) - 1)

#define BLK_STAT_SIZE_MASK	\

	(((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)

#define BLK_STAT_RES_MASK	(~((1ULL << BLK_STAT_RES_SHIFT) - 1))



/**

 * struct blk_stat_callback - Block statistics callback.

 *
@@ -82,34 +67,6 @@ void blk_free_queue_stats(struct blk_queue_stats *); 


void blk_stat_add(struct request *);



static inline u64 __blk_stat_time(u64 time)

{

	return time & BLK_STAT_TIME_MASK;

}



static inline u64 blk_stat_time(struct blk_issue_stat *stat)

{

	return __blk_stat_time(stat->stat);

}



static inline sector_t blk_capped_size(sector_t size)

{

	return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);

}



static inline sector_t blk_stat_size(struct blk_issue_stat *stat)

{

	return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;

}



static inline void blk_stat_set_issue(struct blk_issue_stat *stat,

	sector_t size)

{

	stat->stat = (stat->stat & BLK_STAT_RES_MASK) |

		(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |

		(((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);

}



/* record time/size info in request but not add a callback */

void blk_stat_enable_accounting(struct request_queue *q);

block/blk-throttle.c

+1 −2
Original line number Diff line number Diff line
@@ -2279,8 +2279,7 @@ void blk_throtl_stat_add(struct request *rq, u64 time_ns) 
	struct request_queue *q = rq->q;

	struct throtl_data *td = q->td;



	throtl_track_latency(td, blk_stat_size(&rq->issue_stat),

		req_op(rq), time_ns >> 10);

	throtl_track_latency(td, rq->throtl_size, req_op(rq), time_ns >> 10);

}



void blk_throtl_bio_endio(struct bio *bio)