Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5238dcf4 authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe
Browse files

block: replace bio->bi_issue_stat with bio-specific type 



struct blk_issue_stat is going away, and bio->bi_issue_stat doesn't even
use the blk-stats interface, so we can provide a separate implementation
specific for bios. The helpers work the same way as the blk-stats
helpers.

Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent a8a45941

block/blk-throttle.c

+8 −11
Original line number Diff line number Diff line
@@ -36,8 +36,6 @@ static int throtl_quantum = 32; 
 */

#define LATENCY_FILTERED_HD (1000L) /* 1ms */



#define SKIP_LATENCY (((u64)1) << BLK_STAT_RES_SHIFT)



static struct blkcg_policy blkcg_policy_throtl;



/* A workqueue to queue throttle related work */
@@ -2139,7 +2137,7 @@ static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio) 
		bio->bi_cg_private = tg;

		blkg_get(tg_to_blkg(tg));

	}

	blk_stat_set_issue(&bio->bi_issue_stat, bio_sectors(bio));

	bio_issue_init(&bio->bi_issue, bio_sectors(bio));

#endif

}
@@ -2251,7 +2249,7 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, 


#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

	if (throttled || !td->track_bio_latency)

		bio->bi_issue_stat.stat |= SKIP_LATENCY;

		bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY;

#endif

	return throttled;

}
@@ -2302,8 +2300,8 @@ void blk_throtl_bio_endio(struct bio *bio) 
	finish_time_ns = ktime_get_ns();

	tg->last_finish_time = finish_time_ns >> 10;



	start_time = blk_stat_time(&bio->bi_issue_stat) >> 10;

	finish_time = __blk_stat_time(finish_time_ns) >> 10;

	start_time = bio_issue_time(&bio->bi_issue) >> 10;

	finish_time = __bio_issue_time(finish_time_ns) >> 10;

	if (!start_time || finish_time <= start_time) {

		blkg_put(tg_to_blkg(tg));

		return;
@@ -2311,16 +2309,15 @@ void blk_throtl_bio_endio(struct bio *bio) 


	lat = finish_time - start_time;

	/* this is only for bio based driver */

	if (!(bio->bi_issue_stat.stat & SKIP_LATENCY))

		throtl_track_latency(tg->td, blk_stat_size(&bio->bi_issue_stat),

	if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY))

		throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue),

				     bio_op(bio), lat);



	if (tg->latency_target && lat >= tg->td->filtered_latency) {

		int bucket;

		unsigned int threshold;



		bucket = request_bucket_index(

			blk_stat_size(&bio->bi_issue_stat));

		bucket = request_bucket_index(bio_issue_size(&bio->bi_issue));

		threshold = tg->td->avg_buckets[rw][bucket].latency +

			tg->latency_target;

		if (lat > threshold)

include/linux/blk_types.h

+48 −1
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 


#include <linux/types.h>

#include <linux/bvec.h>

#include <linux/ktime.h>



struct bio_set;

struct bio;
@@ -94,6 +95,52 @@ struct blk_issue_stat { 
	u64 stat;

};



/*

 * From most significant bit:

 * 1 bit: reserved for other usage, see below

 * 12 bits: original size of bio

 * 51 bits: issue time of bio

 */

#define BIO_ISSUE_RES_BITS      1

#define BIO_ISSUE_SIZE_BITS     12

#define BIO_ISSUE_RES_SHIFT     (64 - BIO_ISSUE_RES_BITS)

#define BIO_ISSUE_SIZE_SHIFT    (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)

#define BIO_ISSUE_TIME_MASK     ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)

#define BIO_ISSUE_SIZE_MASK     \

	(((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)

#define BIO_ISSUE_RES_MASK      (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))



/* Reserved bit for blk-throtl */

#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)



struct bio_issue {

	u64 value;

};



static inline u64 __bio_issue_time(u64 time)

{

	return time & BIO_ISSUE_TIME_MASK;

}



static inline u64 bio_issue_time(struct bio_issue *issue)

{

	return __bio_issue_time(issue->value);

}



static inline sector_t bio_issue_size(struct bio_issue *issue)

{

	return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);

}



static inline void bio_issue_init(struct bio_issue *issue,

				       sector_t size)

{

	size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;

	issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |

			(ktime_get_ns() & BIO_ISSUE_TIME_MASK) |

			((u64)size << BIO_ISSUE_SIZE_SHIFT));

}



/*

 * main unit of I/O for the block layer and lower layers (ie drivers and

 * stacking drivers)
@@ -138,7 +185,7 @@ struct bio { 
	struct cgroup_subsys_state *bi_css;

#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

	void			*bi_cg_private;

	struct blk_issue_stat	bi_issue_stat;

	struct bio_issue	bi_issue;

#endif

#endif

	union {