Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b3b9f24f authored by Dennis Zhou (Facebook)'s avatar Dennis Zhou (Facebook) Committed by Jens Axboe
Browse files

blkcg: change blkg reference counting to use percpu_ref 



Now that every bio is associated with a blkg, this puts the use of
blkg_get, blkg_try_get, and blkg_put on the hot path. This switches over
the refcnt in blkg to use percpu_ref.

Signed-off-by: default avatarDennis Zhou <dennisszhou@gmail.com>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e2b09899

block/blk-cgroup.c

+39 −25
Original line number Diff line number Diff line
@@ -84,6 +84,37 @@ static void blkg_free(struct blkcg_gq *blkg) 
	kfree(blkg);

}



static void __blkg_release(struct rcu_head *rcu)

{

	struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);



	percpu_ref_exit(&blkg->refcnt);



	/* release the blkcg and parent blkg refs this blkg has been holding */

	css_put(&blkg->blkcg->css);

	if (blkg->parent)

		blkg_put(blkg->parent);



	wb_congested_put(blkg->wb_congested);



	blkg_free(blkg);

}



/*

 * A group is RCU protected, but having an rcu lock does not mean that one

 * can access all the fields of blkg and assume these are valid.  For

 * example, don't try to follow throtl_data and request queue links.

 *

 * Having a reference to blkg under an rcu allows accesses to only values

 * local to groups like group stats and group rate limits.

 */

static void blkg_release(struct percpu_ref *ref)

{

	struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);



	call_rcu(&blkg->rcu_head, __blkg_release);

}



/**

 * blkg_alloc - allocate a blkg

 * @blkcg: block cgroup the new blkg is associated with
@@ -110,7 +141,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, 
	blkg->q = q;

	INIT_LIST_HEAD(&blkg->q_node);

	blkg->blkcg = blkcg;

	atomic_set(&blkg->refcnt, 1);



	/* root blkg uses @q->root_rl, init rl only for !root blkgs */

	if (blkcg != &blkcg_root) {
@@ -217,6 +247,11 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, 
		blkg_get(blkg->parent);

	}



	ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,

			      GFP_NOWAIT | __GFP_NOWARN);

	if (ret)

		goto err_cancel_ref;



	/* invoke per-policy init */

	for (i = 0; i < BLKCG_MAX_POLS; i++) {

		struct blkcg_policy *pol = blkcg_policy[i];
@@ -249,6 +284,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, 
	blkg_put(blkg);

	return ERR_PTR(ret);



err_cancel_ref:

	percpu_ref_exit(&blkg->refcnt);

err_put_congested:

	wb_congested_put(wb_congested);

err_put_css:
@@ -387,7 +424,7 @@ static void blkg_destroy(struct blkcg_gq *blkg) 
	 * Put the reference taken at the time of creation so that when all

	 * queues are gone, group can be destroyed.

	 */

	blkg_put(blkg);

	percpu_ref_kill(&blkg->refcnt);

}



/**
@@ -414,29 +451,6 @@ static void blkg_destroy_all(struct request_queue *q) 
	q->root_rl.blkg = NULL;

}



/*

 * A group is RCU protected, but having an rcu lock does not mean that one

 * can access all the fields of blkg and assume these are valid.  For

 * example, don't try to follow throtl_data and request queue links.

 *

 * Having a reference to blkg under an rcu allows accesses to only values

 * local to groups like group stats and group rate limits.

 */

void __blkg_release_rcu(struct rcu_head *rcu_head)

{

	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);



	/* release the blkcg and parent blkg refs this blkg has been holding */

	css_put(&blkg->blkcg->css);

	if (blkg->parent)

		blkg_put(blkg->parent);



	wb_congested_put(blkg->wb_congested);



	blkg_free(blkg);

}

EXPORT_SYMBOL_GPL(__blkg_release_rcu);



/*

 * The next function used by blk_queue_for_each_rl().  It's a bit tricky

 * because the root blkg uses @q->root_rl instead of its own rl.

include/linux/blk-cgroup.h

+5 −10
Original line number Diff line number Diff line
@@ -126,7 +126,7 @@ struct blkcg_gq { 
	struct request_list		rl;



	/* reference count */

	atomic_t			refcnt;

	struct percpu_ref		refcnt;



	/* is this blkg online? protected by both blkcg and q locks */

	bool				online;
@@ -490,8 +490,7 @@ static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen) 
 */

static inline void blkg_get(struct blkcg_gq *blkg)

{

	WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);

	atomic_inc(&blkg->refcnt);

	percpu_ref_get(&blkg->refcnt);

}



/**
@@ -503,7 +502,7 @@ static inline void blkg_get(struct blkcg_gq *blkg) 
 */

static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg)

{

	if (atomic_inc_not_zero(&blkg->refcnt))

	if (percpu_ref_tryget(&blkg->refcnt))

		return blkg;

	return NULL;

}
@@ -517,23 +516,19 @@ static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg) 
 */

static inline struct blkcg_gq *blkg_try_get_closest(struct blkcg_gq *blkg)

{

	while (!atomic_inc_not_zero(&blkg->refcnt))

	while (!percpu_ref_tryget(&blkg->refcnt))

		blkg = blkg->parent;



	return blkg;

}



void __blkg_release_rcu(struct rcu_head *rcu);



/**

 * blkg_put - put a blkg reference

 * @blkg: blkg to put

 */

static inline void blkg_put(struct blkcg_gq *blkg)

{

	WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);

	if (atomic_dec_and_test(&blkg->refcnt))

		call_rcu(&blkg->rcu_head, __blkg_release_rcu);

	percpu_ref_put(&blkg->refcnt);

}



/**