Merge tag 'for-linus-20181102' of git://git.kernel.dk/linux-block

  wbc_init_bio(@wbc, @bio)

	Should be called for each bio carrying writeback data and

	associates the bio with the inode's owner cgroup and the

	corresponding request queue.  This must be called after

	a queue (device) has been associated with the bio and

	before submission.

	associates the bio with the inode's owner cgroup.  Can be

	called anytime between bio allocation and submission.

  wbc_account_io(@wbc, @page, @bytes)

	Should be called for each data segment being written out.

the writeback session is holding shared resources, e.g. a journal

entry, may lead to priority inversion.  There is no one easy solution

for the problem.  Filesystems can try to work around specific problem

cases by skipping wbc_init_bio() or using bio_associate_create_blkg()

cases by skipping wbc_init_bio() or using bio_associate_blkcg()

directly.

	uint64_t serial_nr;

	rcu_read_lock();

	serial_nr = __bio_blkcg(bio)->css.serial_nr;

	serial_nr = bio_blkcg(bio)->css.serial_nr;

	/*

	 * Check whether blkcg has changed.  The condition may trigger

	if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))

		goto out;

	bfqg = __bfq_bic_change_cgroup(bfqd, bic, __bio_blkcg(bio));

	bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio));

	/*

	 * Update blkg_path for bfq_log_* functions. We cache this

	 * path, and update it here, for the following

	rcu_read_lock();

	bfqg = bfq_find_set_group(bfqd, __bio_blkcg(bio));

	bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio));

	if (!bfqg) {

		bfqq = &bfqd->oom_bfqq;

		goto out;

	bio->bi_iter = bio_src->bi_iter;

	bio->bi_io_vec = bio_src->bi_io_vec;

	bio_clone_blkg_association(bio, bio_src);

	blkcg_bio_issue_init(bio);

	bio_clone_blkcg_association(bio, bio_src);

}

EXPORT_SYMBOL(__bio_clone_fast);

#ifdef CONFIG_BLK_CGROUP

/**

 * bio_associate_blkg - associate a bio with the a blkg

 * @bio: target bio

 * @blkg: the blkg to associate

 *

 * This tries to associate @bio with the specified blkg.  Association failure

 * is handled by walking up the blkg tree.  Therefore, the blkg associated can

 * be anything between @blkg and the root_blkg.  This situation only happens

 * when a cgroup is dying and then the remaining bios will spill to the closest

 * alive blkg.

 *

 * A reference will be taken on the @blkg and will be released when @bio is

 * freed.

 */

int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)

{

	if (unlikely(bio->bi_blkg))

		return -EBUSY;

	bio->bi_blkg = blkg_tryget_closest(blkg);

	return 0;

}

/**

 * __bio_associate_blkg_from_css - internal blkg association function

 *

 * This in the core association function that all association paths rely on.

 * A blkg reference is taken which is released upon freeing of the bio.

 */

static int __bio_associate_blkg_from_css(struct bio *bio,

					 struct cgroup_subsys_state *css)

{

	struct request_queue *q = bio->bi_disk->queue;

	struct blkcg_gq *blkg;

	int ret;

	rcu_read_lock();

	if (!css || !css->parent)

		blkg = q->root_blkg;

	else

		blkg = blkg_lookup_create(css_to_blkcg(css), q);

	ret = bio_associate_blkg(bio, blkg);

	rcu_read_unlock();

	return ret;

}

/**

 * bio_associate_blkg_from_css - associate a bio with a specified css

 * @bio: target bio

 * @css: target css

 *

 * Associate @bio with the blkg found by combining the css's blkg and the

 * request_queue of the @bio.  This falls back to the queue's root_blkg if

 * the association fails with the css.

 */

int bio_associate_blkg_from_css(struct bio *bio,

				struct cgroup_subsys_state *css)

{

	if (unlikely(bio->bi_blkg))

		return -EBUSY;

	return __bio_associate_blkg_from_css(bio, css);

}

EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);

#ifdef CONFIG_MEMCG

/**

 * bio_associate_blkg_from_page - associate a bio with the page's blkg

 * bio_associate_blkcg_from_page - associate a bio with the page's blkcg

 * @bio: target bio

 * @page: the page to lookup the blkcg from

 *

 * Associate @bio with the blkg from @page's owning memcg and the respective

 * request_queue.  If cgroup_e_css returns NULL, fall back to the queue's

 * root_blkg.

 *

 * Note: this must be called after bio has an associated device.

 * Associate @bio with the blkcg from @page's owning memcg.  This works like

 * every other associate function wrt references.

 */

int bio_associate_blkg_from_page(struct bio *bio, struct page *page)

int bio_associate_blkcg_from_page(struct bio *bio, struct page *page)

{

	struct cgroup_subsys_state *css;

	int ret;

	struct cgroup_subsys_state *blkcg_css;

	if (unlikely(bio->bi_blkg))

	if (unlikely(bio->bi_css))

		return -EBUSY;

	if (!page->mem_cgroup)

		return 0;

	rcu_read_lock();

	css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);

	ret = __bio_associate_blkg_from_css(bio, css);

	rcu_read_unlock();

	return ret;

	blkcg_css = cgroup_get_e_css(page->mem_cgroup->css.cgroup,

				     &io_cgrp_subsys);

	bio->bi_css = blkcg_css;

	return 0;

}

#endif /* CONFIG_MEMCG */

/**

 * bio_associate_create_blkg - associate a bio with a blkg from q

 * @q: request_queue where bio is going

 * bio_associate_blkcg - associate a bio with the specified blkcg

 * @bio: target bio

 * @blkcg_css: css of the blkcg to associate

 *

 * Associate @bio with the blkg found from the bio's css and the request_queue.

 * If one is not found, bio_lookup_blkg creates the blkg.  This falls back to

 * the queue's root_blkg if association fails.

 * Associate @bio with the blkcg specified by @blkcg_css.  Block layer will

 * treat @bio as if it were issued by a task which belongs to the blkcg.

 *

 * This function takes an extra reference of @blkcg_css which will be put

 * when @bio is released.  The caller must own @bio and is responsible for

 * synchronizing calls to this function.

 */

int bio_associate_create_blkg(struct request_queue *q, struct bio *bio)

int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css)

{

	struct cgroup_subsys_state *css;

	int ret = 0;

	/* someone has already associated this bio with a blkg */

	if (bio->bi_blkg)

		return ret;

	rcu_read_lock();

	css = blkcg_css();

	ret = __bio_associate_blkg_from_css(bio, css);

	rcu_read_unlock();

	return ret;

	if (unlikely(bio->bi_css))

		return -EBUSY;

	css_get(blkcg_css);

	bio->bi_css = blkcg_css;

	return 0;

}

EXPORT_SYMBOL_GPL(bio_associate_blkcg);

/**

 * bio_reassociate_blkg - reassociate a bio with a blkg from q

 * @q: request_queue where bio is going

 * bio_associate_blkg - associate a bio with the specified blkg

 * @bio: target bio

 * @blkg: the blkg to associate

 *

 * When submitting a bio, multiple recursive calls to make_request() may occur.

 * This causes the initial associate done in blkcg_bio_issue_check() to be

 * incorrect and reference the prior request_queue.  This performs reassociation

 * when this situation happens.

 * Associate @bio with the blkg specified by @blkg.  This is the queue specific

 * blkcg information associated with the @bio, a reference will be taken on the

 * @blkg and will be freed when the bio is freed.

 */

int bio_reassociate_blkg(struct request_queue *q, struct bio *bio)

int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)

{

	if (bio->bi_blkg) {

		blkg_put(bio->bi_blkg);

		bio->bi_blkg = NULL;

	}

	return bio_associate_create_blkg(q, bio);

	if (unlikely(bio->bi_blkg))

		return -EBUSY;

	if (!blkg_try_get(blkg))

		return -ENODEV;

	bio->bi_blkg = blkg;

	return 0;

}

/**

		put_io_context(bio->bi_ioc);

		bio->bi_ioc = NULL;

	}

	if (bio->bi_css) {

		css_put(bio->bi_css);

		bio->bi_css = NULL;

	}

	if (bio->bi_blkg) {

		blkg_put(bio->bi_blkg);

		bio->bi_blkg = NULL;

}

/**

 * bio_clone_blkg_association - clone blkg association from src to dst bio

 * bio_clone_blkcg_association - clone blkcg association from src to dst bio

 * @dst: destination bio

 * @src: source bio

 */

void bio_clone_blkg_association(struct bio *dst, struct bio *src)

void bio_clone_blkcg_association(struct bio *dst, struct bio *src)

{

	if (src->bi_blkg)

		bio_associate_blkg(dst, src->bi_blkg);

	if (src->bi_css)

		WARN_ON(bio_associate_blkcg(dst, src->bi_css));

}

EXPORT_SYMBOL_GPL(bio_clone_blkg_association);

EXPORT_SYMBOL_GPL(bio_clone_blkcg_association);

#endif /* CONFIG_BLK_CGROUP */

static void __init biovec_init_slabs(void)

	kfree(blkg);

}

static void __blkg_release(struct rcu_head *rcu)

{

	struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);

	percpu_ref_exit(&blkg->refcnt);

	/* release the blkcg and parent blkg refs this blkg has been holding */

	css_put(&blkg->blkcg->css);

	if (blkg->parent)

		blkg_put(blkg->parent);

	wb_congested_put(blkg->wb_congested);

	blkg_free(blkg);

}

/*

 * A group is RCU protected, but having an rcu lock does not mean that one

 * can access all the fields of blkg and assume these are valid.  For

 * example, don't try to follow throtl_data and request queue links.

 *

 * Having a reference to blkg under an rcu allows accesses to only values

 * local to groups like group stats and group rate limits.

 */

static void blkg_release(struct percpu_ref *ref)

{

	struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);

	call_rcu(&blkg->rcu_head, __blkg_release);

}

/**

 * blkg_alloc - allocate a blkg

 * @blkcg: block cgroup the new blkg is associated with

	blkg->q = q;

	INIT_LIST_HEAD(&blkg->q_node);

	blkg->blkcg = blkcg;

	atomic_set(&blkg->refcnt, 1);

	/* root blkg uses @q->root_rl, init rl only for !root blkgs */

	if (blkcg != &blkcg_root) {

		blkg_get(blkg->parent);

	}

	ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,

			      GFP_NOWAIT | __GFP_NOWARN);

	if (ret)

		goto err_cancel_ref;

	/* invoke per-policy init */

	for (i = 0; i < BLKCG_MAX_POLS; i++) {

		struct blkcg_policy *pol = blkcg_policy[i];

	blkg_put(blkg);

	return ERR_PTR(ret);

err_cancel_ref:

	percpu_ref_exit(&blkg->refcnt);

err_put_congested:

	wb_congested_put(wb_congested);

err_put_css:

}

/**

 * __blkg_lookup_create - lookup blkg, try to create one if not there

 * blkg_lookup_create - lookup blkg, try to create one if not there

 * @blkcg: blkcg of interest

 * @q: request_queue of interest

 *

 * that all non-root blkg's have access to the parent blkg.  This function

 * should be called under RCU read lock and @q->queue_lock.

 *

 * Returns the blkg or the closest blkg if blkg_create fails as it walks

 * down from root.

 * Returns pointer to the looked up or created blkg on success, ERR_PTR()

 * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not

 * dead and bypassing, returns ERR_PTR(-EBUSY).

 */

struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,

struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,

				    struct request_queue *q)

{

	struct blkcg_gq *blkg;

	 * we shouldn't allow anything to go through for a bypassing queue.

	 */

	if (unlikely(blk_queue_bypass(q)))

		return q->root_blkg;

		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);

	blkg = __blkg_lookup(blkcg, q, true);

	if (blkg)

	/*

	 * Create blkgs walking down from blkcg_root to @blkcg, so that all

	 * non-root blkgs have access to their parents.  Returns the closest

	 * blkg to the intended blkg should blkg_create() fail.

	 * non-root blkgs have access to their parents.

	 */

	while (true) {

		struct blkcg *pos = blkcg;

		struct blkcg *parent = blkcg_parent(blkcg);

		struct blkcg_gq *ret_blkg = q->root_blkg;

		while (parent) {

			blkg = __blkg_lookup(parent, q, false);

			if (blkg) {

				/* remember closest blkg */

				ret_blkg = blkg;

				break;

			}

		while (parent && !__blkg_lookup(parent, q, false)) {

			pos = parent;

			parent = blkcg_parent(parent);

		}

		blkg = blkg_create(pos, q, NULL);

		if (IS_ERR(blkg))

			return ret_blkg;

		if (pos == blkcg)

		if (pos == blkcg || IS_ERR(blkg))

			return blkg;

	}

}

/**

 * blkg_lookup_create - find or create a blkg

 * @blkcg: target block cgroup

 * @q: target request_queue

 *

 * This looks up or creates the blkg representing the unique pair

 * of the blkcg and the request_queue.

 */

struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,

				    struct request_queue *q)

{

	struct blkcg_gq *blkg = blkg_lookup(blkcg, q);

	unsigned long flags;

	if (unlikely(!blkg)) {

		spin_lock_irqsave(q->queue_lock, flags);

		blkg = __blkg_lookup_create(blkcg, q);

		spin_unlock_irqrestore(q->queue_lock, flags);

	}

	return blkg;

}

static void blkg_destroy(struct blkcg_gq *blkg)

{

	struct blkcg *blkcg = blkg->blkcg;

	 * Put the reference taken at the time of creation so that when all

	 * queues are gone, group can be destroyed.

	 */

	percpu_ref_kill(&blkg->refcnt);

	blkg_put(blkg);

}

/**

	q->root_rl.blkg = NULL;

}

/*

 * A group is RCU protected, but having an rcu lock does not mean that one

 * can access all the fields of blkg and assume these are valid.  For

 * example, don't try to follow throtl_data and request queue links.

 *

 * Having a reference to blkg under an rcu allows accesses to only values

 * local to groups like group stats and group rate limits.

 */

void __blkg_release_rcu(struct rcu_head *rcu_head)

{

	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);

	/* release the blkcg and parent blkg refs this blkg has been holding */

	css_put(&blkg->blkcg->css);

	if (blkg->parent)

		blkg_put(blkg->parent);

	wb_congested_put(blkg->wb_congested);

	blkg_free(blkg);

}

EXPORT_SYMBOL_GPL(__blkg_release_rcu);

/*

 * The next function used by blk_queue_for_each_rl().  It's a bit tricky

 * because the root blkg uses @q->root_rl instead of its own rl.

	blkg = blkg_lookup(blkcg, q);

	if (!blkg)

		goto out;

	if (!blkg_tryget(blkg))

	blkg = blkg_try_get(blkg);

	if (!blkg)

		goto out;

	rcu_read_unlock();