Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9f060e22 authored by Kent Overstreet's avatar Kent Overstreet
Browse files

block: Convert integrity to bvec_alloc_bs() 



This adds a pointer to the bvec array to struct bio_integrity_payload,
instead of the bvecs always being inline; then the bvecs are allocated
with bvec_alloc_bs().

Changed bvec_alloc_bs() and bvec_free_bs() to take a pointer to a
mempool instead of the bioset, so that bio integrity can use a different
mempool for its bvecs, and thus avoid a potential deadlock.

This is eventually for immutable bio vecs - immutable bvecs aren't
useful if we still have to copy them, hence the need for the pointer.
Less code is always nice too, though.

Also, bio_integrity_alloc() was using fs_bio_set if no bio_set was
specified. This was wrong - using the bio_set doesn't protect us from
memory allocation failures, because we just used kmalloc for the
bio_integrity_payload. But it does introduce the possibility of
deadlock, if for some reason we weren't supposed to be using fs_bio_set.

Signed-off-by: default avatarKent Overstreet <koverstreet@google.com>
CC: Jens Axboe <axboe@kernel.dk>
CC: Martin K. Petersen <martin.petersen@oracle.com>
parent 6fda981c

fs/bio-integrity.c

+48 −84
Original line number Original line Diff line number Diff line
@@ -27,48 +27,11 @@ 
#include <linux/workqueue.h>

#include <linux/workqueue.h>

#include <linux/slab.h>

#include <linux/slab.h>





struct integrity_slab {

#define BIP_INLINE_VECS	4

	struct kmem_cache *slab;

	unsigned short nr_vecs;

	char name[8];

};



#define IS(x) { .nr_vecs = x, .name = "bip-"__stringify(x) }

struct integrity_slab bip_slab[BIOVEC_NR_POOLS] __read_mostly = {

	IS(1), IS(4), IS(16), IS(64), IS(128), IS(BIO_MAX_PAGES),

};

#undef IS





static struct kmem_cache *bip_slab;

static struct workqueue_struct *kintegrityd_wq;

static struct workqueue_struct *kintegrityd_wq;





static inline unsigned int vecs_to_idx(unsigned int nr)

{

	switch (nr) {

	case 1:

		return 0;

	case 2 ... 4:

		return 1;

	case 5 ... 16:

		return 2;

	case 17 ... 64:

		return 3;

	case 65 ... 128:

		return 4;

	case 129 ... BIO_MAX_PAGES:

		return 5;

	default:

		BUG();

	}

}



static inline int use_bip_pool(unsigned int idx)

{

	if (idx == BIOVEC_MAX_IDX)

		return 1;



	return 0;

}



/**

/**

 * bio_integrity_alloc - Allocate integrity payload and attach it to bio

 * bio_integrity_alloc - Allocate integrity payload and attach it to bio

 * @bio:	bio to attach integrity metadata to

 * @bio:	bio to attach integrity metadata to
@@ -84,38 +47,41 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 
						  unsigned int nr_vecs)

						  unsigned int nr_vecs)

{

{

	struct bio_integrity_payload *bip;

	struct bio_integrity_payload *bip;

	unsigned int idx = vecs_to_idx(nr_vecs);

	struct bio_set *bs = bio->bi_pool;

	struct bio_set *bs = bio->bi_pool;



	unsigned long idx = BIO_POOL_NONE;

	if (!bs)

	unsigned inline_vecs;

		bs = fs_bio_set;





	if (!bs) {

	BUG_ON(bio == NULL);

		bip = kmalloc(sizeof(struct bio_integrity_payload) +

	bip = NULL;

			      sizeof(struct bio_vec) * nr_vecs, gfp_mask);



		inline_vecs = nr_vecs;

	/* Lower order allocations come straight from slab */

	} else {

	if (!use_bip_pool(idx))

		bip = kmem_cache_alloc(bip_slab[idx].slab, gfp_mask);



	/* Use mempool if lower order alloc failed or max vecs were requested */

	if (bip == NULL) {

		idx = BIOVEC_MAX_IDX;  /* so we free the payload properly later */

		bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);

		bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);

		inline_vecs = BIP_INLINE_VECS;

	}





		if (unlikely(bip == NULL)) {

	if (unlikely(!bip))

			printk(KERN_ERR "%s: could not alloc bip\n", __func__);

		return NULL;

		return NULL;

		}

	}





	memset(bip, 0, sizeof(*bip));

	memset(bip, 0, sizeof(*bip));





	if (nr_vecs > inline_vecs) {

		bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,

					  bs->bvec_integrity_pool);

		if (!bip->bip_vec)

			goto err;

	} else {

		bip->bip_vec = bip->bip_inline_vecs;

	}



	bip->bip_slab = idx;

	bip->bip_slab = idx;

	bip->bip_bio = bio;

	bip->bip_bio = bio;

	bip->bip_vec = bip->bip_inline_vecs;

	bio->bi_integrity = bip;

	bio->bi_integrity = bip;





	return bip;

	return bip;

err:

	mempool_free(bip, bs->bio_integrity_pool);

	return NULL;

}

}

EXPORT_SYMBOL(bio_integrity_alloc);

EXPORT_SYMBOL(bio_integrity_alloc);
@@ -131,20 +97,20 @@ void bio_integrity_free(struct bio *bio) 
	struct bio_integrity_payload *bip = bio->bi_integrity;

	struct bio_integrity_payload *bip = bio->bi_integrity;

	struct bio_set *bs = bio->bi_pool;

	struct bio_set *bs = bio->bi_pool;





	if (!bs)

		bs = fs_bio_set;



	BUG_ON(bip == NULL);



	/* A cloned bio doesn't own the integrity metadata */

	/* A cloned bio doesn't own the integrity metadata */

	if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)

	if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)

	    && bip->bip_buf != NULL)

	    && bip->bip_buf != NULL)

		kfree(bip->bip_buf);

		kfree(bip->bip_buf);





	if (use_bip_pool(bip->bip_slab))

	if (bs) {

		if (bip->bip_slab != BIO_POOL_NONE)

			bvec_free(bs->bvec_integrity_pool, bip->bip_vec,

				  bip->bip_slab);



		mempool_free(bip, bs->bio_integrity_pool);

		mempool_free(bip, bs->bio_integrity_pool);

	else

	} else {

		kmem_cache_free(bip_slab[bip->bip_slab].slab, bip);

		kfree(bip);

	}





	bio->bi_integrity = NULL;

	bio->bi_integrity = NULL;

}

}
@@ -747,13 +713,14 @@ EXPORT_SYMBOL(bio_integrity_clone); 




int bioset_integrity_create(struct bio_set *bs, int pool_size)

int bioset_integrity_create(struct bio_set *bs, int pool_size)

{

{

	unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES);



	if (bs->bio_integrity_pool)

	if (bs->bio_integrity_pool)

		return 0;

		return 0;





	bs->bio_integrity_pool =

	bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);

		mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab);



	bs->bvec_integrity_pool = biovec_create_pool(bs, pool_size);

	if (!bs->bvec_integrity_pool)

		return -1;





	if (!bs->bio_integrity_pool)

	if (!bs->bio_integrity_pool)

		return -1;

		return -1;
@@ -766,13 +733,14 @@ void bioset_integrity_free(struct bio_set *bs) 
{

{

	if (bs->bio_integrity_pool)

	if (bs->bio_integrity_pool)

		mempool_destroy(bs->bio_integrity_pool);

		mempool_destroy(bs->bio_integrity_pool);



	if (bs->bvec_integrity_pool)

		mempool_destroy(bs->bio_integrity_pool);

}

}

EXPORT_SYMBOL(bioset_integrity_free);

EXPORT_SYMBOL(bioset_integrity_free);





void __init bio_integrity_init(void)

void __init bio_integrity_init(void)

{

{

	unsigned int i;



	/*

	/*

	 * kintegrityd won't block much but may burn a lot of CPU cycles.

	 * kintegrityd won't block much but may burn a lot of CPU cycles.

	 * Make it highpri CPU intensive wq with max concurrency of 1.

	 * Make it highpri CPU intensive wq with max concurrency of 1.
@@ -782,14 +750,10 @@ void __init bio_integrity_init(void) 
	if (!kintegrityd_wq)

	if (!kintegrityd_wq)

		panic("Failed to create kintegrityd\n");

		panic("Failed to create kintegrityd\n");





	for (i = 0 ; i < BIOVEC_NR_POOLS ; i++) {

	bip_slab = kmem_cache_create("bio_integrity_payload",

		unsigned int size;

				     sizeof(struct bio_integrity_payload) +



				     sizeof(struct bio_vec) * BIP_INLINE_VECS,

		size = sizeof(struct bio_integrity_payload)

				     0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);

			+ bip_slab[i].nr_vecs * sizeof(struct bio_vec);

	if (!bip_slab)



		panic("Failed to create slab\n");

		bip_slab[i].slab =

			kmem_cache_create(bip_slab[i].name, size, 0,

					  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);

	}

}
 
}

fs/bio.c

+15 −21
Original line number Original line Diff line number Diff line
@@ -160,12 +160,12 @@ unsigned int bvec_nr_vecs(unsigned short idx) 
	return bvec_slabs[idx].nr_vecs;

	return bvec_slabs[idx].nr_vecs;

}

}





void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx)

void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)

{

{

	BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);

	BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);





	if (idx == BIOVEC_MAX_IDX)

	if (idx == BIOVEC_MAX_IDX)

		mempool_free(bv, bs->bvec_pool);

		mempool_free(bv, pool);

	else {

	else {

		struct biovec_slab *bvs = bvec_slabs + idx;

		struct biovec_slab *bvs = bvec_slabs + idx;
@@ -173,8 +173,8 @@ void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx) 
	}

	}

}

}





struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx,

struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,

			      struct bio_set *bs)

			   mempool_t *pool)

{

{

	struct bio_vec *bvl;

	struct bio_vec *bvl;
@@ -210,7 +210,7 @@ struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, 
	 */

	 */

	if (*idx == BIOVEC_MAX_IDX) {

	if (*idx == BIOVEC_MAX_IDX) {

fallback:

fallback:

		bvl = mempool_alloc(bs->bvec_pool, gfp_mask);

		bvl = mempool_alloc(pool, gfp_mask);

	} else {

	} else {

		struct biovec_slab *bvs = bvec_slabs + *idx;

		struct biovec_slab *bvs = bvec_slabs + *idx;

		gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);

		gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
@@ -253,7 +253,7 @@ static void bio_free(struct bio *bio) 




	if (bs) {

	if (bs) {

		if (bio_has_allocated_vec(bio))

		if (bio_has_allocated_vec(bio))

			bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));

			bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));





		/*

		/*

		 * If we have front padding, adjust the bio pointer before freeing

		 * If we have front padding, adjust the bio pointer before freeing
@@ -442,11 +442,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) 
	bio_init(bio);

	bio_init(bio);





	if (nr_iovecs > inline_vecs) {

	if (nr_iovecs > inline_vecs) {

		bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);

		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);

		if (!bvl && gfp_mask != saved_gfp) {

		if (!bvl && gfp_mask != saved_gfp) {

			punt_bios_to_rescuer(bs);

			punt_bios_to_rescuer(bs);

			gfp_mask = saved_gfp;

			gfp_mask = saved_gfp;

			bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);

			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);

		}

		}





		if (unlikely(!bvl))

		if (unlikely(!bvl))
@@ -1661,20 +1661,11 @@ EXPORT_SYMBOL(bio_sector_offset); 
 * create memory pools for biovec's in a bio_set.

 * create memory pools for biovec's in a bio_set.

 * use the global biovec slabs created for general use.

 * use the global biovec slabs created for general use.

 */

 */

static int biovec_create_pools(struct bio_set *bs, int pool_entries)

mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries)

{

{

	struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;

	struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;





	bs->bvec_pool = mempool_create_slab_pool(pool_entries, bp->slab);

	return mempool_create_slab_pool(pool_entries, bp->slab);

	if (!bs->bvec_pool)

		return -ENOMEM;



	return 0;

}



static void biovec_free_pools(struct bio_set *bs)

{

	mempool_destroy(bs->bvec_pool);

}

}





void bioset_free(struct bio_set *bs)

void bioset_free(struct bio_set *bs)
@@ -1685,8 +1676,10 @@ void bioset_free(struct bio_set *bs) 
	if (bs->bio_pool)

	if (bs->bio_pool)

		mempool_destroy(bs->bio_pool);

		mempool_destroy(bs->bio_pool);





	if (bs->bvec_pool)

		mempool_destroy(bs->bvec_pool);



	bioset_integrity_free(bs);

	bioset_integrity_free(bs);

	biovec_free_pools(bs);

	bio_put_slab(bs);

	bio_put_slab(bs);





	kfree(bs);

	kfree(bs);
@@ -1731,7 +1724,8 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) 
	if (!bs->bio_pool)

	if (!bs->bio_pool)

		goto bad;

		goto bad;





	if (biovec_create_pools(bs, pool_size))

	bs->bvec_pool = biovec_create_pool(bs, pool_size);

	if (!bs->bvec_pool)

		goto bad;

		goto bad;





	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);

	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);

include/linux/bio.h

+5 −3
Original line number Original line Diff line number Diff line
@@ -213,6 +213,7 @@ extern void bio_pair_release(struct bio_pair *dbio); 




extern struct bio_set *bioset_create(unsigned int, unsigned int);

extern struct bio_set *bioset_create(unsigned int, unsigned int);

extern void bioset_free(struct bio_set *);

extern void bioset_free(struct bio_set *);

extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);





extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);

extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);

extern void bio_put(struct bio *);

extern void bio_put(struct bio *);
@@ -288,8 +289,8 @@ extern struct bio *bio_copy_user_iov(struct request_queue *, 
				     int, int, gfp_t);

				     int, int, gfp_t);

extern int bio_uncopy_user(struct bio *);

extern int bio_uncopy_user(struct bio *);

void zero_fill_bio(struct bio *bio);

void zero_fill_bio(struct bio *bio);

extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);

extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);

extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int);

extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);

extern unsigned int bvec_nr_vecs(unsigned short idx);

extern unsigned int bvec_nr_vecs(unsigned short idx);





#ifdef CONFIG_BLK_CGROUP

#ifdef CONFIG_BLK_CGROUP
@@ -511,10 +512,11 @@ struct bio_set { 
	unsigned int front_pad;

	unsigned int front_pad;





	mempool_t *bio_pool;

	mempool_t *bio_pool;

	mempool_t *bvec_pool;

#if defined(CONFIG_BLK_DEV_INTEGRITY)

#if defined(CONFIG_BLK_DEV_INTEGRITY)

	mempool_t *bio_integrity_pool;

	mempool_t *bio_integrity_pool;

	mempool_t *bvec_integrity_pool;

#endif

#endif

	mempool_t *bvec_pool;





	/*

	/*

	 * Deadlock avoidance for stacking block drivers: see comments in

	 * Deadlock avoidance for stacking block drivers: see comments in