Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 003b5c57 authored by Kent Overstreet's avatar Kent Overstreet
Browse files

block: Convert drivers to immutable biovecs 



Now that we've got a mechanism for immutable biovecs -
bi_iter.bi_bvec_done - we need to convert drivers to use primitives that
respect it instead of using the bvec array directly.

Signed-off-by: default avatarKent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: NeilBrown <neilb@suse.de>
Cc: Alasdair Kergon <agk@redhat.com>
Cc: dm-devel@redhat.com
parent 458b76ed

drivers/block/umem.c

+23 −27
Original line number Diff line number Diff line
@@ -108,8 +108,7 @@ struct cardinfo { 
				    * have been written

				    */

	struct bio	*bio, *currentbio, **biotail;

	int		current_idx;

	sector_t	current_sector;

	struct bvec_iter current_iter;



	struct request_queue *queue;
@@ -118,7 +117,7 @@ struct cardinfo { 
		struct mm_dma_desc	*desc;

		int	 		cnt, headcnt;

		struct bio		*bio, **biotail;

		int			idx;

		struct bvec_iter	iter;

	} mm_pages[2];

#define DESC_PER_PAGE ((PAGE_SIZE*2)/sizeof(struct mm_dma_desc))
@@ -344,16 +343,13 @@ static int add_bio(struct cardinfo *card) 
	dma_addr_t dma_handle;

	int offset;

	struct bio *bio;

	struct bio_vec *vec;

	int idx;

	struct bio_vec vec;

	int rw;

	int len;



	bio = card->currentbio;

	if (!bio && card->bio) {

		card->currentbio = card->bio;

		card->current_idx = card->bio->bi_iter.bi_idx;

		card->current_sector = card->bio->bi_iter.bi_sector;

		card->current_iter = card->bio->bi_iter;

		card->bio = card->bio->bi_next;

		if (card->bio == NULL)

			card->biotail = &card->bio;
@@ -362,18 +358,17 @@ static int add_bio(struct cardinfo *card) 
	}

	if (!bio)

		return 0;

	idx = card->current_idx;



	rw = bio_rw(bio);

	if (card->mm_pages[card->Ready].cnt >= DESC_PER_PAGE)

		return 0;



	vec = bio_iovec_idx(bio, idx);

	len = vec->bv_len;

	vec = bio_iter_iovec(bio, card->current_iter);



	dma_handle = pci_map_page(card->dev,

				  vec->bv_page,

				  vec->bv_offset,

				  len,

				  vec.bv_page,

				  vec.bv_offset,

				  vec.bv_len,

				  (rw == READ) ?

				  PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE);
@@ -381,7 +376,7 @@ static int add_bio(struct cardinfo *card) 
	desc = &p->desc[p->cnt];

	p->cnt++;

	if (p->bio == NULL)

		p->idx = idx;

		p->iter = card->current_iter;

	if ((p->biotail) != &bio->bi_next) {

		*(p->biotail) = bio;

		p->biotail = &(bio->bi_next);
@@ -391,8 +386,8 @@ static int add_bio(struct cardinfo *card) 
	desc->data_dma_handle = dma_handle;



	desc->pci_addr = cpu_to_le64((u64)desc->data_dma_handle);

	desc->local_addr = cpu_to_le64(card->current_sector << 9);

	desc->transfer_size = cpu_to_le32(len);

	desc->local_addr = cpu_to_le64(card->current_iter.bi_sector << 9);

	desc->transfer_size = cpu_to_le32(vec.bv_len);

	offset = (((char *)&desc->sem_control_bits) - ((char *)p->desc));

	desc->sem_addr = cpu_to_le64((u64)(p->page_dma+offset));

	desc->zero1 = desc->zero2 = 0;
@@ -407,10 +402,9 @@ static int add_bio(struct cardinfo *card) 
		desc->control_bits |= cpu_to_le32(DMASCR_TRANSFER_READ);

	desc->sem_control_bits = desc->control_bits;



	card->current_sector += (len >> 9);

	idx++;

	card->current_idx = idx;

	if (idx >= bio->bi_vcnt)



	bio_advance_iter(bio, &card->current_iter, vec.bv_len);

	if (!card->current_iter.bi_size)

		card->currentbio = NULL;



	return 1;
@@ -439,23 +433,25 @@ static void process_page(unsigned long data) 
		struct mm_dma_desc *desc = &page->desc[page->headcnt];

		int control = le32_to_cpu(desc->sem_control_bits);

		int last = 0;

		int idx;

		struct bio_vec vec;



		if (!(control & DMASCR_DMA_COMPLETE)) {

			control = dma_status;

			last = 1;

		}



		page->headcnt++;

		idx = page->idx;

		page->idx++;

		if (page->idx >= bio->bi_vcnt) {

		vec = bio_iter_iovec(bio, page->iter);

		bio_advance_iter(bio, &page->iter, vec.bv_len);



		if (!page->iter.bi_size) {

			page->bio = bio->bi_next;

			if (page->bio)

				page->idx = page->bio->bi_iter.bi_idx;

				page->iter = page->bio->bi_iter;

		}



		pci_unmap_page(card->dev, desc->data_dma_handle,

			       bio_iovec_idx(bio, idx)->bv_len,

			       vec.bv_len,

				 (control & DMASCR_TRANSFER_READ) ?

				PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);

		if (control & DMASCR_HARD_ERROR) {

drivers/md/dm-crypt.c

+18 −31
Original line number Diff line number Diff line
@@ -39,10 +39,8 @@ struct convert_context { 
	struct completion restart;

	struct bio *bio_in;

	struct bio *bio_out;

	unsigned int offset_in;

	unsigned int offset_out;

	unsigned int idx_in;

	unsigned int idx_out;

	struct bvec_iter iter_in;

	struct bvec_iter iter_out;

	sector_t cc_sector;

	atomic_t cc_pending;

};
@@ -826,10 +824,10 @@ static void crypt_convert_init(struct crypt_config *cc, 
{

	ctx->bio_in = bio_in;

	ctx->bio_out = bio_out;

	ctx->offset_in = 0;

	ctx->offset_out = 0;

	ctx->idx_in = bio_in ? bio_in->bi_iter.bi_idx : 0;

	ctx->idx_out = bio_out ? bio_out->bi_iter.bi_idx : 0;

	if (bio_in)

		ctx->iter_in = bio_in->bi_iter;

	if (bio_out)

		ctx->iter_out = bio_out->bi_iter;

	ctx->cc_sector = sector + cc->iv_offset;

	init_completion(&ctx->restart);

}
@@ -857,8 +855,8 @@ static int crypt_convert_block(struct crypt_config *cc, 
			       struct convert_context *ctx,

			       struct ablkcipher_request *req)

{

	struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);

	struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);

	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);

	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);

	struct dm_crypt_request *dmreq;

	u8 *iv;

	int r;
@@ -869,24 +867,15 @@ static int crypt_convert_block(struct crypt_config *cc, 
	dmreq->iv_sector = ctx->cc_sector;

	dmreq->ctx = ctx;

	sg_init_table(&dmreq->sg_in, 1);

	sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT,

		    bv_in->bv_offset + ctx->offset_in);

	sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT,

		    bv_in.bv_offset);



	sg_init_table(&dmreq->sg_out, 1);

	sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT,

		    bv_out->bv_offset + ctx->offset_out);

	sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT,

		    bv_out.bv_offset);



	ctx->offset_in += 1 << SECTOR_SHIFT;

	if (ctx->offset_in >= bv_in->bv_len) {

		ctx->offset_in = 0;

		ctx->idx_in++;

	}



	ctx->offset_out += 1 << SECTOR_SHIFT;

	if (ctx->offset_out >= bv_out->bv_len) {

		ctx->offset_out = 0;

		ctx->idx_out++;

	}

	bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT);

	bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT);



	if (cc->iv_gen_ops) {

		r = cc->iv_gen_ops->generator(cc, iv, dmreq);
@@ -937,8 +926,7 @@ static int crypt_convert(struct crypt_config *cc, 


	atomic_set(&ctx->cc_pending, 1);



	while(ctx->idx_in < ctx->bio_in->bi_vcnt &&

	      ctx->idx_out < ctx->bio_out->bi_vcnt) {

	while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {



		crypt_alloc_req(cc, ctx);
@@ -1207,7 +1195,7 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 
	}



	/* crypt_convert should have filled the clone bio */

	BUG_ON(io->ctx.idx_out < clone->bi_vcnt);

	BUG_ON(io->ctx.iter_out.bi_size);



	clone->bi_iter.bi_sector = cc->start + io->sector;
@@ -1246,7 +1234,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 
		}



		io->ctx.bio_out = clone;

		io->ctx.idx_out = 0;

		io->ctx.iter_out = clone->bi_iter;



		remaining -= clone->bi_iter.bi_size;

		sector += bio_sectors(clone);
@@ -1290,8 +1278,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 
			crypt_inc_pending(new_io);

			crypt_convert_init(cc, &new_io->ctx, NULL,

					   io->base_bio, sector);

			new_io->ctx.idx_in = io->ctx.idx_in;

			new_io->ctx.offset_in = io->ctx.offset_in;

			new_io->ctx.iter_in = io->ctx.iter_in;



			/*

			 * Fragments after the first use the base_io

drivers/md/dm-io.c

+17 −14
Original line number Diff line number Diff line
@@ -201,26 +201,29 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offse 
/*

 * Functions for getting the pages from a bvec.

 */

static void bvec_get_page(struct dpages *dp,

static void bio_get_page(struct dpages *dp,

		  struct page **p, unsigned long *len, unsigned *offset)

{

	struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;

	*p = bvec->bv_page;

	*len = bvec->bv_len;

	*offset = bvec->bv_offset;

	struct bio *bio = dp->context_ptr;

	struct bio_vec bvec = bio_iovec(bio);

	*p = bvec.bv_page;

	*len = bvec.bv_len;

	*offset = bvec.bv_offset;

}



static void bvec_next_page(struct dpages *dp)

static void bio_next_page(struct dpages *dp)

{

	struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;

	dp->context_ptr = bvec + 1;

	struct bio *bio = dp->context_ptr;

	struct bio_vec bvec = bio_iovec(bio);



	bio_advance(bio, bvec.bv_len);

}



static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec)

static void bio_dp_init(struct dpages *dp, struct bio *bio)

{

	dp->get_page = bvec_get_page;

	dp->next_page = bvec_next_page;

	dp->context_ptr = bvec;

	dp->get_page = bio_get_page;

	dp->next_page = bio_next_page;

	dp->context_ptr = bio;

}



/*
@@ -457,8 +460,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, 
		list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset);

		break;



	case DM_IO_BVEC:

		bvec_dp_init(dp, io_req->mem.ptr.bvec);

	case DM_IO_BIO:

		bio_dp_init(dp, io_req->mem.ptr.bio);

		break;



	case DM_IO_VMA:

drivers/md/dm-raid1.c

+4 −4
Original line number Diff line number Diff line
@@ -526,8 +526,8 @@ static void read_async_bio(struct mirror *m, struct bio *bio) 
	struct dm_io_region io;

	struct dm_io_request io_req = {

		.bi_rw = READ,

		.mem.type = DM_IO_BVEC,

		.mem.ptr.bvec = bio->bi_io_vec + bio->bi_iter.bi_idx,

		.mem.type = DM_IO_BIO,

		.mem.ptr.bio = bio,

		.notify.fn = read_callback,

		.notify.context = bio,

		.client = m->ms->io_client,
@@ -629,8 +629,8 @@ static void do_write(struct mirror_set *ms, struct bio *bio) 
	struct mirror *m;

	struct dm_io_request io_req = {

		.bi_rw = WRITE | (bio->bi_rw & WRITE_FLUSH_FUA),

		.mem.type = DM_IO_BVEC,

		.mem.ptr.bvec = bio->bi_io_vec + bio->bi_iter.bi_idx,

		.mem.type = DM_IO_BIO,

		.mem.ptr.bio = bio,

		.notify.fn = write_callback,

		.notify.context = bio,

		.client = ms->io_client,

drivers/md/dm-verity.c

+14 −38
Original line number Diff line number Diff line
@@ -73,15 +73,10 @@ struct dm_verity_io { 
	sector_t block;

	unsigned n_blocks;



	/* saved bio vector */

	struct bio_vec *io_vec;

	unsigned io_vec_size;

	struct bvec_iter iter;



	struct work_struct work;



	/* A space for short vectors; longer vectors are allocated separately. */

	struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];



	/*

	 * Three variably-size fields follow this struct:

	 *
@@ -284,9 +279,10 @@ static int verity_verify_level(struct dm_verity_io *io, sector_t block, 
static int verity_verify_io(struct dm_verity_io *io)

{

	struct dm_verity *v = io->v;

	struct bio *bio = dm_bio_from_per_bio_data(io,

						   v->ti->per_bio_data_size);

	unsigned b;

	int i;

	unsigned vector = 0, offset = 0;



	for (b = 0; b < io->n_blocks; b++) {

		struct shash_desc *desc;
@@ -336,31 +332,22 @@ static int verity_verify_io(struct dm_verity_io *io) 
		}



		todo = 1 << v->data_dev_block_bits;

		do {

			struct bio_vec *bv;

		while (io->iter.bi_size) {

			u8 *page;

			unsigned len;



			BUG_ON(vector >= io->io_vec_size);

			bv = &io->io_vec[vector];

			page = kmap_atomic(bv->bv_page);

			len = bv->bv_len - offset;

			if (likely(len >= todo))

				len = todo;

			r = crypto_shash_update(desc,

					page + bv->bv_offset + offset, len);

			struct bio_vec bv = bio_iter_iovec(bio, io->iter);



			page = kmap_atomic(bv.bv_page);

			r = crypto_shash_update(desc, page + bv.bv_offset,

						bv.bv_len);

			kunmap_atomic(page);



			if (r < 0) {

				DMERR("crypto_shash_update failed: %d", r);

				return r;

			}

			offset += len;

			if (likely(offset == bv->bv_len)) {

				offset = 0;

				vector++;



			bio_advance_iter(bio, &io->iter, bv.bv_len);

		}

			todo -= len;

		} while (todo);



		if (!v->version) {

			r = crypto_shash_update(desc, v->salt, v->salt_size);
@@ -383,8 +370,6 @@ static int verity_verify_io(struct dm_verity_io *io) 
			return -EIO;

		}

	}

	BUG_ON(vector != io->io_vec_size);

	BUG_ON(offset);



	return 0;

}
@@ -400,9 +385,6 @@ static void verity_finish_io(struct dm_verity_io *io, int error) 
	bio->bi_end_io = io->orig_bi_end_io;

	bio->bi_private = io->orig_bi_private;



	if (io->io_vec != io->io_vec_inline)

		mempool_free(io->io_vec, v->vec_mempool);



	bio_endio(bio, error);

}
@@ -519,13 +501,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio) 


	bio->bi_end_io = verity_end_io;

	bio->bi_private = io;

	io->io_vec_size = bio_segments(bio);

	if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)

		io->io_vec = io->io_vec_inline;

	else

		io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);

	memcpy(io->io_vec, __bio_iovec(bio),

	       io->io_vec_size * sizeof(struct bio_vec));

	io->iter = bio->bi_iter;



	verity_submit_prefetch(v, io);