pnfs/blocklayout: remove read-modify-write handling in bl_write_pagelist (3a6fd1f0) · Commits · e / devices / android_kernel_sony_msm8998

fs/nfs/blocklayout/blocklayout.c

+63 −435

Original line number	Diff line number	Diff line
		@@ -35,7 +35,6 @@
		#include <linux/mount.h>
		#include <linux/namei.h>
		#include <linux/bio.h> /* struct bio */
		#include <linux/buffer_head.h> /* various write calls */
		#include <linux/prefetch.h>
		#include <linux/pagevec.h>

		@@ -188,16 +187,6 @@ retry:
		return bio;
		}

		static struct bio bl_add_page_to_bio(struct bio bio, int npg, int rw,
		sector_t isect, struct page *page,
		struct pnfs_block_extent *be,
		void (end_io)(struct bio , int err),
		struct parallel_io *par)
		{
		return do_add_page_to_bio(bio, npg, rw, isect, page, be,
		end_io, par, 0, PAGE_CACHE_SIZE);
		}

		/* This is basically copied from mpage_end_io_read */
		static void bl_end_io_read(struct bio *bio, int err)
		{
		@@ -293,8 +282,8 @@ bl_read_pagelist(struct nfs_pgio_header *hdr)
		}
		}

		if (is_dio) {
		pg_offset = f_offset & ~PAGE_CACHE_MASK;
		if (is_dio) {
		if (pg_offset + bytes_left > PAGE_CACHE_SIZE)
		pg_len = PAGE_CACHE_SIZE - pg_offset;
		else
		@@ -305,7 +294,7 @@ bl_read_pagelist(struct nfs_pgio_header *hdr)
		isect += (pg_offset >> SECTOR_SHIFT);
		extent_length -= (pg_offset >> SECTOR_SHIFT);
		} else {
		pg_offset = 0;
		BUG_ON(pg_offset != 0);
		pg_len = PAGE_CACHE_SIZE;
		}

		@@ -383,29 +372,6 @@ static void mark_extents_written(struct pnfs_block_layout *bl,
		}
		}

		static void bl_end_io_write_zero(struct bio *bio, int err)
		{
		struct parallel_io *par = bio->bi_private;
		struct bio_vec *bvec;
		int i;

		bio_for_each_segment_all(bvec, bio, i) {
		/* This is the zeroing page we added */
		end_page_writeback(bvec->bv_page);
		page_cache_release(bvec->bv_page);
		}

		if (unlikely(err)) {
		struct nfs_pgio_header *header = par->data;

		if (!header->pnfs_error)
		header->pnfs_error = -EIO;
		pnfs_set_lo_fail(header->lseg);
		}
		bio_put(bio);
		put_parallel(par);
		}

		static void bl_end_io_write(struct bio *bio, int err)
		{
		struct parallel_io *par = bio->bi_private;
		@@ -455,256 +421,22 @@ static void bl_end_par_io_write(void *data, int num_se)
		schedule_work(&hdr->task.u.tk_work);
		}

		/* FIXME STUB - mark intersection of layout and page as bad, so is not
		* used again.
		*/
		static void mark_bad_read(void)
		{
		return;
		}

		/*
		* map_block: map a requested I/0 block (isect) into an offset in the LVM
		* block_device
		*/
		static void
		map_block(struct buffer_head bh, sector_t isect, struct pnfs_block_extent be)
		{
		dprintk("%s enter be=%p\n", __func__, be);

		set_buffer_mapped(bh);
		bh->b_bdev = be->be_mdev;
		bh->b_blocknr = (isect - be->be_f_offset + be->be_v_offset) >>
		(be->be_mdev->bd_inode->i_blkbits - SECTOR_SHIFT);

		dprintk("%s isect %llu, bh->b_blocknr %ld, using bsize %Zd\n",
		__func__, (unsigned long long)isect, (long)bh->b_blocknr,
		bh->b_size);
		return;
		}

		static void
		bl_read_single_end_io(struct bio *bio, int error)
		{
		struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
		struct page *page = bvec->bv_page;

		/* Only one page in bvec */
		unlock_page(page);
		}

		static int
		bl_do_readpage_sync(struct page page, struct pnfs_block_extent be,
		unsigned int offset, unsigned int len)
		{
		struct bio *bio;
		struct page *shadow_page;
		sector_t isect;
		char kaddr, kshadow_addr;
		int ret = 0;

		dprintk("%s: offset %u len %u\n", __func__, offset, len);

		shadow_page = alloc_page(GFP_NOFS \| __GFP_HIGHMEM);
		if (shadow_page == NULL)
		return -ENOMEM;

		bio = bio_alloc(GFP_NOIO, 1);
		if (bio == NULL)
		return -ENOMEM;

		isect = (page->index << PAGE_CACHE_SECTOR_SHIFT) +
		(offset / SECTOR_SIZE);

		bio->bi_iter.bi_sector = isect - be->be_f_offset + be->be_v_offset;
		bio->bi_bdev = be->be_mdev;
		bio->bi_end_io = bl_read_single_end_io;

		lock_page(shadow_page);
		if (bio_add_page(bio, shadow_page,
		SECTOR_SIZE, round_down(offset, SECTOR_SIZE)) == 0) {
		unlock_page(shadow_page);
		bio_put(bio);
		return -EIO;
		}

		submit_bio(READ, bio);
		wait_on_page_locked(shadow_page);
		if (unlikely(!test_bit(BIO_UPTODATE, &bio->bi_flags))) {
		ret = -EIO;
		} else {
		kaddr = kmap_atomic(page);
		kshadow_addr = kmap_atomic(shadow_page);
		memcpy(kaddr + offset, kshadow_addr + offset, len);
		kunmap_atomic(kshadow_addr);
		kunmap_atomic(kaddr);
		}
		__free_page(shadow_page);
		bio_put(bio);

		return ret;
		}

		static int
		bl_read_partial_page_sync(struct page page, struct pnfs_block_extent be,
		unsigned int dirty_offset, unsigned int dirty_len,
		bool full_page)
		{
		int ret = 0;
		unsigned int start, end;

		if (full_page) {
		start = 0;
		end = PAGE_CACHE_SIZE;
		} else {
		start = round_down(dirty_offset, SECTOR_SIZE);
		end = round_up(dirty_offset + dirty_len, SECTOR_SIZE);
		}

		dprintk("%s: offset %u len %d\n", __func__, dirty_offset, dirty_len);
		if (!be) {
		zero_user_segments(page, start, dirty_offset,
		dirty_offset + dirty_len, end);
		if (start == 0 && end == PAGE_CACHE_SIZE &&
		trylock_page(page)) {
		SetPageUptodate(page);
		unlock_page(page);
		}
		return ret;
		}

		if (start != dirty_offset)
		ret = bl_do_readpage_sync(page, be, start, dirty_offset - start);

		if (!ret && (dirty_offset + dirty_len < end))
		ret = bl_do_readpage_sync(page, be, dirty_offset + dirty_len,
		end - dirty_offset - dirty_len);

		return ret;
		}

		/* Given an unmapped page, zero it or read in page for COW, page is locked
		* by caller.
		*/
		static int
		init_page_for_write(struct page page, struct pnfs_block_extent cow_read)
		{
		struct buffer_head *bh = NULL;
		int ret = 0;
		sector_t isect;

		dprintk("%s enter, %p\n", __func__, page);
		BUG_ON(PageUptodate(page));
		if (!cow_read) {
		zero_user_segment(page, 0, PAGE_SIZE);
		SetPageUptodate(page);
		goto cleanup;
		}

		bh = alloc_page_buffers(page, PAGE_CACHE_SIZE, 0);
		if (!bh) {
		ret = -ENOMEM;
		goto cleanup;
		}

		isect = (sector_t) page->index << PAGE_CACHE_SECTOR_SHIFT;
		map_block(bh, isect, cow_read);
		if (!bh_uptodate_or_lock(bh))
		ret = bh_submit_read(bh);
		if (ret)
		goto cleanup;
		SetPageUptodate(page);

		cleanup:
		if (bh)
		free_buffer_head(bh);
		if (ret) {
		/* Need to mark layout with bad read...should now
		* just use nfs4 for reads and writes.
		*/
		mark_bad_read();
		}
		return ret;
		}

		/* Find or create a zeroing page marked being writeback.
		* Return ERR_PTR on error, NULL to indicate skip this page and page itself
		* to indicate write out.
		*/
		static struct page *
		bl_find_get_zeroing_page(struct inode *inode, pgoff_t index,
		struct pnfs_block_extent *cow_read)
		{
		struct page *page;
		int locked = 0;
		page = find_get_page(inode->i_mapping, index);
		if (page)
		goto check_page;

		page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
		if (unlikely(!page)) {
		dprintk("%s oom\n", __func__);
		return ERR_PTR(-ENOMEM);
		}
		locked = 1;

		check_page:
		/* PageDirty: Other will write this out
		* PageWriteback: Other is writing this out
		* PageUptodate: It was read before
		*/
		if (PageDirty(page) \|\| PageWriteback(page)) {
		print_page(page);
		if (locked)
		unlock_page(page);
		page_cache_release(page);
		return NULL;
		}

		if (!locked) {
		lock_page(page);
		locked = 1;
		goto check_page;
		}
		if (!PageUptodate(page)) {
		/* New page, readin or zero it */
		init_page_for_write(page, cow_read);
		}
		set_page_writeback(page);
		unlock_page(page);

		return page;
		}

		static enum pnfs_try_status
		bl_write_pagelist(struct nfs_pgio_header *header, int sync)
		{
		int i, ret, npg_zero, pg_index, last = 0;
		int i, ret;
		struct bio *bio = NULL;
		struct pnfs_block_extent be = NULL, cow_read = NULL;
		sector_t isect, last_isect = 0, extent_length = 0;
		struct pnfs_block_extent *be = NULL;
		sector_t isect, extent_length = 0;
		struct parallel_io *par = NULL;
		loff_t offset = header->args.offset;
		size_t count = header->args.count;
		unsigned int pg_offset, pg_len, saved_len;
		struct page **pages = header->args.pages;
		struct page *page;
		pgoff_t index;
		u64 temp;
		int npg_per_block =
		NFS_SERVER(header->inode)->pnfs_blksize >> PAGE_CACHE_SHIFT;
		int pg_index = pg_index = header->args.pgbase >> PAGE_CACHE_SHIFT;
		struct blk_plug plug;

		dprintk("%s enter, %Zu@%lld\n", __func__, count, offset);

		blk_start_plug(&plug);

		if (header->dreq != NULL &&
		(!IS_ALIGNED(offset, NFS_SERVER(header->inode)->pnfs_blksize) \|\|
		!IS_ALIGNED(count, NFS_SERVER(header->inode)->pnfs_blksize))) {
		dprintk("pnfsblock nonblock aligned DIO writes. Resend MDS\n");
		goto out_mds;
		}
		/* At this point, header->page_aray is a (sequential) list of nfs_pages.
		* We want to write each, and if there is an error set pnfs_error
		* to have it redone using nfs.
		@@ -715,97 +447,20 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync)
		par->pnfs_callback = bl_end_par_io_write;
		/* At this point, have to be more careful with error handling */

		isect = (sector_t) ((offset & (long)PAGE_CACHE_MASK) >> SECTOR_SHIFT);
		be = bl_find_get_extent(BLK_LSEG2EXT(header->lseg), isect, &cow_read);
		if (!be \|\| !is_writable(be, isect)) {
		dprintk("%s no matching extents!\n", __func__);
		goto out_mds;
		}

		/* First page inside INVALID extent */
		if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
		if (likely(!bl_push_one_short_extent(be->be_inval)))
		par->bse_count++;
		else
		goto out_mds;
		temp = offset >> PAGE_CACHE_SHIFT;
		npg_zero = do_div(temp, npg_per_block);
		isect = (sector_t) (((offset - npg_zero * PAGE_CACHE_SIZE) &
		(long)PAGE_CACHE_MASK) >> SECTOR_SHIFT);
		extent_length = be->be_length - (isect - be->be_f_offset);

		fill_invalid_ext:
		dprintk("%s need to zero %d pages\n", __func__, npg_zero);
		for (;npg_zero > 0; npg_zero--) {
		if (bl_is_sector_init(be->be_inval, isect)) {
		dprintk("isect %llu already init\n",
		(unsigned long long)isect);
		goto next_page;
		}
		/* page ref released in bl_end_io_write_zero */
		index = isect >> PAGE_CACHE_SECTOR_SHIFT;
		dprintk("%s zero %dth page: index %lu isect %llu\n",
		__func__, npg_zero, index,
		(unsigned long long)isect);
		page = bl_find_get_zeroing_page(header->inode, index,
		cow_read);
		if (unlikely(IS_ERR(page))) {
		header->pnfs_error = PTR_ERR(page);
		goto out;
		} else if (page == NULL)
		goto next_page;
		blk_start_plug(&plug);

		ret = bl_mark_sectors_init(be->be_inval, isect,
		PAGE_CACHE_SECTORS);
		if (unlikely(ret)) {
		dprintk("%s bl_mark_sectors_init fail %d\n",
		__func__, ret);
		end_page_writeback(page);
		page_cache_release(page);
		header->pnfs_error = ret;
		goto out;
		}
		if (likely(!bl_push_one_short_extent(be->be_inval)))
		par->bse_count++;
		else {
		end_page_writeback(page);
		page_cache_release(page);
		header->pnfs_error = -ENOMEM;
		goto out;
		}
		/* FIXME: This should be done in bi_end_io */
		mark_extents_written(BLK_LSEG2EXT(header->lseg),
		page->index << PAGE_CACHE_SHIFT,
		PAGE_CACHE_SIZE);
		/* we always write out the whole page */
		offset = offset & (loff_t)PAGE_CACHE_MASK;
		isect = offset >> SECTOR_SHIFT;

		bio = bl_add_page_to_bio(bio, npg_zero, WRITE,
		isect, page, be,
		bl_end_io_write_zero, par);
		if (IS_ERR(bio)) {
		header->pnfs_error = PTR_ERR(bio);
		bio = NULL;
		goto out;
		}
		next_page:
		isect += PAGE_CACHE_SECTORS;
		extent_length -= PAGE_CACHE_SECTORS;
		}
		if (last)
		goto write_done;
		}
		bio = bl_submit_bio(WRITE, bio);

		/* Middle pages */
		pg_index = header->args.pgbase >> PAGE_CACHE_SHIFT;
		for (i = pg_index; i < header->page_array.npages; i++) {
		if (extent_length <= 0) {
		/* We've used up the previous extent */
		bl_put_extent(be);
		bl_put_extent(cow_read);
		bio = bl_submit_bio(WRITE, bio);
		/* Get the next one */
		be = bl_find_get_extent(BLK_LSEG2EXT(header->lseg),
		isect, &cow_read);
		isect, NULL);
		if (!be \|\| !is_writable(be, isect)) {
		header->pnfs_error = -EINVAL;
		goto out;
		@@ -823,25 +478,10 @@ next_page:
		(isect - be->be_f_offset);
		}

		dprintk("%s offset %lld count %Zu\n", __func__, offset, count);
		pg_offset = offset & ~PAGE_CACHE_MASK;
		if (pg_offset + count > PAGE_CACHE_SIZE)
		pg_len = PAGE_CACHE_SIZE - pg_offset;
		else
		pg_len = count;
		BUG_ON(offset & ~PAGE_CACHE_MASK);

		saved_len = pg_len;
		if (be->be_state == PNFS_BLOCK_INVALID_DATA &&
		!bl_is_sector_init(be->be_inval, isect)) {
		ret = bl_read_partial_page_sync(pages[i], cow_read,
		pg_offset, pg_len, true);
		if (ret) {
		dprintk("%s bl_read_partial_page_sync fail %d\n",
		__func__, ret);
		header->pnfs_error = ret;
		goto out;
		}

		ret = bl_mark_sectors_init(be->be_inval, isect,
		PAGE_CACHE_SECTORS);
		if (unlikely(ret)) {
		@@ -850,66 +490,31 @@ next_page:
		header->pnfs_error = ret;
		goto out;
		}

		/* Expand to full page write */
		pg_offset = 0;
		pg_len = PAGE_CACHE_SIZE;
		} else if ((pg_offset & (SECTOR_SIZE - 1)) \|\|
		(pg_len & (SECTOR_SIZE - 1))){
		/* ahh, nasty case. We have to do sync full sector
		* read-modify-write cycles.
		*/
		unsigned int saved_offset = pg_offset;
		ret = bl_read_partial_page_sync(pages[i], be, pg_offset,
		pg_len, false);
		pg_offset = round_down(pg_offset, SECTOR_SIZE);
		pg_len = round_up(saved_offset + pg_len, SECTOR_SIZE)
		- pg_offset;
		}


		bio = do_add_page_to_bio(bio, header->page_array.npages - i,
		WRITE,
		isect, pages[i], be,
		WRITE, isect, pages[i], be,
		bl_end_io_write, par,
		pg_offset, pg_len);
		0, PAGE_CACHE_SIZE);
		if (IS_ERR(bio)) {
		header->pnfs_error = PTR_ERR(bio);
		bio = NULL;
		goto out;
		}
		offset += saved_len;
		count -= saved_len;
		offset += PAGE_CACHE_SIZE;
		count -= PAGE_CACHE_SIZE;
		isect += PAGE_CACHE_SECTORS;
		last_isect = isect;
		extent_length -= PAGE_CACHE_SECTORS;
		}

		/* Last page inside INVALID extent */
		if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
		bio = bl_submit_bio(WRITE, bio);
		temp = last_isect >> PAGE_CACHE_SECTOR_SHIFT;
		npg_zero = npg_per_block - do_div(temp, npg_per_block);
		if (npg_zero < npg_per_block) {
		last = 1;
		goto fill_invalid_ext;
		}
		}

		write_done:
		header->res.count = header->args.count;
		out:
		bl_put_extent(be);
		bl_put_extent(cow_read);
		bl_submit_bio(WRITE, bio);
		blk_finish_plug(&plug);
		put_parallel(par);
		return PNFS_ATTEMPTED;
		out_mds:
		blk_finish_plug(&plug);
		bl_put_extent(be);
		bl_put_extent(cow_read);
		kfree(par);
		return PNFS_NOT_ATTEMPTED;
		}

		@@ -1188,19 +793,44 @@ bl_clear_layoutdriver(struct nfs_server *server)
		}

		static bool
		is_aligned_req(struct nfs_page *req, unsigned int alignment)
		is_aligned_req(struct nfs_pageio_descriptor *pgio,
		struct nfs_page *req, unsigned int alignment)
		{
		return IS_ALIGNED(req->wb_offset, alignment) &&
		IS_ALIGNED(req->wb_bytes, alignment);
		/*
		* Always accept buffered writes, higher layers take care of the
		* right alignment.
		*/
		if (pgio->pg_dreq == NULL)
		return true;

		if (!IS_ALIGNED(req->wb_offset, alignment))
		return false;

		if (IS_ALIGNED(req->wb_bytes, alignment))
		return true;

		if (req_offset(req) + req->wb_bytes == i_size_read(pgio->pg_inode)) {
		/*
		* If the write goes up to the inode size, just write
		* the full page. Data past the inode size is
		* guaranteed to be zeroed by the higher level client
		* code, and this behaviour is mandated by RFC 5663
		* section 2.3.2.
		*/
		return true;
		}

		return false;
		}

		static void
		bl_pg_init_read(struct nfs_pageio_descriptor pgio, struct nfs_page req)
		{
		if (pgio->pg_dreq != NULL &&
		!is_aligned_req(req, SECTOR_SIZE))
		if (!is_aligned_req(pgio, req, SECTOR_SIZE)) {
		nfs_pageio_reset_read_mds(pgio);
		else
		return;
		}

		pnfs_generic_pg_init_read(pgio, req);
		}

		@@ -1212,10 +842,8 @@ static size_t
		bl_pg_test_read(struct nfs_pageio_descriptor pgio, struct nfs_page prev,
		struct nfs_page *req)
		{
		if (pgio->pg_dreq != NULL &&
		!is_aligned_req(req, SECTOR_SIZE))
		if (!is_aligned_req(pgio, req, SECTOR_SIZE))
		return 0;

		return pnfs_generic_pg_test(pgio, prev, req);
		}

		@@ -1245,11 +873,13 @@ static u64 pnfs_num_cont_bytes(struct inode *inode, pgoff_t idx)
		static void
		bl_pg_init_write(struct nfs_pageio_descriptor pgio, struct nfs_page req)
		{
		if (pgio->pg_dreq != NULL &&
		!is_aligned_req(req, PAGE_CACHE_SIZE)) {
		nfs_pageio_reset_write_mds(pgio);
		} else {
		u64 wb_size;

		if (!is_aligned_req(pgio, req, PAGE_SIZE)) {
		nfs_pageio_reset_write_mds(pgio);
		return;
		}

		if (pgio->pg_dreq == NULL)
		wb_size = pnfs_num_cont_bytes(pgio->pg_inode,
		req->wb_index);
		@@ -1258,7 +888,6 @@ bl_pg_init_write(struct nfs_pageio_descriptor pgio, struct nfs_page req)

		pnfs_generic_pg_init_write(pgio, req, wb_size);
		}
		}

		/*
		* Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
		@@ -1268,10 +897,8 @@ static size_t
		bl_pg_test_write(struct nfs_pageio_descriptor pgio, struct nfs_page prev,
		struct nfs_page *req)
		{
		if (pgio->pg_dreq != NULL &&
		!is_aligned_req(req, PAGE_CACHE_SIZE))
		if (!is_aligned_req(pgio, req, PAGE_SIZE))
		return 0;

		return pnfs_generic_pg_test(pgio, prev, req);
		}

		@@ -1291,6 +918,7 @@ static struct pnfs_layoutdriver_type blocklayout_type = {
		.id = LAYOUT_BLOCK_VOLUME,
		.name = "LAYOUT_BLOCK_VOLUME",
		.owner = THIS_MODULE,
		.flags = PNFS_READ_WHOLE_PAGE,
		.read_pagelist = bl_read_pagelist,
		.write_pagelist = bl_write_pagelist,
		.alloc_layout_hdr = bl_alloc_layout_hdr,