nfs: add mirroring support to pgio layer (a7d42ddb) · Commits · e / devices / android_kernel_oneplus_sm8150

fs/nfs/direct.c

+14 −3

Original line number	Original line	Diff line number	Diff line
	@@ -360,8 +360,14 @@ static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
	spin_lock(&dreq->lock);		spin_lock(&dreq->lock);
	if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
	dreq->error = hdr->error;		dreq->error = hdr->error;
	else		else {
			/*
			* FIXME: right now this only accounts for bytes written
			* to the first mirror
			*/
			if (hdr->pgio_mirror_idx == 0)
	dreq->count += hdr->good_bytes;		dreq->count += hdr->good_bytes;
			}
	spin_unlock(&dreq->lock);		spin_unlock(&dreq->lock);

	while (!list_empty(&hdr->pages)) {		while (!list_empty(&hdr->pages)) {
	@@ -724,6 +730,11 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
	dreq->error = hdr->error;		dreq->error = hdr->error;
	}		}
	if (dreq->error == 0) {		if (dreq->error == 0) {
			/*
			* FIXME: right now this only accounts for bytes written
			* to the first mirror
			*/
			if (hdr->pgio_mirror_idx == 0)
	dreq->count += hdr->good_bytes;		dreq->count += hdr->good_bytes;
	if (nfs_write_need_commit(hdr)) {		if (nfs_write_need_commit(hdr)) {
	if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)

fs/nfs/internal.h

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -469,6 +469,7 @@ void nfs_init_cinfo(struct nfs_commit_info *cinfo,
	struct nfs_direct_req *dreq);		struct nfs_direct_req *dreq);
	int nfs_key_timeout_notify(struct file filp, struct inode inode);		int nfs_key_timeout_notify(struct file filp, struct inode inode);
	bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx);		bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx);
			void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio);

	#ifdef CONFIG_MIGRATION		#ifdef CONFIG_MIGRATION
	extern int nfs_migrate_page(struct address_space *,		extern int nfs_migrate_page(struct address_space *,

fs/nfs/objlayout/objio_osd.c

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -537,11 +537,12 @@ int objio_write_pagelist(struct nfs_pgio_header *hdr, int how)
	static size_t objio_pg_test(struct nfs_pageio_descriptor *pgio,		static size_t objio_pg_test(struct nfs_pageio_descriptor *pgio,
	struct nfs_page prev, struct nfs_page req)		struct nfs_page prev, struct nfs_page req)
	{		{
			struct nfs_pgio_mirror *mirror = &pgio->pg_mirrors[pgio->pg_mirror_idx];
	unsigned int size;		unsigned int size;

	size = pnfs_generic_pg_test(pgio, prev, req);		size = pnfs_generic_pg_test(pgio, prev, req);

	if (!size \|\| pgio->pg_count + req->wb_bytes >		if (!size \|\| mirror->pg_count + req->wb_bytes >
	(unsigned long)pgio->pg_layout_private)		(unsigned long)pgio->pg_layout_private)
	return 0;		return 0;

fs/nfs/pagelist.c

+225 −45

Original line number	Original line	Diff line number	Diff line
	@@ -46,17 +46,22 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
	struct nfs_pgio_header *hdr,		struct nfs_pgio_header *hdr,
	void (release)(struct nfs_pgio_header hdr))		void (release)(struct nfs_pgio_header hdr))
	{		{
	hdr->req = nfs_list_entry(desc->pg_list.next);		struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


			hdr->req = nfs_list_entry(mirror->pg_list.next);
	hdr->inode = desc->pg_inode;		hdr->inode = desc->pg_inode;
	hdr->cred = hdr->req->wb_context->cred;		hdr->cred = hdr->req->wb_context->cred;
	hdr->io_start = req_offset(hdr->req);		hdr->io_start = req_offset(hdr->req);
	hdr->good_bytes = desc->pg_count;		hdr->good_bytes = mirror->pg_count;
	hdr->dreq = desc->pg_dreq;		hdr->dreq = desc->pg_dreq;
	hdr->layout_private = desc->pg_layout_private;		hdr->layout_private = desc->pg_layout_private;
	hdr->release = release;		hdr->release = release;
	hdr->completion_ops = desc->pg_completion_ops;		hdr->completion_ops = desc->pg_completion_ops;
	if (hdr->completion_ops->init_hdr)		if (hdr->completion_ops->init_hdr)
	hdr->completion_ops->init_hdr(hdr);		hdr->completion_ops->init_hdr(hdr);

			hdr->pgio_mirror_idx = desc->pg_mirror_idx;
	}		}
	EXPORT_SYMBOL_GPL(nfs_pgheader_init);		EXPORT_SYMBOL_GPL(nfs_pgheader_init);

	@@ -480,7 +485,10 @@ nfs_wait_on_request(struct nfs_page *req)
	size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,		size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
	struct nfs_page prev, struct nfs_page req)		struct nfs_page prev, struct nfs_page req)
	{		{
	if (desc->pg_count > desc->pg_bsize) {		struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


			if (mirror->pg_count > mirror->pg_bsize) {
	/* should never happen */		/* should never happen */
	WARN_ON_ONCE(1);		WARN_ON_ONCE(1);
	return 0;		return 0;
	@@ -490,11 +498,11 @@ size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
	* Limit the request size so that we can still allocate a page array		* Limit the request size so that we can still allocate a page array
	* for it without upsetting the slab allocator.		* for it without upsetting the slab allocator.
	*/		*/
	if (((desc->pg_count + req->wb_bytes) >> PAGE_SHIFT) *		if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
	sizeof(struct page) > PAGE_SIZE)		sizeof(struct page) > PAGE_SIZE)
	return 0;		return 0;

	return min(desc->pg_bsize - desc->pg_count, (size_t)req->wb_bytes);		return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
	}		}
	EXPORT_SYMBOL_GPL(nfs_generic_pg_test);		EXPORT_SYMBOL_GPL(nfs_generic_pg_test);

	@@ -651,10 +659,18 @@ EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
	static int nfs_pgio_error(struct nfs_pageio_descriptor *desc,		static int nfs_pgio_error(struct nfs_pageio_descriptor *desc,
	struct nfs_pgio_header *hdr)		struct nfs_pgio_header *hdr)
	{		{
			struct nfs_pgio_mirror *mirror;
			u32 midx;

	set_bit(NFS_IOHDR_REDO, &hdr->flags);		set_bit(NFS_IOHDR_REDO, &hdr->flags);
	nfs_pgio_data_destroy(hdr);		nfs_pgio_data_destroy(hdr);
	hdr->completion_ops->completion(hdr);		hdr->completion_ops->completion(hdr);
	desc->pg_completion_ops->error_cleanup(&desc->pg_list);		/* TODO: Make sure it's right to clean up all mirrors here
			* and not just hdr->pgio_mirror_idx */
			for (midx = 0; midx < desc->pg_mirror_count; midx++) {
			mirror = &desc->pg_mirrors[midx];
			desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
			}
	return -ENOMEM;		return -ENOMEM;
	}		}

	@@ -671,6 +687,17 @@ static void nfs_pgio_release(void *calldata)
	hdr->completion_ops->completion(hdr);		hdr->completion_ops->completion(hdr);
	}		}

			static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
			unsigned int bsize)
			{
			INIT_LIST_HEAD(&mirror->pg_list);
			mirror->pg_bytes_written = 0;
			mirror->pg_count = 0;
			mirror->pg_bsize = bsize;
			mirror->pg_base = 0;
			mirror->pg_recoalesce = 0;
			}

	/**		/**
	* nfs_pageio_init - initialise a page io descriptor		* nfs_pageio_init - initialise a page io descriptor
	* @desc: pointer to descriptor		* @desc: pointer to descriptor
	@@ -687,13 +714,10 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
	size_t bsize,		size_t bsize,
	int io_flags)		int io_flags)
	{		{
	INIT_LIST_HEAD(&desc->pg_list);		struct nfs_pgio_mirror *new;
	desc->pg_bytes_written = 0;		int i;
	desc->pg_count = 0;
	desc->pg_bsize = bsize;
	desc->pg_base = 0;
	desc->pg_moreio = 0;		desc->pg_moreio = 0;
	desc->pg_recoalesce = 0;
	desc->pg_inode = inode;		desc->pg_inode = inode;
	desc->pg_ops = pg_ops;		desc->pg_ops = pg_ops;
	desc->pg_completion_ops = compl_ops;		desc->pg_completion_ops = compl_ops;
	@@ -703,6 +727,26 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
	desc->pg_lseg = NULL;		desc->pg_lseg = NULL;
	desc->pg_dreq = NULL;		desc->pg_dreq = NULL;
	desc->pg_layout_private = NULL;		desc->pg_layout_private = NULL;
			desc->pg_bsize = bsize;

			desc->pg_mirror_count = 1;
			desc->pg_mirror_idx = 0;

			if (pg_ops->pg_get_mirror_count) {
			/* until we have a request, we don't have an lseg and no
			* idea how many mirrors there will be */
			new = kcalloc(NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX,
			sizeof(struct nfs_pgio_mirror), GFP_KERNEL);
			desc->pg_mirrors_dynamic = new;
			desc->pg_mirrors = new;

			for (i = 0; i < NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX; i++)
			nfs_pageio_mirror_init(&desc->pg_mirrors[i], bsize);
			} else {
			desc->pg_mirrors_dynamic = NULL;
			desc->pg_mirrors = desc->pg_mirrors_static;
			nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
			}
	}		}
	EXPORT_SYMBOL_GPL(nfs_pageio_init);		EXPORT_SYMBOL_GPL(nfs_pageio_init);

	@@ -738,14 +782,16 @@ static void nfs_pgio_result(struct rpc_task task, void calldata)
	int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,		int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
	struct nfs_pgio_header *hdr)		struct nfs_pgio_header *hdr)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	struct nfs_page *req;		struct nfs_page *req;
	struct page **pages,		struct page **pages,
	*last_page;		*last_page;
	struct list_head *head = &desc->pg_list;		struct list_head *head = &mirror->pg_list;
	struct nfs_commit_info cinfo;		struct nfs_commit_info cinfo;
	unsigned int pagecount, pageused;		unsigned int pagecount, pageused;

	pagecount = nfs_page_array_len(desc->pg_base, desc->pg_count);		pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
	if (!nfs_pgarray_set(&hdr->page_array, pagecount))		if (!nfs_pgarray_set(&hdr->page_array, pagecount))
	return nfs_pgio_error(desc, hdr);		return nfs_pgio_error(desc, hdr);

	@@ -773,7 +819,7 @@ int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
	desc->pg_ioflags &= ~FLUSH_COND_STABLE;		desc->pg_ioflags &= ~FLUSH_COND_STABLE;

	/* Set up the argument struct */		/* Set up the argument struct */
	nfs_pgio_rpcsetup(hdr, desc->pg_count, 0, desc->pg_ioflags, &cinfo);		nfs_pgio_rpcsetup(hdr, mirror->pg_count, 0, desc->pg_ioflags, &cinfo);
	desc->pg_rpc_callops = &nfs_pgio_common_ops;		desc->pg_rpc_callops = &nfs_pgio_common_ops;
	return 0;		return 0;
	}		}
	@@ -781,12 +827,17 @@ EXPORT_SYMBOL_GPL(nfs_generic_pgio);

	static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)		static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
	{		{
			struct nfs_pgio_mirror *mirror;
	struct nfs_pgio_header *hdr;		struct nfs_pgio_header *hdr;
	int ret;		int ret;

			mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);		hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {		if (!hdr) {
	desc->pg_completion_ops->error_cleanup(&desc->pg_list);		/* TODO: make sure this is right with mirroring - or
			* should it back out all mirrors? */
			desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
	return -ENOMEM;		return -ENOMEM;
	}		}
	nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);		nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
	@@ -801,6 +852,49 @@ static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
	return ret;		return ret;
	}		}

			/*
			* nfs_pageio_setup_mirroring - determine if mirroring is to be used
			* by calling the pg_get_mirror_count op
			*/
			static int nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
			struct nfs_page *req)
			{
			int mirror_count = 1;

			if (!pgio->pg_ops->pg_get_mirror_count)
			return 0;

			mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);

			if (!mirror_count \|\| mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX)
			return -EINVAL;

			if (WARN_ON_ONCE(!pgio->pg_mirrors_dynamic))
			return -EINVAL;

			pgio->pg_mirror_count = mirror_count;

			return 0;
			}

			/*
			* nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
			*/
			void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
			{
			pgio->pg_mirror_count = 1;
			pgio->pg_mirror_idx = 0;
			}

			static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
			{
			pgio->pg_mirror_count = 1;
			pgio->pg_mirror_idx = 0;
			pgio->pg_mirrors = pgio->pg_mirrors_static;
			kfree(pgio->pg_mirrors_dynamic);
			pgio->pg_mirrors_dynamic = NULL;
			}

	static bool nfs_match_open_context(const struct nfs_open_context *ctx1,		static bool nfs_match_open_context(const struct nfs_open_context *ctx1,
	const struct nfs_open_context *ctx2)		const struct nfs_open_context *ctx2)
	{		{
	@@ -867,19 +961,22 @@ static bool nfs_can_coalesce_requests(struct nfs_page *prev,
	static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,		static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
	struct nfs_page *req)		struct nfs_page *req)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	struct nfs_page *prev = NULL;		struct nfs_page *prev = NULL;
	if (desc->pg_count != 0) {
	prev = nfs_list_entry(desc->pg_list.prev);		if (mirror->pg_count != 0) {
			prev = nfs_list_entry(mirror->pg_list.prev);
	} else {		} else {
	if (desc->pg_ops->pg_init)		if (desc->pg_ops->pg_init)
	desc->pg_ops->pg_init(desc, req);		desc->pg_ops->pg_init(desc, req);
	desc->pg_base = req->wb_pgbase;		mirror->pg_base = req->wb_pgbase;
	}		}
	if (!nfs_can_coalesce_requests(prev, req, desc))		if (!nfs_can_coalesce_requests(prev, req, desc))
	return 0;		return 0;
	nfs_list_remove_request(req);		nfs_list_remove_request(req);
	nfs_list_add_request(req, &desc->pg_list);		nfs_list_add_request(req, &mirror->pg_list);
	desc->pg_count += req->wb_bytes;		mirror->pg_count += req->wb_bytes;
	return 1;		return 1;
	}		}

	@@ -888,16 +985,19 @@ static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
	*/		*/
	static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)		static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
	{		{
	if (!list_empty(&desc->pg_list)) {		struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


			if (!list_empty(&mirror->pg_list)) {
	int error = desc->pg_ops->pg_doio(desc);		int error = desc->pg_ops->pg_doio(desc);
	if (error < 0)		if (error < 0)
	desc->pg_error = error;		desc->pg_error = error;
	else		else
	desc->pg_bytes_written += desc->pg_count;		mirror->pg_bytes_written += mirror->pg_count;
	}		}
	if (list_empty(&desc->pg_list)) {		if (list_empty(&mirror->pg_list)) {
	desc->pg_count = 0;		mirror->pg_count = 0;
	desc->pg_base = 0;		mirror->pg_base = 0;
	}		}
	}		}

	@@ -915,10 +1015,14 @@ static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
	static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,		static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
	struct nfs_page *req)		struct nfs_page *req)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	struct nfs_page *subreq;		struct nfs_page *subreq;
	unsigned int bytes_left = 0;		unsigned int bytes_left = 0;
	unsigned int offset, pgbase;		unsigned int offset, pgbase;

			WARN_ON_ONCE(desc->pg_mirror_idx >= desc->pg_mirror_count);

	nfs_page_group_lock(req, false);		nfs_page_group_lock(req, false);

	subreq = req;		subreq = req;
	@@ -938,7 +1042,7 @@ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
	nfs_pageio_doio(desc);		nfs_pageio_doio(desc);
	if (desc->pg_error < 0)		if (desc->pg_error < 0)
	return 0;		return 0;
	if (desc->pg_recoalesce)		if (mirror->pg_recoalesce)
	return 0;		return 0;
	/* retry add_request for this subreq */		/* retry add_request for this subreq */
	nfs_page_group_lock(req, false);		nfs_page_group_lock(req, false);
	@@ -976,14 +1080,16 @@ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,

	static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)		static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
	LIST_HEAD(head);		LIST_HEAD(head);

	do {		do {
	list_splice_init(&desc->pg_list, &head);		list_splice_init(&mirror->pg_list, &head);
	desc->pg_bytes_written -= desc->pg_count;		mirror->pg_bytes_written -= mirror->pg_count;
	desc->pg_count = 0;		mirror->pg_count = 0;
	desc->pg_base = 0;		mirror->pg_base = 0;
	desc->pg_recoalesce = 0;		mirror->pg_recoalesce = 0;

	desc->pg_moreio = 0;		desc->pg_moreio = 0;

	while (!list_empty(&head)) {		while (!list_empty(&head)) {
	@@ -997,11 +1103,11 @@ static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
	return 0;		return 0;
	break;		break;
	}		}
	} while (desc->pg_recoalesce);		} while (mirror->pg_recoalesce);
	return 1;		return 1;
	}		}

	int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,		static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
	struct nfs_page *req)		struct nfs_page *req)
	{		{
	int ret;		int ret;
	@@ -1014,9 +1120,78 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
	break;		break;
	ret = nfs_do_recoalesce(desc);		ret = nfs_do_recoalesce(desc);
	} while (ret);		} while (ret);

	return ret;		return ret;
	}		}

			int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
			struct nfs_page *req)
			{
			u32 midx;
			unsigned int pgbase, offset, bytes;
			struct nfs_page dupreq, lastreq;

			pgbase = req->wb_pgbase;
			offset = req->wb_offset;
			bytes = req->wb_bytes;

			nfs_pageio_setup_mirroring(desc, req);

			for (midx = 0; midx < desc->pg_mirror_count; midx++) {
			if (midx) {
			nfs_page_group_lock(req, false);

			/* find the last request */
			for (lastreq = req->wb_head;
			lastreq->wb_this_page != req->wb_head;
			lastreq = lastreq->wb_this_page)
			;

			dupreq = nfs_create_request(req->wb_context,
			req->wb_page, lastreq, pgbase, bytes);

			if (IS_ERR(dupreq)) {
			nfs_page_group_unlock(req);
			return 0;
			}

			nfs_lock_request(dupreq);
			nfs_page_group_unlock(req);
			dupreq->wb_offset = offset;
			dupreq->wb_index = req->wb_index;
			} else
			dupreq = req;

			desc->pg_mirror_idx = midx;
			if (!nfs_pageio_add_request_mirror(desc, dupreq))
			return 0;
			}

			return 1;
			}

			/*
			* nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
			* nfs_pageio_descriptor
			* @desc: pointer to io descriptor
			*/
			static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
			u32 mirror_idx)
			{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
			u32 restore_idx = desc->pg_mirror_idx;

			desc->pg_mirror_idx = mirror_idx;
			for (;;) {
			nfs_pageio_doio(desc);
			if (!mirror->pg_recoalesce)
			break;
			if (!nfs_do_recoalesce(desc))
			break;
			}
			desc->pg_mirror_idx = restore_idx;
			}

	/*		/*
	* nfs_pageio_resend - Transfer requests to new descriptor and resend		* nfs_pageio_resend - Transfer requests to new descriptor and resend
	* @hdr - the pgio header to move request from		* @hdr - the pgio header to move request from
	@@ -1055,16 +1230,14 @@ EXPORT_SYMBOL_GPL(nfs_pageio_resend);
	*/		*/
	void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)		void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
	{		{
	for (;;) {		u32 midx;
	nfs_pageio_doio(desc);
	if (!desc->pg_recoalesce)		for (midx = 0; midx < desc->pg_mirror_count; midx++)
	break;		nfs_pageio_complete_mirror(desc, midx);
	if (!nfs_do_recoalesce(desc))
	break;
	}

	if (desc->pg_ops->pg_cleanup)		if (desc->pg_ops->pg_cleanup)
	desc->pg_ops->pg_cleanup(desc);		desc->pg_ops->pg_cleanup(desc);
			nfs_pageio_cleanup_mirroring(desc);
	}		}

	/**		/**
	@@ -1080,10 +1253,17 @@ void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
	*/		*/
	void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)		void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
	{		{
	if (!list_empty(&desc->pg_list)) {		struct nfs_pgio_mirror *mirror;
	struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);		struct nfs_page *prev;
			u32 midx;

			for (midx = 0; midx < desc->pg_mirror_count; midx++) {
			mirror = &desc->pg_mirrors[midx];
			if (!list_empty(&mirror->pg_list)) {
			prev = nfs_list_entry(mirror->pg_list.prev);
	if (index != prev->wb_index + 1)		if (index != prev->wb_index + 1)
	nfs_pageio_complete(desc);		nfs_pageio_complete_mirror(desc, midx);
			}
	}		}
	}		}

fs/nfs/pnfs.c

+18 −8

Original line number	Original line	Diff line number	Diff line
	@@ -1646,8 +1646,8 @@ EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);
	* of bytes (maximum @req->wb_bytes) that can be coalesced.		* of bytes (maximum @req->wb_bytes) that can be coalesced.
	*/		*/
	size_t		size_t
	pnfs_generic_pg_test(struct nfs_pageio_descriptor pgio, struct nfs_page prev,		pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
	struct nfs_page *req)		struct nfs_page prev, struct nfs_page req)
	{		{
	unsigned int size;		unsigned int size;
	u64 seg_end, req_start, seg_left;		u64 seg_end, req_start, seg_left;
	@@ -1729,10 +1729,12 @@ static void
	pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,		pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
	struct nfs_pgio_header *hdr)		struct nfs_pgio_header *hdr)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
	list_splice_tail_init(&hdr->pages, &desc->pg_list);		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
	nfs_pageio_reset_write_mds(desc);		nfs_pageio_reset_write_mds(desc);
	desc->pg_recoalesce = 1;		mirror->pg_recoalesce = 1;
	}		}
	nfs_pgio_data_destroy(hdr);		nfs_pgio_data_destroy(hdr);
	}		}
	@@ -1781,12 +1783,14 @@ EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
	int		int
	pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)		pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	struct nfs_pgio_header *hdr;		struct nfs_pgio_header *hdr;
	int ret;		int ret;

	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);		hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {		if (!hdr) {
	desc->pg_completion_ops->error_cleanup(&desc->pg_list);		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
	return -ENOMEM;		return -ENOMEM;
	}		}
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);		nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
	@@ -1795,6 +1799,7 @@ pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
	ret = nfs_generic_pgio(desc, hdr);		ret = nfs_generic_pgio(desc, hdr);
	if (!ret)		if (!ret)
	pnfs_do_write(desc, hdr, desc->pg_ioflags);		pnfs_do_write(desc, hdr, desc->pg_ioflags);

	return ret;		return ret;
	}		}
	EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);		EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
	@@ -1839,10 +1844,13 @@ static void
	pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,		pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
	struct nfs_pgio_header *hdr)		struct nfs_pgio_header *hdr)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
	list_splice_tail_init(&hdr->pages, &desc->pg_list);		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
	nfs_pageio_reset_read_mds(desc);		nfs_pageio_reset_read_mds(desc);
	desc->pg_recoalesce = 1;		mirror->pg_recoalesce = 1;
	}		}
	nfs_pgio_data_destroy(hdr);		nfs_pgio_data_destroy(hdr);
	}		}
	@@ -1893,12 +1901,14 @@ EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
	int		int
	pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)		pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
	{		{
			struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

	struct nfs_pgio_header *hdr;		struct nfs_pgio_header *hdr;
	int ret;		int ret;

	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);		hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {		if (!hdr) {
	desc->pg_completion_ops->error_cleanup(&desc->pg_list);		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
	return -ENOMEM;		return -ENOMEM;
	}		}
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);		nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);