Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit da4f61b9 authored by Chao Yu's avatar Chao Yu Committed by Jaegeuk Kim
Browse files

f2fs: introduce io_list for serialize data/node IOs 



Serialize data/node IOs by using fifo list instead of mutex lock,
it will help to enhance concurrency of f2fs, meanwhile keeping LFS
IO semantics.

Signed-off-by: default avatarChao Yu <yuchao0@huawei.com>
Signed-off-by: default avatarJaegeuk Kim <jaegeuk@kernel.org>
parent d40e5506

fs/f2fs/checkpoint.c

+1 −0
Original line number Diff line number Diff line
@@ -163,6 +163,7 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, 
		.op = REQ_OP_READ,

		.op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,

		.encrypted_page = NULL,

		.in_list = false,

	};

	struct blk_plug plug;

fs/f2fs/data.c

+20 −4
Original line number Diff line number Diff line
@@ -397,6 +397,20 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio) 


	f2fs_bug_on(sbi, is_read_io(fio->op));



	down_write(&io->io_rwsem);

next:

	if (fio->in_list) {

		spin_lock(&io->io_lock);

		if (list_empty(&io->io_list)) {

			spin_unlock(&io->io_lock);

			goto out_fail;

		}

		fio = list_first_entry(&io->io_list,

						struct f2fs_io_info, list);

		list_del(&fio->list);

		spin_unlock(&io->io_lock);

	}



	if (fio->old_blkaddr != NEW_ADDR)

		verify_block_addr(sbi, fio->old_blkaddr);

	verify_block_addr(sbi, fio->new_blkaddr);
@@ -408,8 +422,6 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio) 


	inc_page_count(sbi, WB_DATA_TYPE(bio_page));



	down_write(&io->io_rwsem);



	if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||

	    (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||

			!__same_bdev(sbi, fio->new_blkaddr, io->bio)))
@@ -434,9 +446,13 @@ alloc_new: 


	io->last_block_in_bio = fio->new_blkaddr;

	f2fs_trace_ios(fio, 0);



	trace_f2fs_submit_page_write(fio->page, fio);



	if (fio->in_list)

		goto next;

out_fail:

	up_write(&io->io_rwsem);

	trace_f2fs_submit_page_write(fio->page, fio);

	return err;

}
@@ -749,7 +765,7 @@ alloc: 
	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);



	allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,

						&sum, CURSEG_WARM_DATA);

					&sum, CURSEG_WARM_DATA, NULL, false);

	set_data_blkaddr(dn);



	/* update i_size */

fs/f2fs/f2fs.h

+6 −1
Original line number Diff line number Diff line
@@ -914,8 +914,10 @@ struct f2fs_io_info { 
	block_t old_blkaddr;	/* old block address before Cow */

	struct page *page;	/* page to be written */

	struct page *encrypted_page;	/* encrypted page */

	struct list_head list;		/* serialize IOs */

	bool submitted;		/* indicate IO submission */

	int need_lock;		/* indicate we need to lock cp_rwsem */

	bool in_list;		/* indicate fio is in io_list */

};



#define is_read_io(rw) ((rw) == READ)
@@ -925,6 +927,8 @@ struct f2fs_bio_info { 
	sector_t last_block_in_bio;	/* last block number */

	struct f2fs_io_info fio;	/* store buffered io info. */

	struct rw_semaphore io_rwsem;	/* blocking op for bio */

	spinlock_t io_lock;		/* serialize DATA/NODE IOs */

	struct list_head io_list;	/* track fios */

};



#define FDEV(i)				(sbi->devs[i])
@@ -2394,7 +2398,8 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 
			bool recover_newaddr);

void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,

			block_t old_blkaddr, block_t *new_blkaddr,

			struct f2fs_summary *sum, int type);

			struct f2fs_summary *sum, int type,

			struct f2fs_io_info *fio, bool add_list);

void f2fs_wait_on_page_writeback(struct page *page,

			enum page_type type, bool ordered);

void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,

fs/f2fs/gc.c

+2 −1
Original line number Diff line number Diff line
@@ -600,6 +600,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx, 
		.op = REQ_OP_READ,

		.op_flags = 0,

		.encrypted_page = NULL,

		.in_list = false,

	};

	struct dnode_of_data dn;

	struct f2fs_summary sum;
@@ -643,7 +644,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx, 
	fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;



	allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,

							&sum, CURSEG_COLD_DATA);

					&sum, CURSEG_COLD_DATA, NULL, false);



	fio.encrypted_page = pagecache_get_page(META_MAPPING(fio.sbi), newaddr,

					FGP_LOCK | FGP_CREAT, GFP_NOFS);

fs/f2fs/segment.c

+15 −7
Original line number Diff line number Diff line
@@ -2188,7 +2188,8 @@ static int __get_segment_type(struct f2fs_io_info *fio) 


void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,

		block_t old_blkaddr, block_t *new_blkaddr,

		struct f2fs_summary *sum, int type)

		struct f2fs_summary *sum, int type,

		struct f2fs_io_info *fio, bool add_list)

{

	struct sit_info *sit_i = SIT_I(sbi);

	struct curseg_info *curseg = CURSEG_I(sbi, type);
@@ -2224,6 +2225,17 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 
	if (page && IS_NODESEG(type))

		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));



	if (add_list) {

		struct f2fs_bio_info *io;



		INIT_LIST_HEAD(&fio->list);

		fio->in_list = true;

		io = sbi->write_io[fio->type] + fio->temp;

		spin_lock(&io->io_lock);

		list_add_tail(&fio->list, &io->io_list);

		spin_unlock(&io->io_lock);

	}



	mutex_unlock(&curseg->curseg_mutex);

}
@@ -2232,11 +2244,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 
	int type = __get_segment_type(fio);

	int err;



	if (fio->type == NODE || fio->type == DATA)

		mutex_lock(&fio->sbi->wio_mutex[fio->type][fio->temp]);

reallocate:

	allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,

					&fio->new_blkaddr, sum, type);

			&fio->new_blkaddr, sum, type, fio, true);



	/* writeout dirty page into bdev */

	err = f2fs_submit_page_write(fio);
@@ -2244,9 +2254,6 @@ reallocate: 
		fio->old_blkaddr = fio->new_blkaddr;

		goto reallocate;

	}



	if (fio->type == NODE || fio->type == DATA)

		mutex_unlock(&fio->sbi->wio_mutex[fio->type][fio->temp]);

}



void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
@@ -2260,6 +2267,7 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) 
		.new_blkaddr = page->index,

		.page = page,

		.encrypted_page = NULL,

		.in_list = false,

	};



	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))