new helper: iov_iter_get_pages() (7b2c99d1) · Commits · e / devices / android_kernel_xiaomi_markw

fs/direct-io.c

+38 −73

Original line number	Original line	Diff line number	Diff line
	@@ -77,7 +77,6 @@ struct dio_submit {
	unsigned blocks_available; /* At block_in_file. changes */		unsigned blocks_available; /* At block_in_file. changes */
	int reap_counter; /* rate limit reaping */		int reap_counter; /* rate limit reaping */
	sector_t final_block_in_request;/* doesn't change */		sector_t final_block_in_request;/* doesn't change */
	unsigned first_block_in_page; /* doesn't change, Used only once */
	int boundary; /* prev block is at a boundary */		int boundary; /* prev block is at a boundary */
	get_block_t get_block; / block mapping function */		get_block_t get_block; / block mapping function */
	dio_submit_t submit_io; / IO submition function */		dio_submit_t submit_io; / IO submition function */
	@@ -98,19 +97,14 @@ struct dio_submit {
	sector_t cur_page_block; /* Where it starts */		sector_t cur_page_block; /* Where it starts */
	loff_t cur_page_fs_offset; /* Offset in file */		loff_t cur_page_fs_offset; /* Offset in file */

	/*		struct iov_iter *iter;
	* Page fetching state. These variables belong to dio_refill_pages().
	*/
	int curr_page; /* changes */
	int total_pages; /* doesn't change */
	unsigned long curr_user_address;/* changes */

	/*		/*
	* Page queue. These variables belong to dio_refill_pages() and		* Page queue. These variables belong to dio_refill_pages() and
	* dio_get_page().		* dio_get_page().
	*/		*/
	unsigned head; /* next page to process */		unsigned head; /* next page to process */
	unsigned tail; /* last valid page + 1 */		unsigned tail; /* last valid page + 1 */
			size_t from, to;
	};		};

	/* dio_state communicated between submission path and end_io */		/* dio_state communicated between submission path and end_io */
	@@ -163,15 +157,10 @@ static inline unsigned dio_pages_present(struct dio_submit *sdio)
	*/		*/
	static inline int dio_refill_pages(struct dio dio, struct dio_submit sdio)		static inline int dio_refill_pages(struct dio dio, struct dio_submit sdio)
	{		{
	int ret;		ssize_t ret;
	int nr_pages;

	nr_pages = min(sdio->total_pages - sdio->curr_page, DIO_PAGES);		ret = iov_iter_get_pages(sdio->iter, dio->pages, DIO_PAGES * PAGE_SIZE,
	ret = get_user_pages_fast(		&sdio->from);
	sdio->curr_user_address, /* Where from? */
	nr_pages, /* How many pages? */
	dio->rw == READ, /* Write to memory? */
	&dio->pages[0]); /* Put results here */

	if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {		if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {
	struct page *page = ZERO_PAGE(0);		struct page *page = ZERO_PAGE(0);
	@@ -186,18 +175,19 @@ static inline int dio_refill_pages(struct dio dio, struct dio_submit sdio)
	dio->pages[0] = page;		dio->pages[0] = page;
	sdio->head = 0;		sdio->head = 0;
	sdio->tail = 1;		sdio->tail = 1;
	ret = 0;		sdio->from = 0;
	goto out;		sdio->to = PAGE_SIZE;
			return 0;
	}		}

	if (ret >= 0) {		if (ret >= 0) {
	sdio->curr_user_address += ret * PAGE_SIZE;		iov_iter_advance(sdio->iter, ret);
	sdio->curr_page += ret;		ret += sdio->from;
	sdio->head = 0;		sdio->head = 0;
	sdio->tail = ret;		sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
	ret = 0;		sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;
			return 0;
	}		}
	out:
	return ret;		return ret;
	}		}

	@@ -208,8 +198,9 @@ out:
	* L1 cache.		* L1 cache.
	*/		*/
	static inline struct page dio_get_page(struct dio dio,		static inline struct page dio_get_page(struct dio dio,
	struct dio_submit *sdio)		struct dio_submit sdio, size_t from, size_t *to)
	{		{
			int n;
	if (dio_pages_present(sdio) == 0) {		if (dio_pages_present(sdio) == 0) {
	int ret;		int ret;

	@@ -218,7 +209,10 @@ static inline struct page dio_get_page(struct dio dio,
	return ERR_PTR(ret);		return ERR_PTR(ret);
	BUG_ON(dio_pages_present(sdio) == 0);		BUG_ON(dio_pages_present(sdio) == 0);
	}		}
	return dio->pages[sdio->head++];		n = sdio->head++;
			*from = n ? 0 : sdio->from;
			*to = (n == sdio->tail - 1) ? sdio->to : PAGE_SIZE;
			return dio->pages[n];
	}		}

	/**		/**
	@@ -422,8 +416,8 @@ static inline void dio_bio_submit(struct dio dio, struct dio_submit sdio)
	*/		*/
	static inline void dio_cleanup(struct dio dio, struct dio_submit sdio)		static inline void dio_cleanup(struct dio dio, struct dio_submit sdio)
	{		{
	while (dio_pages_present(sdio))		while (sdio->head < sdio->tail)
	page_cache_release(dio_get_page(dio, sdio));		page_cache_release(dio->pages[sdio->head++]);
	}		}

	/*		/*
	@@ -912,23 +906,18 @@ static int do_direct_IO(struct dio dio, struct dio_submit sdio,
	struct buffer_head *map_bh)		struct buffer_head *map_bh)
	{		{
	const unsigned blkbits = sdio->blkbits;		const unsigned blkbits = sdio->blkbits;
	const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
	struct page *page;
	unsigned block_in_page;
	int ret = 0;		int ret = 0;

	/* The I/O can start at any block offset within the first page */
	block_in_page = sdio->first_block_in_page;

	while (sdio->block_in_file < sdio->final_block_in_request) {		while (sdio->block_in_file < sdio->final_block_in_request) {
	page = dio_get_page(dio, sdio);		struct page *page;
			size_t from, to;
			page = dio_get_page(dio, sdio, &from, &to);
	if (IS_ERR(page)) {		if (IS_ERR(page)) {
	ret = PTR_ERR(page);		ret = PTR_ERR(page);
	goto out;		goto out;
	}		}

	while (block_in_page < blocks_per_page) {		while (from < to) {
	unsigned offset_in_page = block_in_page << blkbits;
	unsigned this_chunk_bytes; /* # of bytes mapped */		unsigned this_chunk_bytes; /* # of bytes mapped */
	unsigned this_chunk_blocks; /* # of blocks */		unsigned this_chunk_blocks; /* # of blocks */
	unsigned u;		unsigned u;
	@@ -999,10 +988,9 @@ do_holes:
	page_cache_release(page);		page_cache_release(page);
	goto out;		goto out;
	}		}
	zero_user(page, block_in_page << blkbits,		zero_user(page, from, 1 << blkbits);
	1 << blkbits);
	sdio->block_in_file++;		sdio->block_in_file++;
	block_in_page++;		from += 1 << blkbits;
	dio->result += 1 << blkbits;		dio->result += 1 << blkbits;
	goto next_block;		goto next_block;
	}		}
	@@ -1020,7 +1008,7 @@ do_holes:
	* can add to this page		* can add to this page
	*/		*/
	this_chunk_blocks = sdio->blocks_available;		this_chunk_blocks = sdio->blocks_available;
	u = (PAGE_SIZE - offset_in_page) >> blkbits;		u = (to - from) >> blkbits;
	if (this_chunk_blocks > u)		if (this_chunk_blocks > u)
	this_chunk_blocks = u;		this_chunk_blocks = u;
	u = sdio->final_block_in_request - sdio->block_in_file;		u = sdio->final_block_in_request - sdio->block_in_file;
	@@ -1032,7 +1020,7 @@ do_holes:
	if (this_chunk_blocks == sdio->blocks_available)		if (this_chunk_blocks == sdio->blocks_available)
	sdio->boundary = buffer_boundary(map_bh);		sdio->boundary = buffer_boundary(map_bh);
	ret = submit_page_section(dio, sdio, page,		ret = submit_page_section(dio, sdio, page,
	offset_in_page,		from,
	this_chunk_bytes,		this_chunk_bytes,
	sdio->next_block_for_io,		sdio->next_block_for_io,
	map_bh);		map_bh);
	@@ -1043,9 +1031,9 @@ do_holes:
	sdio->next_block_for_io += this_chunk_blocks;		sdio->next_block_for_io += this_chunk_blocks;

	sdio->block_in_file += this_chunk_blocks;		sdio->block_in_file += this_chunk_blocks;
	block_in_page += this_chunk_blocks;		from += this_chunk_bytes;
			dio->result += this_chunk_bytes;
	sdio->blocks_available -= this_chunk_blocks;		sdio->blocks_available -= this_chunk_blocks;
	dio->result += this_chunk_blocks << blkbits;
	next_block:		next_block:
	BUG_ON(sdio->block_in_file > sdio->final_block_in_request);		BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
	if (sdio->block_in_file == sdio->final_block_in_request)		if (sdio->block_in_file == sdio->final_block_in_request)
	@@ -1054,7 +1042,6 @@ next_block:

	/* Drop the ref which was taken in get_user_pages() */		/* Drop the ref which was taken in get_user_pages() */
	page_cache_release(page);		page_cache_release(page);
	block_in_page = 0;
	}		}
	out:		out:
	return ret;		return ret;
	@@ -1122,7 +1109,6 @@ do_blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode,
	struct dio *dio;		struct dio *dio;
	struct dio_submit sdio = { 0, };		struct dio_submit sdio = { 0, };
	unsigned long user_addr;		unsigned long user_addr;
	size_t bytes;
	struct buffer_head map_bh = { 0, };		struct buffer_head map_bh = { 0, };
	struct blk_plug plug;		struct blk_plug plug;
	unsigned long align = offset \| iov_iter_alignment(iter);		unsigned long align = offset \| iov_iter_alignment(iter);
	@@ -1234,6 +1220,10 @@ do_blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode,
	spin_lock_init(&dio->bio_lock);		spin_lock_init(&dio->bio_lock);
	dio->refcount = 1;		dio->refcount = 1;

			sdio.iter = iter;
			sdio.final_block_in_request =
			(offset + iov_iter_count(iter)) >> blkbits;

	/*		/*
	* In case of non-aligned buffers, we may need 2 more		* In case of non-aligned buffers, we may need 2 more
	* pages since we need to zero out first and last block.		* pages since we need to zero out first and last block.
	@@ -1250,34 +1240,9 @@ do_blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode,

	blk_start_plug(&plug);		blk_start_plug(&plug);

	for (seg = 0; seg < iter->nr_segs; seg++) {
	user_addr = (unsigned long)iter->iov[seg].iov_base;
	sdio.size += bytes = iter->iov[seg].iov_len;

	/* Index into the first page of the first block */
	sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
	sdio.final_block_in_request = sdio.block_in_file +
	(bytes >> blkbits);
	/* Page fetching state */
	sdio.head = 0;
	sdio.tail = 0;
	sdio.curr_page = 0;

	sdio.total_pages = 0;
	if (user_addr & (PAGE_SIZE-1)) {
	sdio.total_pages++;
	bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
	}
	sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
	sdio.curr_user_address = user_addr;

	retval = do_direct_IO(dio, &sdio, &map_bh);		retval = do_direct_IO(dio, &sdio, &map_bh);
			if (retval)
	if (retval) {
	dio_cleanup(dio, &sdio);		dio_cleanup(dio, &sdio);
	break;
	}
	} /* end iovec loop */

	if (retval == -ENOTBLK) {		if (retval == -ENOTBLK) {
	/*		/*

include/linux/uio.h

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -71,6 +71,8 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
	unsigned long iov_iter_alignment(const struct iov_iter *i);		unsigned long iov_iter_alignment(const struct iov_iter *i);
	void iov_iter_init(struct iov_iter i, int direction, const struct iovec iov,		void iov_iter_init(struct iov_iter i, int direction, const struct iovec iov,
	unsigned long nr_segs, size_t count);		unsigned long nr_segs, size_t count);
			ssize_t iov_iter_get_pages(struct iov_iter i, struct page *pages,
			size_t maxsize, size_t *start);

	static inline size_t iov_iter_count(struct iov_iter *i)		static inline size_t iov_iter_count(struct iov_iter *i)
	{		{

mm/iov_iter.c

+27 −0

Original line number	Original line	Diff line number	Diff line
	@@ -235,3 +235,30 @@ void iov_iter_init(struct iov_iter *i, int direction,
	i->count = count;		i->count = count;
	}		}
	EXPORT_SYMBOL(iov_iter_init);		EXPORT_SYMBOL(iov_iter_init);

			ssize_t iov_iter_get_pages(struct iov_iter *i,
			struct page **pages, size_t maxsize,
			size_t *start)
			{
			size_t offset = i->iov_offset;
			const struct iovec *iov = i->iov;
			size_t len;
			unsigned long addr;
			int n;
			int res;

			len = iov->iov_len - offset;
			if (len > i->count)
			len = i->count;
			if (len > maxsize)
			len = maxsize;
			addr = (unsigned long)iov->iov_base + offset;
			len += *start = addr & (PAGE_SIZE - 1);
			addr &= ~(PAGE_SIZE - 1);
			n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
			res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
			if (unlikely(res < 0))
			return res;
			return (res == n ? len : res * PAGE_SIZE) - *start;
			}
			EXPORT_SYMBOL(iov_iter_get_pages);