Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2edab9e3 authored by Eric Biggers's avatar Eric Biggers Committed by Jaegeuk Kim
Browse files

fscrypt: simplify bounce page handling 



Currently, bounce page handling for writes to encrypted files is
unnecessarily complicated.  A fscrypt_ctx is allocated along with each
bounce page, page_private(bounce_page) points to this fscrypt_ctx, and
fscrypt_ctx::w::control_page points to the original pagecache page.

However, because writes don't use the fscrypt_ctx for anything else,
there's no reason why page_private(bounce_page) can't just point to the
original pagecache page directly.

Therefore, this patch makes this change.  In the process, it also cleans
up the API exposed to filesystems that allows testing whether a page is
a bounce page, getting the pagecache page from a bounce page, and
freeing a bounce page.

Reviewed-by: default avatarChandan Rajendra <chandan@linux.ibm.com>
Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
parent 94472d52

fs/crypto/bio.c

+5 −33
Original line number Diff line number Diff line
@@ -70,46 +70,18 @@ void fscrypt_enqueue_decrypt_bio(struct fscrypt_ctx *ctx, struct bio *bio) 
}

EXPORT_SYMBOL(fscrypt_enqueue_decrypt_bio);



void fscrypt_pullback_bio_page(struct page **page, bool restore)

{

	struct fscrypt_ctx *ctx;

	struct page *bounce_page;



	/* The bounce data pages are unmapped. */

	if ((*page)->mapping)

		return;



	/* The bounce data page is unmapped. */

	bounce_page = *page;

	ctx = (struct fscrypt_ctx *)page_private(bounce_page);



	/* restore control page */

	*page = ctx->w.control_page;



	if (restore)

		fscrypt_restore_control_page(bounce_page);

}

EXPORT_SYMBOL(fscrypt_pullback_bio_page);



int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,

				sector_t pblk, unsigned int len)

{

	struct fscrypt_ctx *ctx;

	struct page *ciphertext_page = NULL;

	struct page *ciphertext_page;

	struct bio *bio;

	int ret, err = 0;



	BUG_ON(inode->i_sb->s_blocksize != PAGE_SIZE);



	ctx = fscrypt_get_ctx(GFP_NOFS);

	if (IS_ERR(ctx))

		return PTR_ERR(ctx);



	ciphertext_page = fscrypt_alloc_bounce_page(ctx, GFP_NOWAIT);

	if (IS_ERR(ciphertext_page)) {

		err = PTR_ERR(ciphertext_page);

		goto errout;

	}

	ciphertext_page = fscrypt_alloc_bounce_page(GFP_NOWAIT);

	if (!ciphertext_page)

		return -ENOMEM;



	while (len--) {

		err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk,
@@ -147,7 +119,7 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, 
	}

	err = 0;

errout:

	fscrypt_release_ctx(ctx);

	fscrypt_free_bounce_page(ciphertext_page);

	return err;

}

EXPORT_SYMBOL(fscrypt_zeroout_range);

fs/crypto/crypto.c

+34 −70
Original line number Diff line number Diff line
@@ -63,18 +63,11 @@ EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); 
 *

 * If the encryption context was allocated from the pre-allocated pool, returns

 * it to that pool. Else, frees it.

 *

 * If there's a bounce page in the context, this frees that.

 */

void fscrypt_release_ctx(struct fscrypt_ctx *ctx)

{

	unsigned long flags;



	if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) {

		mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);

		ctx->w.bounce_page = NULL;

	}

	ctx->w.control_page = NULL;

	if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {

		kmem_cache_free(fscrypt_ctx_cachep, ctx);

	} else {
@@ -99,14 +92,8 @@ struct fscrypt_ctx *fscrypt_get_ctx(gfp_t gfp_flags) 
	unsigned long flags;



	/*

	 * We first try getting the ctx from a free list because in

	 * the common case the ctx will have an allocated and

	 * initialized crypto tfm, so it's probably a worthwhile

	 * optimization. For the bounce page, we first try getting it

	 * from the kernel allocator because that's just about as fast

	 * as getting it from a list and because a cache of free pages

	 * should generally be a "last resort" option for a filesystem

	 * to be able to do its job.

	 * First try getting a ctx from the free list so that we don't have to

	 * call into the slab allocator.

	 */

	spin_lock_irqsave(&fscrypt_ctx_lock, flags);

	ctx = list_first_entry_or_null(&fscrypt_free_ctxs,
@@ -122,11 +109,31 @@ struct fscrypt_ctx *fscrypt_get_ctx(gfp_t gfp_flags) 
	} else {

		ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;

	}

	ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL;

	return ctx;

}

EXPORT_SYMBOL(fscrypt_get_ctx);



struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags)

{

	return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);

}



/**

 * fscrypt_free_bounce_page() - free a ciphertext bounce page

 *

 * Free a bounce page that was allocated by fscrypt_encrypt_page(), or by

 * fscrypt_alloc_bounce_page() directly.

 */

void fscrypt_free_bounce_page(struct page *bounce_page)

{

	if (!bounce_page)

		return;

	set_page_private(bounce_page, (unsigned long)NULL);

	ClearPagePrivate(bounce_page);

	mempool_free(bounce_page, fscrypt_bounce_page_pool);

}

EXPORT_SYMBOL(fscrypt_free_bounce_page);



void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,

			 const struct fscrypt_info *ci)

{
@@ -185,16 +192,6 @@ int fscrypt_do_page_crypto(const struct inode *inode, fscrypt_direction_t rw, 
	return 0;

}



struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx,

				       gfp_t gfp_flags)

{

	ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);

	if (ctx->w.bounce_page == NULL)

		return ERR_PTR(-ENOMEM);

	ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL;

	return ctx->w.bounce_page;

}



/**

 * fscypt_encrypt_page() - Encrypts a page

 * @inode:     The inode for which the encryption should take place
@@ -209,22 +206,12 @@ struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx, 
 *             previously written data.

 * @gfp_flags: The gfp flag for memory allocation

 *

 * Encrypts @page using the ctx encryption context. Performs encryption

 * either in-place or into a newly allocated bounce page.

 * Called on the page write path.

 *

 * Bounce page allocation is the default.

 * In this case, the contents of @page are encrypted and stored in an

 * allocated bounce page. @page has to be locked and the caller must call

 * fscrypt_restore_control_page() on the returned ciphertext page to

 * release the bounce buffer and the encryption context.

 * Encrypts @page.  If the filesystem set FS_CFLG_OWN_PAGES, then the data is

 * encrypted in-place and @page is returned.  Else, a bounce page is allocated,

 * the data is encrypted into the bounce page, and the bounce page is returned.

 * The caller is responsible for calling fscrypt_free_bounce_page().

 *

 * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in

 * fscrypt_operations. Here, the input-page is returned with its content

 * encrypted.

 *

 * Return: A page with the encrypted content on success. Else, an

 * error value or NULL.

 * Return: A page containing the encrypted data on success, else an ERR_PTR()

 */

struct page *fscrypt_encrypt_page(const struct inode *inode,

				struct page *page,
@@ -233,7 +220,6 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, 
				u64 lblk_num, gfp_t gfp_flags)



{

	struct fscrypt_ctx *ctx;

	struct page *ciphertext_page = page;

	int err;
@@ -252,30 +238,20 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, 


	BUG_ON(!PageLocked(page));



	ctx = fscrypt_get_ctx(gfp_flags);

	if (IS_ERR(ctx))

		return ERR_CAST(ctx);



	/* The encryption operation will require a bounce page. */

	ciphertext_page = fscrypt_alloc_bounce_page(ctx, gfp_flags);

	if (IS_ERR(ciphertext_page))

		goto errout;

	ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags);

	if (!ciphertext_page)

		return ERR_PTR(-ENOMEM);



	ctx->w.control_page = page;

	err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num,

				     page, ciphertext_page, len, offs,

				     gfp_flags);

	if (err) {

		ciphertext_page = ERR_PTR(err);

		goto errout;

		fscrypt_free_bounce_page(ciphertext_page);

		return ERR_PTR(err);

	}

	SetPagePrivate(ciphertext_page);

	set_page_private(ciphertext_page, (unsigned long)ctx);

	lock_page(ciphertext_page);

	return ciphertext_page;



errout:

	fscrypt_release_ctx(ctx);

	set_page_private(ciphertext_page, (unsigned long)page);

	return ciphertext_page;

}

EXPORT_SYMBOL(fscrypt_encrypt_page);
@@ -354,18 +330,6 @@ const struct dentry_operations fscrypt_d_ops = { 
	.d_revalidate = fscrypt_d_revalidate,

};



void fscrypt_restore_control_page(struct page *page)

{

	struct fscrypt_ctx *ctx;



	ctx = (struct fscrypt_ctx *)page_private(page);

	set_page_private(page, (unsigned long)NULL);

	ClearPagePrivate(page);

	unlock_page(page);

	fscrypt_release_ctx(ctx);

}

EXPORT_SYMBOL(fscrypt_restore_control_page);



static void fscrypt_destroy(void)

{

	struct fscrypt_ctx *pos, *n;

fs/crypto/fscrypt_private.h

+1 −3
Original line number Diff line number Diff line
@@ -94,7 +94,6 @@ typedef enum { 
} fscrypt_direction_t;



#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL		0x00000001

#define FS_CTX_HAS_BOUNCE_BUFFER_FL		0x00000002



static inline bool fscrypt_valid_enc_modes(u32 contents_mode,

					   u32 filenames_mode)
@@ -123,8 +122,7 @@ extern int fscrypt_do_page_crypto(const struct inode *inode, 
				  struct page *dest_page,

				  unsigned int len, unsigned int offs,

				  gfp_t gfp_flags);

extern struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx,

					      gfp_t gfp_flags);

extern struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags);

extern const struct dentry_operations fscrypt_d_ops;



extern void __printf(3, 4) __cold

fs/ext4/page-io.c

+13 −23
Original line number Diff line number Diff line
@@ -66,9 +66,7 @@ static void ext4_finish_bio(struct bio *bio) 


	bio_for_each_segment_all(bvec, bio, i) {

		struct page *page = bvec->bv_page;

#ifdef CONFIG_FS_ENCRYPTION

		struct page *data_page = NULL;

#endif

		struct page *bounce_page = NULL;

		struct buffer_head *bh, *head;

		unsigned bio_start = bvec->bv_offset;

		unsigned bio_end = bio_start + bvec->bv_len;
@@ -78,13 +76,10 @@ static void ext4_finish_bio(struct bio *bio) 
		if (!page)

			continue;



#ifdef CONFIG_FS_ENCRYPTION

		if (!page->mapping) {

			/* The bounce data pages are unmapped. */

			data_page = page;

			fscrypt_pullback_bio_page(&page, false);

		if (fscrypt_is_bounce_page(page)) {

			bounce_page = page;

			page = fscrypt_pagecache_page(bounce_page);

		}

#endif



		if (bio->bi_status) {

			SetPageError(page);
@@ -111,10 +106,7 @@ static void ext4_finish_bio(struct bio *bio) 
		bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);

		local_irq_restore(flags);

		if (!under_io) {

#ifdef CONFIG_FS_ENCRYPTION

			if (data_page)

				fscrypt_restore_control_page(data_page);

#endif

			fscrypt_free_bounce_page(bounce_page);

			end_page_writeback(page);

		}

	}
@@ -415,7 +407,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io, 
			struct writeback_control *wbc,

			bool keep_towrite)

{

	struct page *data_page = NULL;

	struct page *bounce_page = NULL;

	struct inode *inode = page->mapping->host;

	unsigned block_start;

	struct buffer_head *bh, *head;
@@ -481,10 +473,10 @@ int ext4_bio_write_page(struct ext4_io_submit *io, 
		gfp_t gfp_flags = GFP_NOFS;



	retry_encrypt:

		data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0,

		bounce_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0,

						   page->index, gfp_flags);

		if (IS_ERR(data_page)) {

			ret = PTR_ERR(data_page);

		if (IS_ERR(bounce_page)) {

			ret = PTR_ERR(bounce_page);

			if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) {

				if (io->io_bio) {

					ext4_io_submit(io);
@@ -493,7 +485,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io, 
				gfp_flags |= __GFP_NOFAIL;

				goto retry_encrypt;

			}

			data_page = NULL;

			bounce_page = NULL;

			goto out;

		}

	}
@@ -502,8 +494,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io, 
	do {

		if (!buffer_async_write(bh))

			continue;

		ret = io_submit_add_bh(io, inode,

				       data_page ? data_page : page, bh);

		ret = io_submit_add_bh(io, inode, bounce_page ?: page, bh);

		if (ret) {

			/*

			 * We only get here on ENOMEM.  Not much else
@@ -519,8 +510,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io, 
	/* Error stopped previous loop? Clean up buffers... */

	if (ret) {

	out:

		if (data_page)

			fscrypt_restore_control_page(data_page);

		fscrypt_free_bounce_page(bounce_page);

		printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);

		redirty_page_for_writepage(wbc, page);

		do {

fs/f2fs/data.c

+5 −7
Original line number Diff line number Diff line
@@ -186,7 +186,7 @@ static void f2fs_write_end_io(struct bio *bio) 
			continue;

		}



		fscrypt_pullback_bio_page(&page, true);

		fscrypt_finalize_bounce_page(&page);



		if (unlikely(bio->bi_status)) {

			mapping_set_error(page->mapping, -EIO);
@@ -363,10 +363,9 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode, 


	bio_for_each_segment_all(bvec, bio, i) {



		if (bvec->bv_page->mapping)

		target = bvec->bv_page;

		else

			target = fscrypt_control_page(bvec->bv_page);

		if (fscrypt_is_bounce_page(target))

			target = fscrypt_pagecache_page(target);



		if (inode && inode == target->mapping->host)

			return true;
@@ -1958,8 +1957,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) 
		err = f2fs_inplace_write_data(fio);

		if (err) {

			if (f2fs_encrypted_file(inode))

				fscrypt_pullback_bio_page(&fio->encrypted_page,

									true);

				fscrypt_finalize_bounce_page(&fio->encrypted_page);

			if (PageWriteback(page))

				end_page_writeback(page);

		} else {