Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6 (00d94a6a) · Commits · e / devices / android_kernel_teracube_mt6765

arch/x86/crypto/aesni-intel_asm.S

+3 −2

Original line number	Original line	Diff line number	Diff line
	@@ -845,7 +845,7 @@ ENTRY(aesni_cbc_enc)
	*/		*/
	ENTRY(aesni_cbc_dec)		ENTRY(aesni_cbc_dec)
	cmp $16, LEN		cmp $16, LEN
	jb .Lcbc_dec_ret		jb .Lcbc_dec_just_ret
	mov 480(KEYP), KLEN		mov 480(KEYP), KLEN
	add $240, KEYP		add $240, KEYP
	movups (IVP), IV		movups (IVP), IV
	@@ -891,6 +891,7 @@ ENTRY(aesni_cbc_dec)
	add $16, OUTP		add $16, OUTP
	cmp $16, LEN		cmp $16, LEN
	jge .Lcbc_dec_loop1		jge .Lcbc_dec_loop1
	movups IV, (IVP)
	.Lcbc_dec_ret:		.Lcbc_dec_ret:
			movups IV, (IVP)
			.Lcbc_dec_just_ret:
	ret		ret

arch/x86/crypto/aesni-intel_glue.c

+4 −0

Original line number	Original line	Diff line number	Diff line
	@@ -198,6 +198,7 @@ static int ecb_encrypt(struct blkcipher_desc *desc,

	blkcipher_walk_init(&walk, dst, src, nbytes);		blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);		err = blkcipher_walk_virt(desc, &walk);
			desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	kernel_fpu_begin();		kernel_fpu_begin();
	while ((nbytes = walk.nbytes)) {		while ((nbytes = walk.nbytes)) {
	@@ -221,6 +222,7 @@ static int ecb_decrypt(struct blkcipher_desc *desc,

	blkcipher_walk_init(&walk, dst, src, nbytes);		blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);		err = blkcipher_walk_virt(desc, &walk);
			desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	kernel_fpu_begin();		kernel_fpu_begin();
	while ((nbytes = walk.nbytes)) {		while ((nbytes = walk.nbytes)) {
	@@ -266,6 +268,7 @@ static int cbc_encrypt(struct blkcipher_desc *desc,

	blkcipher_walk_init(&walk, dst, src, nbytes);		blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);		err = blkcipher_walk_virt(desc, &walk);
			desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	kernel_fpu_begin();		kernel_fpu_begin();
	while ((nbytes = walk.nbytes)) {		while ((nbytes = walk.nbytes)) {
	@@ -289,6 +292,7 @@ static int cbc_decrypt(struct blkcipher_desc *desc,

	blkcipher_walk_init(&walk, dst, src, nbytes);		blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);		err = blkcipher_walk_virt(desc, &walk);
			desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	kernel_fpu_begin();		kernel_fpu_begin();
	while ((nbytes = walk.nbytes)) {		while ((nbytes = walk.nbytes)) {

arch/x86/crypto/fpu.c

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -48,7 +48,7 @@ static int crypto_fpu_encrypt(struct blkcipher_desc *desc_in,
	struct blkcipher_desc desc = {		struct blkcipher_desc desc = {
	.tfm = child,		.tfm = child,
	.info = desc_in->info,		.info = desc_in->info,
	.flags = desc_in->flags,		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
	};		};

	kernel_fpu_begin();		kernel_fpu_begin();
	@@ -67,7 +67,7 @@ static int crypto_fpu_decrypt(struct blkcipher_desc *desc_in,
	struct blkcipher_desc desc = {		struct blkcipher_desc desc = {
	.tfm = child,		.tfm = child,
	.info = desc_in->info,		.info = desc_in->info,
	.flags = desc_in->flags,		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
	};		};

	kernel_fpu_begin();		kernel_fpu_begin();

drivers/crypto/padlock-aes.c

+98 −40

Original line number	Original line	Diff line number	Diff line
	@@ -18,9 +18,22 @@
	#include <linux/percpu.h>		#include <linux/percpu.h>
	#include <linux/smp.h>		#include <linux/smp.h>
	#include <asm/byteorder.h>		#include <asm/byteorder.h>
			#include <asm/processor.h>
	#include <asm/i387.h>		#include <asm/i387.h>
	#include "padlock.h"		#include "padlock.h"

			/*
			* Number of data blocks actually fetched for each xcrypt insn.
			* Processors with prefetch errata will fetch extra blocks.
			*/
			static unsigned int ecb_fetch_blocks = 2;
			#define MAX_ECB_FETCH_BLOCKS (8)
			#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)

			static unsigned int cbc_fetch_blocks = 1;
			#define MAX_CBC_FETCH_BLOCKS (4)
			#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)

	/* Control word. */		/* Control word. */
	struct cword {		struct cword {
	unsigned int __attribute__ ((__packed__))		unsigned int __attribute__ ((__packed__))
	@@ -172,73 +185,111 @@ static inline void padlock_store_cword(struct cword *cword)
	* should be used only inside the irq_ts_save/restore() context		* should be used only inside the irq_ts_save/restore() context
	*/		*/

	static inline void padlock_xcrypt(const u8 input, u8 output, void *key,		static inline void rep_xcrypt_ecb(const u8 input, u8 output, void *key,
	struct cword *control_word)		struct cword *control_word, int count)
	{		{
	asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */		asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
	: "+S"(input), "+D"(output)		: "+S"(input), "+D"(output)
	: "d"(control_word), "b"(key), "c"(1));		: "d"(control_word), "b"(key), "c"(count));
			}

			static inline u8 rep_xcrypt_cbc(const u8 input, u8 output, void key,
			u8 iv, struct cword control_word, int count)
			{
			asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
			: "+S" (input), "+D" (output), "+a" (iv)
			: "d" (control_word), "b" (key), "c" (count));
			return iv;
	}		}

	static void aes_crypt_copy(const u8 in, u8 out, u32 key, struct cword cword)		static void ecb_crypt_copy(const u8 in, u8 out, u32 *key,
			struct cword *cword, int count)
	{		{
	u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];		/*
			* Padlock prefetches extra data so we must provide mapped input buffers.
			* Assume there are at least 16 bytes of stack already in use.
			*/
			u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
			u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);

			memcpy(tmp, in, count * AES_BLOCK_SIZE);
			rep_xcrypt_ecb(tmp, out, key, cword, count);
			}

			static u8 cbc_crypt_copy(const u8 in, u8 out, u32 key,
			u8 iv, struct cword cword, int count)
			{
			/*
			* Padlock prefetches extra data so we must provide mapped input buffers.
			* Assume there are at least 16 bytes of stack already in use.
			*/
			u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
	u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);		u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);

	memcpy(tmp, in, AES_BLOCK_SIZE);		memcpy(tmp, in, count * AES_BLOCK_SIZE);
	padlock_xcrypt(tmp, out, key, cword);		return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
	}		}

	static inline void aes_crypt(const u8 in, u8 out, u32 *key,		static inline void ecb_crypt(const u8 in, u8 out, u32 *key,
	struct cword *cword)		struct cword *cword, int count)
	{		{
	/* padlock_xcrypt requires at least two blocks of data. */		/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
	if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &		* We could avoid some copying here but it's probably not worth it.
	(PAGE_SIZE - 1)))) {		*/
	aes_crypt_copy(in, out, key, cword);		if (unlikely(((unsigned long)in & PAGE_SIZE) + ecb_fetch_bytes > PAGE_SIZE)) {
			ecb_crypt_copy(in, out, key, cword, count);
	return;		return;
	}		}

	padlock_xcrypt(in, out, key, cword);		rep_xcrypt_ecb(in, out, key, cword, count);
			}

			static inline u8 cbc_crypt(const u8 in, u8 out, u32 key,
			u8 iv, struct cword cword, int count)
			{
			/* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
			if (unlikely(((unsigned long)in & PAGE_SIZE) + cbc_fetch_bytes > PAGE_SIZE))
			return cbc_crypt_copy(in, out, key, iv, cword, count);

			return rep_xcrypt_cbc(in, out, key, iv, cword, count);
	}		}

	static inline void padlock_xcrypt_ecb(const u8 input, u8 output, void *key,		static inline void padlock_xcrypt_ecb(const u8 input, u8 output, void *key,
	void *control_word, u32 count)		void *control_word, u32 count)
	{		{
	if (count == 1) {		u32 initial = count & (ecb_fetch_blocks - 1);
	aes_crypt(input, output, key, control_word);
			if (count < ecb_fetch_blocks) {
			ecb_crypt(input, output, key, control_word, count);
	return;		return;
	}		}

	asm volatile ("test $1, %%cl;"		if (initial)
	"je 1f;"		asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
	#ifndef CONFIG_X86_64
	"lea -1(%%ecx), %%eax;"
	"mov $1, %%ecx;"
	#else
	"lea -1(%%rcx), %%rax;"
	"mov $1, %%rcx;"
	#endif
	".byte 0xf3,0x0f,0xa7,0xc8;" /* rep xcryptecb */
	#ifndef CONFIG_X86_64
	"mov %%eax, %%ecx;"
	#else
	"mov %%rax, %%rcx;"
	#endif
	"1:"
	".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
	: "+S"(input), "+D"(output)		: "+S"(input), "+D"(output)
	: "d"(control_word), "b"(key), "c"(count)		: "d"(control_word), "b"(key), "c"(initial));
	: "ax");
			asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
			: "+S"(input), "+D"(output)
			: "d"(control_word), "b"(key), "c"(count - initial));
	}		}

	static inline u8 padlock_xcrypt_cbc(const u8 input, u8 output, void key,		static inline u8 padlock_xcrypt_cbc(const u8 input, u8 output, void key,
	u8 iv, void control_word, u32 count)		u8 iv, void control_word, u32 count)
	{		{
	/* rep xcryptcbc */		u32 initial = count & (cbc_fetch_blocks - 1);
	asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
			if (count < cbc_fetch_blocks)
			return cbc_crypt(input, output, key, iv, control_word, count);

			if (initial)
			asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
	: "+S" (input), "+D" (output), "+a" (iv)		: "+S" (input), "+D" (output), "+a" (iv)
	: "d" (control_word), "b" (key), "c" (count));		: "d" (control_word), "b" (key), "c" (count));

			asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
			: "+S" (input), "+D" (output), "+a" (iv)
			: "d" (control_word), "b" (key), "c" (count-initial));
	return iv;		return iv;
	}		}

	@@ -249,7 +300,7 @@ static void aes_encrypt(struct crypto_tfm tfm, u8 out, const u8 *in)

	padlock_reset_key(&ctx->cword.encrypt);		padlock_reset_key(&ctx->cword.encrypt);
	ts_state = irq_ts_save();		ts_state = irq_ts_save();
	aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);		ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
	irq_ts_restore(ts_state);		irq_ts_restore(ts_state);
	padlock_store_cword(&ctx->cword.encrypt);		padlock_store_cword(&ctx->cword.encrypt);
	}		}
	@@ -261,7 +312,7 @@ static void aes_decrypt(struct crypto_tfm tfm, u8 out, const u8 *in)

	padlock_reset_key(&ctx->cword.encrypt);		padlock_reset_key(&ctx->cword.encrypt);
	ts_state = irq_ts_save();		ts_state = irq_ts_save();
	aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);		ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
	irq_ts_restore(ts_state);		irq_ts_restore(ts_state);
	padlock_store_cword(&ctx->cword.encrypt);		padlock_store_cword(&ctx->cword.encrypt);
	}		}
	@@ -454,6 +505,7 @@ static struct crypto_alg cbc_aes_alg = {
	static int __init padlock_init(void)		static int __init padlock_init(void)
	{		{
	int ret;		int ret;
			struct cpuinfo_x86 *c = &cpu_data(0);

	if (!cpu_has_xcrypt) {		if (!cpu_has_xcrypt) {
	printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");		printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
	@@ -476,6 +528,12 @@ static int __init padlock_init(void)

	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");		printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");

			if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
			ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
			cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
			printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
			}

	out:		out:
	return ret;		return ret;