Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0bdcaf74 authored by David S. Miller's avatar David S. Miller
Browse files

sparc64: Move AES driver over to a methods based implementation. 



Instead of testing and branching off of the key size on every
encrypt/decrypt call, use method ops assigned at key set time.

Reverse the order of float registers used for decryption to make
future changes easier.

Align all assembler routines on a 32-byte boundary.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 45dfe237

arch/sparc/crypto/aes_asm.S

+591 −288

File changed.

Preview size limit exceeded, changes collapsed.

arch/sparc/crypto/aes_glue.c

+137 −49
Original line number Original line Diff line number Diff line
@@ -26,12 +26,121 @@ 
#include <asm/pstate.h>

#include <asm/pstate.h>

#include <asm/elf.h>

#include <asm/elf.h>





struct aes_ops {

	void (*encrypt)(const u64 *key, const u32 *input, u32 *output);

	void (*decrypt)(const u64 *key, const u32 *input, u32 *output);

	void (*load_encrypt_keys)(const u64 *key);

	void (*load_decrypt_keys)(const u64 *key);

	void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len);

	void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len);

	void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len, u64 *iv);

	void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len, u64 *iv);

};



struct crypto_sparc64_aes_ctx {

struct crypto_sparc64_aes_ctx {

	struct aes_ops *ops;

	u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];

	u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];

	u32 key_length;

	u32 key_length;

	u32 expanded_key_length;

	u32 expanded_key_length;

};

};





extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,

				    u32 *output);



extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,

				    u32 *output);



extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);

extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);

extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);



extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);

extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);

extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);



extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);



extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);



extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



struct aes_ops aes128_ops = {

	.encrypt		= aes_sparc64_encrypt_128,

	.decrypt		= aes_sparc64_decrypt_128,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_128,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_128,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_128,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_128,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_128,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_128,

};



struct aes_ops aes192_ops = {

	.encrypt		= aes_sparc64_encrypt_192,

	.decrypt		= aes_sparc64_decrypt_192,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_192,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_192,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_192,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_192,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_192,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_192,

};



struct aes_ops aes256_ops = {

	.encrypt		= aes_sparc64_encrypt_256,

	.decrypt		= aes_sparc64_decrypt_256,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_256,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_256,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_256,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_256,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_256,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_256,

};



extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,

extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,

				   unsigned int key_len);

				   unsigned int key_len);
@@ -44,14 +153,17 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 
	switch (key_len) {

	switch (key_len) {

	case AES_KEYSIZE_128:

	case AES_KEYSIZE_128:

		ctx->expanded_key_length = 0xb0;

		ctx->expanded_key_length = 0xb0;

		ctx->ops = &aes128_ops;

		break;

		break;





	case AES_KEYSIZE_192:

	case AES_KEYSIZE_192:

		ctx->expanded_key_length = 0xd0;

		ctx->expanded_key_length = 0xd0;

		ctx->ops = &aes192_ops;

		break;

		break;





	case AES_KEYSIZE_256:

	case AES_KEYSIZE_256:

		ctx->expanded_key_length = 0xf0;

		ctx->expanded_key_length = 0xf0;

		ctx->ops = &aes256_ops;

		break;

		break;





	default:

	default:
@@ -65,38 +177,22 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 
	return 0;

	return 0;

}

}





extern void aes_sparc64_encrypt(const u64 *key, const u32 *input,

				u32 *output, unsigned int key_len);



static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);





	aes_sparc64_encrypt(&ctx->key[0], (const u32 *) src,

	ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);

			    (u32 *) dst, ctx->key_length);

}

}





extern void aes_sparc64_decrypt(const u64 *key, const u32 *input,

				u32 *output, unsigned int key_len,

				unsigned int expanded_key_len);



static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);





	aes_sparc64_decrypt(&ctx->key[0], (const u32 *) src,

	ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);

			    (u32 *) dst, ctx->key_length,

			    ctx->expanded_key_length);

}

}





extern void aes_sparc64_load_encrypt_keys(u64 *key);

extern void aes_sparc64_load_decrypt_keys(u64 *key);



#define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))

#define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))





extern void aes_sparc64_ecb_encrypt(u64 *key, const u32 *input, u32 *output,

				    unsigned int key_len, unsigned int len);



static int ecb_encrypt(struct blkcipher_desc *desc,

static int ecb_encrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

		       unsigned int nbytes)
@@ -108,15 +204,15 @@ static int ecb_encrypt(struct blkcipher_desc *desc, 
	blkcipher_walk_init(&walk, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);

	err = blkcipher_walk_virt(desc, &walk);





	aes_sparc64_load_encrypt_keys(&ctx->key[0]);

	ctx->ops->load_encrypt_keys(&ctx->key[0]);

	while ((nbytes = walk.nbytes)) {

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;

		unsigned int block_len = nbytes & AES_BLOCK_MASK;





		if (likely(block_len)) {

		if (likely(block_len)) {

			aes_sparc64_ecb_encrypt(&ctx->key[0],

			ctx->ops->ecb_encrypt(&ctx->key[0],

						(const u32 *)walk.src.virt.addr,

					      (const u64 *)walk.src.virt.addr,

						(u32 *) walk.dst.virt.addr,

					      (u64 *) walk.dst.virt.addr,

						ctx->key_length, block_len);

					      block_len);

		}

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

		err = blkcipher_walk_done(desc, &walk, nbytes);
@@ -125,9 +221,6 @@ static int ecb_encrypt(struct blkcipher_desc *desc, 
	return err;

	return err;

}

}





extern void aes_sparc64_ecb_decrypt(u64 *ekey, const u32 *input, u32 *output,

				    unsigned int key_len, unsigned int len);



static int ecb_decrypt(struct blkcipher_desc *desc,

static int ecb_decrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

		       unsigned int nbytes)
@@ -140,14 +233,16 @@ static int ecb_decrypt(struct blkcipher_desc *desc, 
	blkcipher_walk_init(&walk, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);

	err = blkcipher_walk_virt(desc, &walk);





	aes_sparc64_load_decrypt_keys(&ctx->key[0]);

	ctx->ops->load_decrypt_keys(&ctx->key[0]);

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	while ((nbytes = walk.nbytes)) {

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;

		unsigned int block_len = nbytes & AES_BLOCK_MASK;





		aes_sparc64_ecb_decrypt(key_end, (const u32 *) walk.src.virt.addr,

		if (likely(block_len)) {

					(u32 *) walk.dst.virt.addr, ctx->key_length,

			ctx->ops->ecb_decrypt(key_end,

					block_len);

					      (const u64 *) walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr, block_len);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	}
@@ -156,10 +251,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc, 
	return err;

	return err;

}

}





extern void aes_sparc64_cbc_encrypt(u64 *key, const u32 *input, u32 *output,

				    unsigned int key_len, unsigned int len,

				    u64 *iv);



static int cbc_encrypt(struct blkcipher_desc *desc,

static int cbc_encrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

		       unsigned int nbytes)
@@ -171,16 +262,15 @@ static int cbc_encrypt(struct blkcipher_desc *desc, 
	blkcipher_walk_init(&walk, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);

	err = blkcipher_walk_virt(desc, &walk);





	aes_sparc64_load_encrypt_keys(&ctx->key[0]);

	ctx->ops->load_encrypt_keys(&ctx->key[0]);

	while ((nbytes = walk.nbytes)) {

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;

		unsigned int block_len = nbytes & AES_BLOCK_MASK;





		if (likely(block_len)) {

		if (likely(block_len)) {

			aes_sparc64_cbc_encrypt(&ctx->key[0],

			ctx->ops->cbc_encrypt(&ctx->key[0],

						(const u32 *)walk.src.virt.addr,

					      (const u64 *)walk.src.virt.addr,

						(u32 *) walk.dst.virt.addr,

					      (u64 *) walk.dst.virt.addr,

						ctx->key_length, block_len,

					      block_len, (u64 *) walk.iv);

						(u64 *) walk.iv);

		}

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

		err = blkcipher_walk_done(desc, &walk, nbytes);
@@ -189,10 +279,6 @@ static int cbc_encrypt(struct blkcipher_desc *desc, 
	return err;

	return err;

}

}





extern void aes_sparc64_cbc_decrypt(u64 *ekey, unsigned int key_len,

				    const u32 *input, u32 *output,

				    unsigned int len, u64 *iv);



static int cbc_decrypt(struct blkcipher_desc *desc,

static int cbc_decrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

		       unsigned int nbytes)
@@ -205,15 +291,17 @@ static int cbc_decrypt(struct blkcipher_desc *desc, 
	blkcipher_walk_init(&walk, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);

	err = blkcipher_walk_virt(desc, &walk);





	aes_sparc64_load_decrypt_keys(&ctx->key[0]);

	ctx->ops->load_decrypt_keys(&ctx->key[0]);

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	while ((nbytes = walk.nbytes)) {

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;

		unsigned int block_len = nbytes & AES_BLOCK_MASK;





		aes_sparc64_cbc_decrypt(key_end, ctx->key_length,

		if (likely(block_len)) {

					(const u32 *) walk.src.virt.addr,

			ctx->ops->cbc_decrypt(key_end,

					(u32 *) walk.dst.virt.addr,

					      (const u64 *) walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr,

					      block_len, (u64 *) walk.iv);

					      block_len, (u64 *) walk.iv);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	}