Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 12ac3efe authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64/crypto: use crypto instructions to generate AES key schedule 



This patch implements the AES key schedule generation using ARMv8
Crypto Instructions. It replaces the table based C implementation
in aes_generic.ko, which means we can drop the dependency on that
module.

Tested-by: default avatarSteve Capper <steve.capper@linaro.org>
Acked-by: default avatarSteve Capper <steve.capper@linaro.org>
Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent 286fb1cc

arch/arm64/crypto/Kconfig

+2 −3
Original line number Diff line number Diff line
@@ -27,20 +27,19 @@ config CRYPTO_AES_ARM64_CE 
	tristate "AES core cipher using ARMv8 Crypto Extensions"

	depends on ARM64 && KERNEL_MODE_NEON

	select CRYPTO_ALGAPI

	select CRYPTO_AES



config CRYPTO_AES_ARM64_CE_CCM

	tristate "AES in CCM mode using ARMv8 Crypto Extensions"

	depends on ARM64 && KERNEL_MODE_NEON

	select CRYPTO_ALGAPI

	select CRYPTO_AES

	select CRYPTO_AES_ARM64_CE

	select CRYPTO_AEAD



config CRYPTO_AES_ARM64_CE_BLK

	tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions"

	depends on ARM64 && KERNEL_MODE_NEON

	select CRYPTO_BLKCIPHER

	select CRYPTO_AES

	select CRYPTO_AES_ARM64_CE

	select CRYPTO_ABLK_HELPER



config CRYPTO_AES_ARM64_NEON_BLK

arch/arm64/crypto/aes-ce-ccm-glue.c

+3 −1
Original line number Diff line number Diff line
@@ -16,6 +16,8 @@ 
#include <linux/crypto.h>

#include <linux/module.h>



#include "aes-ce-setkey.h"



static int num_rounds(struct crypto_aes_ctx *ctx)

{

	/*
@@ -48,7 +50,7 @@ static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key, 
	struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);

	int ret;



	ret = crypto_aes_expand_key(ctx, in_key, key_len);

	ret = ce_aes_expandkey(ctx, in_key, key_len);

	if (!ret)

		return 0;

arch/arm64/crypto/aes-ce-cipher.c

+111 −1
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ 
#include <linux/crypto.h>

#include <linux/module.h>



#include "aes-ce-setkey.h"



MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");
@@ -124,6 +126,114 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) 
	kernel_neon_end();

}



/*

 * aes_sub() - use the aese instruction to perform the AES sbox substitution

 *             on each byte in 'input'

 */

static u32 aes_sub(u32 input)

{

	u32 ret;



	__asm__("dup	v1.4s, %w[in]		;"

		"movi	v0.16b, #0		;"

		"aese	v0.16b, v1.16b		;"

		"umov	%w[out], v0.4s[0]	;"



	:	[out]	"=r"(ret)

	:	[in]	"r"(input)

	:		"v0","v1");



	return ret;

}



int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,

		     unsigned int key_len)

{

	/*

	 * The AES key schedule round constants

	 */

	static u8 const rcon[] = {

		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,

	};



	u32 kwords = key_len / sizeof(u32);

	struct aes_block *key_enc, *key_dec;

	int i, j;



	if (key_len != AES_KEYSIZE_128 &&

	    key_len != AES_KEYSIZE_192 &&

	    key_len != AES_KEYSIZE_256)

		return -EINVAL;



	memcpy(ctx->key_enc, in_key, key_len);

	ctx->key_length = key_len;



	kernel_neon_begin_partial(2);

	for (i = 0; i < sizeof(rcon); i++) {

		u32 *rki = ctx->key_enc + (i * kwords);

		u32 *rko = rki + kwords;



		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];

		rko[1] = rko[0] ^ rki[1];

		rko[2] = rko[1] ^ rki[2];

		rko[3] = rko[2] ^ rki[3];



		if (key_len == AES_KEYSIZE_192) {

			if (i >= 7)

				break;

			rko[4] = rko[3] ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

		} else if (key_len == AES_KEYSIZE_256) {

			if (i >= 6)

				break;

			rko[4] = aes_sub(rko[3]) ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

			rko[6] = rko[5] ^ rki[6];

			rko[7] = rko[6] ^ rki[7];

		}

	}



	/*

	 * Generate the decryption keys for the Equivalent Inverse Cipher.

	 * This involves reversing the order of the round keys, and applying

	 * the Inverse Mix Columns transformation on all but the first and

	 * the last one.

	 */

	key_enc = (struct aes_block *)ctx->key_enc;

	key_dec = (struct aes_block *)ctx->key_dec;

	j = num_rounds(ctx);



	key_dec[0] = key_enc[j];

	for (i = 1, j--; j > 0; i++, j--)

		__asm__("ld1	{v0.16b}, %[in]		;"

			"aesimc	v1.16b, v0.16b		;"

			"st1	{v1.16b}, %[out]	;"



		:	[out]	"=Q"(key_dec[i])

		:	[in]	"Q"(key_enc[j])

		:		"v0","v1");

	key_dec[i] = key_enc[0];



	kernel_neon_end();

	return 0;

}

EXPORT_SYMBOL(ce_aes_expandkey);



int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,

		  unsigned int key_len)

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	int ret;



	ret = ce_aes_expandkey(ctx, in_key, key_len);

	if (!ret)

		return 0;



	tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;

	return -EINVAL;

}

EXPORT_SYMBOL(ce_aes_setkey);



static struct crypto_alg aes_alg = {

	.cra_name		= "aes",

	.cra_driver_name	= "aes-ce",
@@ -135,7 +245,7 @@ static struct crypto_alg aes_alg = { 
	.cra_cipher = {

		.cia_min_keysize	= AES_MIN_KEY_SIZE,

		.cia_max_keysize	= AES_MAX_KEY_SIZE,

		.cia_setkey		= crypto_aes_set_key,

		.cia_setkey		= ce_aes_setkey,

		.cia_encrypt		= aes_cipher_encrypt,

		.cia_decrypt		= aes_cipher_decrypt

	}

arch/arm64/crypto/aes-ce-setkey.h

0 → 100644
+5 −0
Original line number Diff line number Diff line 


int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,

		  unsigned int key_len);

int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,

		     unsigned int key_len);

arch/arm64/crypto/aes-glue.c

+12 −6
Original line number Diff line number Diff line
@@ -16,9 +16,13 @@ 
#include <linux/module.h>

#include <linux/cpufeature.h>



#include "aes-ce-setkey.h"



#ifdef USE_V8_CRYPTO_EXTENSIONS

#define MODE			"ce"

#define PRIO			300

#define aes_setkey		ce_aes_setkey

#define aes_expandkey		ce_aes_expandkey

#define aes_ecb_encrypt		ce_aes_ecb_encrypt

#define aes_ecb_decrypt		ce_aes_ecb_decrypt

#define aes_cbc_encrypt		ce_aes_cbc_encrypt
@@ -30,6 +34,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 
#else

#define MODE			"neon"

#define PRIO			200

#define aes_setkey		crypto_aes_set_key

#define aes_expandkey		crypto_aes_expand_key

#define aes_ecb_encrypt		neon_aes_ecb_encrypt

#define aes_ecb_decrypt		neon_aes_ecb_decrypt

#define aes_cbc_encrypt		neon_aes_cbc_encrypt
@@ -79,9 +85,9 @@ static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, 
	struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);

	int ret;



	ret = crypto_aes_expand_key(&ctx->key1, in_key, key_len / 2);

	ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);

	if (!ret)

		ret = crypto_aes_expand_key(&ctx->key2, &in_key[key_len / 2],

		ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],

				    key_len / 2);

	if (!ret)

		return 0;
@@ -288,7 +294,7 @@ static struct crypto_alg aes_algs[] = { { 
		.min_keysize	= AES_MIN_KEY_SIZE,

		.max_keysize	= AES_MAX_KEY_SIZE,

		.ivsize		= AES_BLOCK_SIZE,

		.setkey		= crypto_aes_set_key,

		.setkey		= aes_setkey,

		.encrypt	= ecb_encrypt,

		.decrypt	= ecb_decrypt,

	},
@@ -306,7 +312,7 @@ static struct crypto_alg aes_algs[] = { { 
		.min_keysize	= AES_MIN_KEY_SIZE,

		.max_keysize	= AES_MAX_KEY_SIZE,

		.ivsize		= AES_BLOCK_SIZE,

		.setkey		= crypto_aes_set_key,

		.setkey		= aes_setkey,

		.encrypt	= cbc_encrypt,

		.decrypt	= cbc_decrypt,

	},
@@ -324,7 +330,7 @@ static struct crypto_alg aes_algs[] = { { 
		.min_keysize	= AES_MIN_KEY_SIZE,

		.max_keysize	= AES_MAX_KEY_SIZE,

		.ivsize		= AES_BLOCK_SIZE,

		.setkey		= crypto_aes_set_key,

		.setkey		= aes_setkey,

		.encrypt	= ctr_encrypt,

		.decrypt	= ctr_encrypt,

	},