Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7c20af08 authored by Sami Tolvanen's avatar Sami Tolvanen
Browse files

arch/arm64/crypto: fix CFI in AES CE 



Move inline assembly to a separate object file that's compiled with
LTO disabled due to incompatibility with clang's internal assembler.
Add wrappers to allow indirect calls without tripping CFI.

Bug: 67506682
Change-Id: I582b22dcdbb0bb59149f3b4cfce132b1e2d145cd
Signed-off-by: default avatarSami Tolvanen <samitolvanen@google.com>
parent d6da00dc

arch/arm64/crypto/Makefile

+2 −2
Original line number Diff line number Diff line
@@ -17,8 +17,8 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o 
obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o

ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o



obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o

CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto $(DISABLE_LTO)

obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o aes-ce-cipher-core.o

CFLAGS_aes-ce-cipher-core.o += -march=armv8-a+crypto -Wa,-march=armv8-a+crypto $(DISABLE_LTO)



obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o

aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o

arch/arm64/crypto/aes-ce-cipher-core.c

0 → 100644
+219 −0
Original line number Diff line number Diff line 
/*

 * aes-ce-cipher-core.c - core AES cipher using ARMv8 Crypto Extensions

 *

 * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 */



#include <asm/neon.h>

#include <crypto/aes.h>

#include <linux/crypto.h>



#include "aes-ce-setkey.h"



struct aes_block {

	u8 b[AES_BLOCK_SIZE];

};



static int num_rounds(struct crypto_aes_ctx *ctx)

{

	/*

	 * # of rounds specified by AES:

	 * 128 bit key		10 rounds

	 * 192 bit key		12 rounds

	 * 256 bit key		14 rounds

	 * => n byte key	=> 6 + (n/4) rounds

	 */

	return 6 + ctx->key_length / 4;

}



void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct aes_block *out = (struct aes_block *)dst;

	struct aes_block const *in = (struct aes_block *)src;

	void *dummy0;

	int dummy1;



	kernel_neon_begin_partial(4);



	__asm__("	ld1	{v0.16b}, %[in]			;"

		"	ld1	{v1.16b}, [%[key]], #16		;"

		"	cmp	%w[rounds], #10			;"

		"	bmi	0f				;"

		"	bne	3f				;"

		"	mov	v3.16b, v1.16b			;"

		"	b	2f				;"

		"0:	mov	v2.16b, v1.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"1:	aese	v0.16b, v2.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"2:	ld1	{v1.16b}, [%[key]], #16		;"

		"	aese	v0.16b, v3.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"3:	ld1	{v2.16b}, [%[key]], #16		;"

		"	subs	%w[rounds], %w[rounds], #3	;"

		"	aese	v0.16b, v1.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"	bpl	1b				;"

		"	aese	v0.16b, v2.16b			;"

		"	eor	v0.16b, v0.16b, v3.16b		;"

		"	st1	{v0.16b}, %[out]		;"



	:	[out]		"=Q"(*out),

		[key]		"=r"(dummy0),

		[rounds]	"=r"(dummy1)

	:	[in]		"Q"(*in),

				"1"(ctx->key_enc),

				"2"(num_rounds(ctx) - 2)

	:	"cc");



	kernel_neon_end();

}



void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct aes_block *out = (struct aes_block *)dst;

	struct aes_block const *in = (struct aes_block *)src;

	void *dummy0;

	int dummy1;



	kernel_neon_begin_partial(4);



	__asm__("	ld1	{v0.16b}, %[in]			;"

		"	ld1	{v1.16b}, [%[key]], #16		;"

		"	cmp	%w[rounds], #10			;"

		"	bmi	0f				;"

		"	bne	3f				;"

		"	mov	v3.16b, v1.16b			;"

		"	b	2f				;"

		"0:	mov	v2.16b, v1.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"1:	aesd	v0.16b, v2.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"2:	ld1	{v1.16b}, [%[key]], #16		;"

		"	aesd	v0.16b, v3.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"3:	ld1	{v2.16b}, [%[key]], #16		;"

		"	subs	%w[rounds], %w[rounds], #3	;"

		"	aesd	v0.16b, v1.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"	bpl	1b				;"

		"	aesd	v0.16b, v2.16b			;"

		"	eor	v0.16b, v0.16b, v3.16b		;"

		"	st1	{v0.16b}, %[out]		;"



	:	[out]		"=Q"(*out),

		[key]		"=r"(dummy0),

		[rounds]	"=r"(dummy1)

	:	[in]		"Q"(*in),

				"1"(ctx->key_dec),

				"2"(num_rounds(ctx) - 2)

	:	"cc");



	kernel_neon_end();

}



/*

 * aes_sub() - use the aese instruction to perform the AES sbox substitution

 *             on each byte in 'input'

 */

static u32 aes_sub(u32 input)

{

	u32 ret;



	__asm__("dup	v1.4s, %w[in]		;"

		"movi	v0.16b, #0		;"

		"aese	v0.16b, v1.16b		;"

		"umov	%w[out], v0.4s[0]	;"



	:	[out]	"=r"(ret)

	:	[in]	"r"(input)

	:		"v0","v1");



	return ret;

}



int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,

		     unsigned int key_len)

{

	/*

	 * The AES key schedule round constants

	 */

	static u8 const rcon[] = {

		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,

	};



	u32 kwords = key_len / sizeof(u32);

	struct aes_block *key_enc, *key_dec;

	int i, j;



	if (key_len != AES_KEYSIZE_128 &&

	    key_len != AES_KEYSIZE_192 &&

	    key_len != AES_KEYSIZE_256)

		return -EINVAL;



	memcpy(ctx->key_enc, in_key, key_len);

	ctx->key_length = key_len;



	kernel_neon_begin_partial(2);

	for (i = 0; i < sizeof(rcon); i++) {

		u32 *rki = ctx->key_enc + (i * kwords);

		u32 *rko = rki + kwords;



#ifndef CONFIG_CPU_BIG_ENDIAN

		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];

#else

		rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^

			 rki[0];

#endif

		rko[1] = rko[0] ^ rki[1];

		rko[2] = rko[1] ^ rki[2];

		rko[3] = rko[2] ^ rki[3];



		if (key_len == AES_KEYSIZE_192) {

			if (i >= 7)

				break;

			rko[4] = rko[3] ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

		} else if (key_len == AES_KEYSIZE_256) {

			if (i >= 6)

				break;

			rko[4] = aes_sub(rko[3]) ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

			rko[6] = rko[5] ^ rki[6];

			rko[7] = rko[6] ^ rki[7];

		}

	}



	/*

	 * Generate the decryption keys for the Equivalent Inverse Cipher.

	 * This involves reversing the order of the round keys, and applying

	 * the Inverse Mix Columns transformation on all but the first and

	 * the last one.

	 */

	key_enc = (struct aes_block *)ctx->key_enc;

	key_dec = (struct aes_block *)ctx->key_dec;

	j = num_rounds(ctx);



	key_dec[0] = key_enc[j];

	for (i = 1, j--; j > 0; i++, j--)

		__asm__("ld1	{v0.16b}, %[in]		;"

			"aesimc	v1.16b, v0.16b		;"

			"st1	{v1.16b}, %[out]	;"



		:	[out]	"=Q"(key_dec[i])

		:	[in]	"Q"(key_enc[j])

		:		"v0","v1");

	key_dec[i] = key_enc[0];



	kernel_neon_end();

	return 0;

}

EXPORT_SYMBOL(ce_aes_expandkey);

arch/arm64/crypto/aes-ce-cipher.c

+9 −196
Original line number Diff line number Diff line
@@ -8,7 +8,6 @@ 
 * published by the Free Software Foundation.

 */



#include <asm/neon.h>

#include <crypto/aes.h>

#include <linux/cpufeature.h>

#include <linux/crypto.h>
@@ -20,209 +19,23 @@ MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions"); 
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");



struct aes_block {

	u8 b[AES_BLOCK_SIZE];

};



static int num_rounds(struct crypto_aes_ctx *ctx)

{

	/*

	 * # of rounds specified by AES:

	 * 128 bit key		10 rounds

	 * 192 bit key		12 rounds

	 * 256 bit key		14 rounds

	 * => n byte key	=> 6 + (n/4) rounds

	 */

	return 6 + ctx->key_length / 4;

}

extern void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]);

extern void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]);



static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

#ifdef CONFIG_CFI_CLANG

static inline void __cfi_aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct aes_block *out = (struct aes_block *)dst;

	struct aes_block const *in = (struct aes_block *)src;

	void *dummy0;

	int dummy1;



	kernel_neon_begin_partial(4);



	__asm__("	ld1	{v0.16b}, %[in]			;"

		"	ld1	{v1.16b}, [%[key]], #16		;"

		"	cmp	%w[rounds], #10			;"

		"	bmi	0f				;"

		"	bne	3f				;"

		"	mov	v3.16b, v1.16b			;"

		"	b	2f				;"

		"0:	mov	v2.16b, v1.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"1:	aese	v0.16b, v2.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"2:	ld1	{v1.16b}, [%[key]], #16		;"

		"	aese	v0.16b, v3.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"3:	ld1	{v2.16b}, [%[key]], #16		;"

		"	subs	%w[rounds], %w[rounds], #3	;"

		"	aese	v0.16b, v1.16b			;"

		"	aesmc	v0.16b, v0.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"	bpl	1b				;"

		"	aese	v0.16b, v2.16b			;"

		"	eor	v0.16b, v0.16b, v3.16b		;"

		"	st1	{v0.16b}, %[out]		;"



	:	[out]		"=Q"(*out),

		[key]		"=r"(dummy0),

		[rounds]	"=r"(dummy1)

	:	[in]		"Q"(*in),

				"1"(ctx->key_enc),

				"2"(num_rounds(ctx) - 2)

	:	"cc");



	kernel_neon_end();

	aes_cipher_encrypt(tfm, dst, src);

}



static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

static inline void __cfi_aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct aes_block *out = (struct aes_block *)dst;

	struct aes_block const *in = (struct aes_block *)src;

	void *dummy0;

	int dummy1;



	kernel_neon_begin_partial(4);



	__asm__("	ld1	{v0.16b}, %[in]			;"

		"	ld1	{v1.16b}, [%[key]], #16		;"

		"	cmp	%w[rounds], #10			;"

		"	bmi	0f				;"

		"	bne	3f				;"

		"	mov	v3.16b, v1.16b			;"

		"	b	2f				;"

		"0:	mov	v2.16b, v1.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"1:	aesd	v0.16b, v2.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"2:	ld1	{v1.16b}, [%[key]], #16		;"

		"	aesd	v0.16b, v3.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"3:	ld1	{v2.16b}, [%[key]], #16		;"

		"	subs	%w[rounds], %w[rounds], #3	;"

		"	aesd	v0.16b, v1.16b			;"

		"	aesimc	v0.16b, v0.16b			;"

		"	ld1	{v3.16b}, [%[key]], #16		;"

		"	bpl	1b				;"

		"	aesd	v0.16b, v2.16b			;"

		"	eor	v0.16b, v0.16b, v3.16b		;"

		"	st1	{v0.16b}, %[out]		;"



	:	[out]		"=Q"(*out),

		[key]		"=r"(dummy0),

		[rounds]	"=r"(dummy1)

	:	[in]		"Q"(*in),

				"1"(ctx->key_dec),

				"2"(num_rounds(ctx) - 2)

	:	"cc");



	kernel_neon_end();

}



/*

 * aes_sub() - use the aese instruction to perform the AES sbox substitution

 *             on each byte in 'input'

 */

static u32 aes_sub(u32 input)

{

	u32 ret;



	__asm__("dup	v1.4s, %w[in]		;"

		"movi	v0.16b, #0		;"

		"aese	v0.16b, v1.16b		;"

		"umov	%w[out], v0.4s[0]	;"



	:	[out]	"=r"(ret)

	:	[in]	"r"(input)

	:		"v0","v1");



	return ret;

	aes_cipher_decrypt(tfm, dst, src);

}



int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,

		     unsigned int key_len)

{

	/*

	 * The AES key schedule round constants

	 */

	static u8 const rcon[] = {

		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,

	};



	u32 kwords = key_len / sizeof(u32);

	struct aes_block *key_enc, *key_dec;

	int i, j;



	if (key_len != AES_KEYSIZE_128 &&

	    key_len != AES_KEYSIZE_192 &&

	    key_len != AES_KEYSIZE_256)

		return -EINVAL;



	memcpy(ctx->key_enc, in_key, key_len);

	ctx->key_length = key_len;



	kernel_neon_begin_partial(2);

	for (i = 0; i < sizeof(rcon); i++) {

		u32 *rki = ctx->key_enc + (i * kwords);

		u32 *rko = rki + kwords;



#ifndef CONFIG_CPU_BIG_ENDIAN

		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];

#else

		rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^

			 rki[0];

#define aes_cipher_encrypt __cfi_aes_cipher_encrypt

#define aes_cipher_decrypt __cfi_aes_cipher_decrypt

#endif

		rko[1] = rko[0] ^ rki[1];

		rko[2] = rko[1] ^ rki[2];

		rko[3] = rko[2] ^ rki[3];



		if (key_len == AES_KEYSIZE_192) {

			if (i >= 7)

				break;

			rko[4] = rko[3] ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

		} else if (key_len == AES_KEYSIZE_256) {

			if (i >= 6)

				break;

			rko[4] = aes_sub(rko[3]) ^ rki[4];

			rko[5] = rko[4] ^ rki[5];

			rko[6] = rko[5] ^ rki[6];

			rko[7] = rko[6] ^ rki[7];

		}

	}



	/*

	 * Generate the decryption keys for the Equivalent Inverse Cipher.

	 * This involves reversing the order of the round keys, and applying

	 * the Inverse Mix Columns transformation on all but the first and

	 * the last one.

	 */

	key_enc = (struct aes_block *)ctx->key_enc;

	key_dec = (struct aes_block *)ctx->key_dec;

	j = num_rounds(ctx);



	key_dec[0] = key_enc[j];

	for (i = 1, j--; j > 0; i++, j--)

		__asm__("ld1	{v0.16b}, %[in]		;"

			"aesimc	v1.16b, v0.16b		;"

			"st1	{v1.16b}, %[out]	;"



		:	[out]	"=Q"(key_dec[i])

		:	[in]	"Q"(key_enc[j])

		:		"v0","v1");

	key_dec[i] = key_enc[0];



	kernel_neon_end();

	return 0;

}

EXPORT_SYMBOL(ce_aes_expandkey);



int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,

		  unsigned int key_len)