Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c9f1fd4f authored by Herbert Xu's avatar Herbert Xu
Browse files

Revert "crypto: aegis128 - add support for SIMD acceleration" 



This reverts commit ecc8bc81
("crypto: aegis128 - provide a SIMD implementation based on NEON
intrinsics") and commit 7cdc0ddb
("crypto: aegis128 - add support for SIMD acceleration").

They cause compile errors on platforms other than ARM because
the mechanism to selectively compile the SIMD code is broken.

Repoted-by: default avatarHeiko Carstens <heiko.carstens@de.ibm.com>
Reported-by: default avatarStephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 82cb5485

crypto/Kconfig

+0 −5
Original line number Diff line number Diff line
@@ -306,11 +306,6 @@ config CRYPTO_AEGIS128 
	help

	 Support for the AEGIS-128 dedicated AEAD algorithm.



config CRYPTO_AEGIS128_SIMD

	bool "Support SIMD acceleration for AEGIS-128"

	depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON)

	default y



config CRYPTO_AEGIS128_AESNI_SSE2

	tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)"

	depends on X86 && 64BIT

crypto/Makefile

+0 −12
Original line number Diff line number Diff line
@@ -90,18 +90,6 @@ obj-$(CONFIG_CRYPTO_GCM) += gcm.o 
obj-$(CONFIG_CRYPTO_CCM) += ccm.o

obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o

obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o

aegis128-y := aegis128-core.o



ifeq ($(ARCH),arm)

CFLAGS_aegis128-neon-inner.o += -ffreestanding -march=armv7-a -mfloat-abi=softfp -mfpu=crypto-neon-fp-armv8

aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o

endif

ifeq ($(ARCH),arm64)

CFLAGS_aegis128-neon-inner.o += -ffreestanding -mcpu=generic+crypto

CFLAGS_REMOVE_aegis128-neon-inner.o += -mgeneral-regs-only

aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o

endif



obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o

obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o

obj-$(CONFIG_CRYPTO_DES) += des_generic.o

crypto/aegis128-neon-inner.c

deleted100644 → 0
+0 −149
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>

 */



#ifdef CONFIG_ARM64

#include <asm/neon-intrinsics.h>



#define AES_ROUND	"aese %0.16b, %1.16b \n\t aesmc %0.16b, %0.16b"

#else

#include <arm_neon.h>



#define AES_ROUND	"aese.8 %q0, %q1 \n\t aesmc.8 %q0, %q0"

#endif



#define AEGIS_BLOCK_SIZE	16



#include <stddef.h>



void *memcpy(void *dest, const void *src, size_t n);

void *memset(void *s, int c, size_t n);



struct aegis128_state {

	uint8x16_t v[5];

};



static struct aegis128_state aegis128_load_state_neon(const void *state)

{

	return (struct aegis128_state){ {

		vld1q_u8(state),

		vld1q_u8(state + 16),

		vld1q_u8(state + 32),

		vld1q_u8(state + 48),

		vld1q_u8(state + 64)

	} };

}



static void aegis128_save_state_neon(struct aegis128_state st, void *state)

{

	vst1q_u8(state, st.v[0]);

	vst1q_u8(state + 16, st.v[1]);

	vst1q_u8(state + 32, st.v[2]);

	vst1q_u8(state + 48, st.v[3]);

	vst1q_u8(state + 64, st.v[4]);

}



static uint8x16_t aegis_aes_round(uint8x16_t w)

{

	uint8x16_t z = {};



	/*

	 * We use inline asm here instead of the vaeseq_u8/vaesmcq_u8 intrinsics

	 * to force the compiler to issue the aese/aesmc instructions in pairs.

	 * This is much faster on many cores, where the instruction pair can

	 * execute in a single cycle.

	 */

	asm(AES_ROUND : "+w"(w) : "w"(z));

	return w;

}



static struct aegis128_state aegis128_update_neon(struct aegis128_state st,

						  uint8x16_t m)

{

	uint8x16_t t;



	t        = aegis_aes_round(st.v[3]);

	st.v[3] ^= aegis_aes_round(st.v[2]);

	st.v[2] ^= aegis_aes_round(st.v[1]);

	st.v[1] ^= aegis_aes_round(st.v[0]);

	st.v[0] ^= aegis_aes_round(st.v[4]) ^ m;

	st.v[4] ^= t;



	return st;

}



void crypto_aegis128_update_neon(void *state, const void *msg)

{

	struct aegis128_state st = aegis128_load_state_neon(state);



	st = aegis128_update_neon(st, vld1q_u8(msg));



	aegis128_save_state_neon(st, state);

}



void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src,

					unsigned int size)

{

	struct aegis128_state st = aegis128_load_state_neon(state);

	uint8x16_t tmp;



	while (size >= AEGIS_BLOCK_SIZE) {

		uint8x16_t s = vld1q_u8(src);



		tmp = s ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];

		st = aegis128_update_neon(st, s);

		vst1q_u8(dst, tmp);



		size -= AEGIS_BLOCK_SIZE;

		src += AEGIS_BLOCK_SIZE;

		dst += AEGIS_BLOCK_SIZE;

	}



	if (size > 0) {

		uint8_t buf[AEGIS_BLOCK_SIZE] = {};

		uint8x16_t msg;



		memcpy(buf, src, size);

		msg = vld1q_u8(buf);

		tmp = msg ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];

		st = aegis128_update_neon(st, msg);

		vst1q_u8(buf, tmp);

		memcpy(dst, buf, size);

	}



	aegis128_save_state_neon(st, state);

}



void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src,

					unsigned int size)

{

	struct aegis128_state st = aegis128_load_state_neon(state);

	uint8x16_t tmp;



	while (size >= AEGIS_BLOCK_SIZE) {

		tmp = vld1q_u8(src) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];

		st = aegis128_update_neon(st, tmp);

		vst1q_u8(dst, tmp);



		size -= AEGIS_BLOCK_SIZE;

		src += AEGIS_BLOCK_SIZE;

		dst += AEGIS_BLOCK_SIZE;

	}



	if (size > 0) {

		uint8_t buf[AEGIS_BLOCK_SIZE] = {};

		uint8x16_t msg;



		memcpy(buf, src, size);

		msg = vld1q_u8(buf) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];

		vst1q_u8(buf, msg);

		memcpy(dst, buf, size);



		memset(buf + size, 0, AEGIS_BLOCK_SIZE - size);

		msg = vld1q_u8(buf);

		st = aegis128_update_neon(st, msg);

	}



	aegis128_save_state_neon(st, state);

}

crypto/aegis128-neon.c

deleted100644 → 0
+0 −43
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * Copyright (C) 2019 Linaro Ltd <ard.biesheuvel@linaro.org>

 */



#include <asm/cpufeature.h>

#include <asm/neon.h>



#include "aegis.h"



void crypto_aegis128_update_neon(void *state, const void *msg);

void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src,

					unsigned int size);

void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src,

					unsigned int size);



bool crypto_aegis128_have_simd(void)

{

	return cpu_have_feature(cpu_feature(AES));

}



void crypto_aegis128_update_simd(union aegis_block *state, const void *msg)

{

	kernel_neon_begin();

	crypto_aegis128_update_neon(state, msg);

	kernel_neon_end();

}



void crypto_aegis128_encrypt_chunk_simd(union aegis_block *state, u8 *dst,

					const u8 *src, unsigned int size)

{

	kernel_neon_begin();

	crypto_aegis128_encrypt_chunk_neon(state, dst, src, size);

	kernel_neon_end();

}



void crypto_aegis128_decrypt_chunk_simd(union aegis_block *state, u8 *dst,

					const u8 *src, unsigned int size)

{

	kernel_neon_begin();

	crypto_aegis128_decrypt_chunk_neon(state, dst, src, size);

	kernel_neon_end();

}

crypto/aegis128-core.c→crypto/aegis128.c

+4 −38
Original line number Diff line number Diff line
@@ -8,7 +8,6 @@ 


#include <crypto/algapi.h>

#include <crypto/internal/aead.h>

#include <crypto/internal/simd.h>

#include <crypto/internal/skcipher.h>

#include <crypto/scatterwalk.h>

#include <linux/err.h>
@@ -16,7 +15,6 @@ 
#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/scatterlist.h>

#include <asm/simd.h>



#include "aegis.h"
@@ -42,15 +40,6 @@ struct aegis128_ops { 
			    const u8 *src, unsigned int size);

};



static bool have_simd;



bool crypto_aegis128_have_simd(void);

void crypto_aegis128_update_simd(struct aegis_state *state, const void *msg);

void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst,

					const u8 *src, unsigned int size);

void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst,

					const u8 *src, unsigned int size);



static void crypto_aegis128_update(struct aegis_state *state)

{

	union aegis_block tmp;
@@ -66,22 +55,12 @@ static void crypto_aegis128_update(struct aegis_state *state) 
static void crypto_aegis128_update_a(struct aegis_state *state,

				     const union aegis_block *msg)

{

	if (have_simd && crypto_simd_usable()) {

		crypto_aegis128_update_simd(state, msg);

		return;

	}



	crypto_aegis128_update(state);

	crypto_aegis_block_xor(&state->blocks[0], msg);

}



static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg)

{

	if (have_simd && crypto_simd_usable()) {

		crypto_aegis128_update_simd(state, msg);

		return;

	}



	crypto_aegis128_update(state);

	crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);

}
@@ -386,7 +365,7 @@ static void crypto_aegis128_crypt(struct aead_request *req, 


static int crypto_aegis128_encrypt(struct aead_request *req)

{

	const struct aegis128_ops *ops = &(struct aegis128_ops){

	static const struct aegis128_ops ops = {

		.skcipher_walk_init = skcipher_walk_aead_encrypt,

		.crypt_chunk = crypto_aegis128_encrypt_chunk,

	};
@@ -396,12 +375,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req) 
	unsigned int authsize = crypto_aead_authsize(tfm);

	unsigned int cryptlen = req->cryptlen;



	if (have_simd && crypto_simd_usable())

		ops = &(struct aegis128_ops){

			.skcipher_walk_init = skcipher_walk_aead_encrypt,

			.crypt_chunk = crypto_aegis128_encrypt_chunk_simd };



	crypto_aegis128_crypt(req, &tag, cryptlen, ops);

	crypto_aegis128_crypt(req, &tag, cryptlen, &ops);



	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,

				 authsize, 1);
@@ -410,7 +384,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req) 


static int crypto_aegis128_decrypt(struct aead_request *req)

{

	const struct aegis128_ops *ops = &(struct aegis128_ops){

	static const struct aegis128_ops ops = {

		.skcipher_walk_init = skcipher_walk_aead_decrypt,

		.crypt_chunk = crypto_aegis128_decrypt_chunk,

	};
@@ -424,12 +398,7 @@ static int crypto_aegis128_decrypt(struct aead_request *req) 
	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,

				 authsize, 0);



	if (have_simd && crypto_simd_usable())

		ops = &(struct aegis128_ops){

			.skcipher_walk_init = skcipher_walk_aead_decrypt,

			.crypt_chunk = crypto_aegis128_decrypt_chunk_simd };



	crypto_aegis128_crypt(req, &tag, cryptlen, ops);

	crypto_aegis128_crypt(req, &tag, cryptlen, &ops);



	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;

}
@@ -460,9 +429,6 @@ static struct aead_alg crypto_aegis128_alg = { 


static int __init crypto_aegis128_module_init(void)

{

	if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD))

		have_simd = crypto_aegis128_have_simd();



	return crypto_register_aead(&crypto_aegis128_alg);

}