Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7918ecef authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

crypto: arm64/sha2 - integrate OpenSSL implementations of SHA256/SHA512 



This integrates both the accelerated scalar and the NEON implementations
of SHA-224/256 as well as SHA-384/512 from the OpenSSL project.

Relative performance compared to the respective generic C versions:

                 |  SHA256-scalar  | SHA256-NEON* |  SHA512  |
     ------------+-----------------+--------------+----------+
     Cortex-A53  |      1.63x      |     1.63x    |   2.34x  |
     Cortex-A57  |      1.43x      |     1.59x    |   1.95x  |
     Cortex-A73  |      1.26x      |     1.56x    |     ?    |

The core crypto code was authored by Andy Polyakov of the OpenSSL
project, in collaboration with whom the upstream code was adapted so
that this module can be built from the same version of sha512-armv8.pl.

The version in this patch was taken from OpenSSL commit 32bbb62ea634
("sha/asm/sha512-armv8.pl: fix big-endian support in __KERNEL__ case.")

* The core SHA algorithm is fundamentally sequential, but there is a
  secondary transformation involved, called the schedule update, which
  can be performed independently. The NEON version of SHA-224/SHA-256
  only implements this part of the algorithm using NEON instructions,
  the sequential part is always done using scalar instructions.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent ed424bb3

arch/arm64/crypto/Kconfig

+8 −0
Original line number Diff line number Diff line
@@ -8,6 +8,14 @@ menuconfig ARM64_CRYPTO 


if ARM64_CRYPTO



config CRYPTO_SHA256_ARM64

	tristate "SHA-224/SHA-256 digest algorithm for arm64"

	select CRYPTO_HASH



config CRYPTO_SHA512_ARM64

	tristate "SHA-384/SHA-512 digest algorithm for arm64"

	select CRYPTO_HASH



config CRYPTO_SHA1_ARM64_CE

	tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)"

	depends on ARM64 && KERNEL_MODE_NEON

arch/arm64/crypto/Makefile

+17 −0
Original line number Diff line number Diff line
@@ -29,6 +29,12 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o 
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o

aes-neon-blk-y := aes-glue-neon.o aes-neon.o



obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o

sha256-arm64-y := sha256-glue.o sha256-core.o



obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o

sha512-arm64-y := sha512-glue.o sha512-core.o



AFLAGS_aes-ce.o		:= -DINTERLEAVE=4

AFLAGS_aes-neon.o	:= -DINTERLEAVE=4
@@ -40,3 +46,14 @@ CFLAGS_crc32-arm64.o := -mcpu=generic+crc 


$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE

	$(call if_changed_rule,cc_o_c)



quiet_cmd_perlasm = PERLASM $@

      cmd_perlasm = $(PERL) $(<) void $(@)



$(src)/sha256-core.S_shipped: $(src)/sha512-armv8.pl

	$(call cmd,perlasm)



$(src)/sha512-core.S_shipped: $(src)/sha512-armv8.pl

	$(call cmd,perlasm)



.PRECIOUS: $(obj)/sha256-core.S $(obj)/sha512-core.S

arch/arm64/crypto/sha256-core.S_shipped

0 → 100644
+2061 −0

File added.

Preview size limit exceeded, changes collapsed.

arch/arm64/crypto/sha256-glue.c

0 → 100644
+185 −0
Original line number Diff line number Diff line 
/*

 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64

 *

 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License as published by the Free

 * Software Foundation; either version 2 of the License, or (at your option)

 * any later version.

 *

 */



#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>

#include <crypto/internal/hash.h>

#include <crypto/sha.h>

#include <crypto/sha256_base.h>

#include <linux/cryptohash.h>

#include <linux/types.h>

#include <linux/string.h>



MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");

MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha224");

MODULE_ALIAS_CRYPTO("sha256");



asmlinkage void sha256_block_data_order(u32 *digest, const void *data,

					unsigned int num_blks);



asmlinkage void sha256_block_neon(u32 *digest, const void *data,

				  unsigned int num_blks);



static int sha256_update(struct shash_desc *desc, const u8 *data,

			 unsigned int len)

{

	return sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_data_order);

}



static int sha256_finup(struct shash_desc *desc, const u8 *data,

			unsigned int len, u8 *out)

{

	if (len)

		sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_data_order);

	sha256_base_do_finalize(desc,

				(sha256_block_fn *)sha256_block_data_order);



	return sha256_base_finish(desc, out);

}



static int sha256_final(struct shash_desc *desc, u8 *out)

{

	return sha256_finup(desc, NULL, 0, out);

}



static struct shash_alg algs[] = { {

	.digestsize		= SHA256_DIGEST_SIZE,

	.init			= sha256_base_init,

	.update			= sha256_update,

	.final			= sha256_final,

	.finup			= sha256_finup,

	.descsize		= sizeof(struct sha256_state),

	.base.cra_name		= "sha256",

	.base.cra_driver_name	= "sha256-arm64",

	.base.cra_priority	= 100,

	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,

	.base.cra_blocksize	= SHA256_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.digestsize		= SHA224_DIGEST_SIZE,

	.init			= sha224_base_init,

	.update			= sha256_update,

	.final			= sha256_final,

	.finup			= sha256_finup,

	.descsize		= sizeof(struct sha256_state),

	.base.cra_name		= "sha224",

	.base.cra_driver_name	= "sha224-arm64",

	.base.cra_priority	= 100,

	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,

	.base.cra_blocksize	= SHA224_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int sha256_update_neon(struct shash_desc *desc, const u8 *data,

			      unsigned int len)

{

	/*

	 * Stacking and unstacking a substantial slice of the NEON register

	 * file may significantly affect performance for small updates when

	 * executing in interrupt context, so fall back to the scalar code

	 * in that case.

	 */

	if (!may_use_simd())

		return sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_data_order);



	kernel_neon_begin();

	sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_neon);

	kernel_neon_end();



	return 0;

}



static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,

			     unsigned int len, u8 *out)

{

	if (!may_use_simd()) {

		if (len)

			sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_data_order);

		sha256_base_do_finalize(desc,

				(sha256_block_fn *)sha256_block_data_order);

	} else {

		kernel_neon_begin();

		if (len)

			sha256_base_do_update(desc, data, len,

				(sha256_block_fn *)sha256_block_neon);

		sha256_base_do_finalize(desc,

				(sha256_block_fn *)sha256_block_neon);

		kernel_neon_end();

	}

	return sha256_base_finish(desc, out);

}



static int sha256_final_neon(struct shash_desc *desc, u8 *out)

{

	return sha256_finup_neon(desc, NULL, 0, out);

}



static struct shash_alg neon_algs[] = { {

	.digestsize		= SHA256_DIGEST_SIZE,

	.init			= sha256_base_init,

	.update			= sha256_update_neon,

	.final			= sha256_final_neon,

	.finup			= sha256_finup_neon,

	.descsize		= sizeof(struct sha256_state),

	.base.cra_name		= "sha256",

	.base.cra_driver_name	= "sha256-arm64-neon",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,

	.base.cra_blocksize	= SHA256_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.digestsize		= SHA224_DIGEST_SIZE,

	.init			= sha224_base_init,

	.update			= sha256_update_neon,

	.final			= sha256_final_neon,

	.finup			= sha256_finup_neon,

	.descsize		= sizeof(struct sha256_state),

	.base.cra_name		= "sha224",

	.base.cra_driver_name	= "sha224-arm64-neon",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,

	.base.cra_blocksize	= SHA224_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int __init sha256_mod_init(void)

{

	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));

	if (ret)

		return ret;



	if (elf_hwcap & HWCAP_ASIMD) {

		ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));

		if (ret)

			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

	}

	return ret;

}



static void __exit sha256_mod_fini(void)

{

	if (elf_hwcap & HWCAP_ASIMD)

		crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));

	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

}



module_init(sha256_mod_init);

module_exit(sha256_mod_fini);

arch/arm64/crypto/sha512-armv8.pl

0 → 100644
+778 −0

File added.

Preview size limit exceeded, changes collapsed.