BACKPORT: crypto: arm - add support for SHA-224/256 using ARMv8 Crypto Extensions (458359bf) · Commits · e / devices / android_kernel_xiaomi_markw

arch/arm/crypto/Makefile

+2 −0

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
		obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
		obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
		obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o
		obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o

		aes-arm-y := aes-armv4.o aes_glue.o
		aes-arm-bs-y := aesbs-core.o aesbs-glue.o
		@@ -18,6 +19,7 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
		sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y)
		sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o
		aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
		sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o

		quiet_cmd_perl = PERL $@
		cmd_perl = $(PERL) $(<) > $(@)

arch/arm/crypto/sha2-ce-core.S

0 → 100644

+134 −0

Original line number	Diff line number	Diff line
		/*
		* sha2-ce-core.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions
		*
		* Copyright (C) 2015 Linaro Ltd.
		* Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*/

		#include <linux/linkage.h>
		#include <asm/assembler.h>

		.text
		.fpu crypto-neon-fp-armv8

		k0 .req q7
		k1 .req q8
		rk .req r3

		ta0 .req q9
		ta1 .req q10
		tb0 .req q10
		tb1 .req q9

		dga .req q11
		dgb .req q12

		dg0 .req q13
		dg1 .req q14
		dg2 .req q15

		.macro add_only, ev, s0
		vmov dg2, dg0
		.ifnb \s0
		vld1.32 {k\ev}, [rk, :128]!
		.endif
		sha256h.32 dg0, dg1, tb\ev
		sha256h2.32 dg1, dg2, tb\ev
		.ifnb \s0
		vadd.u32 ta\ev, q\s0, k\ev
		.endif
		.endm

		.macro add_update, ev, s0, s1, s2, s3
		sha256su0.32 q\s0, q\s1
		add_only \ev, \s1
		sha256su1.32 q\s0, q\s2, q\s3
		.endm

		.align 6
		.Lsha256_rcon:
		.word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
		.word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
		.word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
		.word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
		.word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
		.word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
		.word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
		.word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
		.word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
		.word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
		.word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
		.word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
		.word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
		.word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
		.word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
		.word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2

		/*
		* void sha2_ce_transform(int blocks, u8 const src, u32 state,
		* u8 *head);
		*/
		ENTRY(sha2_ce_transform)
		/* load state */
		vld1.32 {dga-dgb}, [r2]

		/* load partial input (if supplied) */
		teq r3, #0
		beq 0f
		vld1.32 {q0-q1}, [r3]!
		vld1.32 {q2-q3}, [r3]
		teq r0, #0
		b 1f

		/* load input */
		0: vld1.32 {q0-q1}, [r1]!
		vld1.32 {q2-q3}, [r1]!
		subs r0, r0, #1

		1:
		#ifndef CONFIG_CPU_BIG_ENDIAN
		vrev32.8 q0, q0
		vrev32.8 q1, q1
		vrev32.8 q2, q2
		vrev32.8 q3, q3
		#endif

		/* load first round constant */
		adr rk, .Lsha256_rcon
		vld1.32 {k0}, [rk, :128]!

		vadd.u32 ta0, q0, k0
		vmov dg0, dga
		vmov dg1, dgb

		add_update 1, 0, 1, 2, 3
		add_update 0, 1, 2, 3, 0
		add_update 1, 2, 3, 0, 1
		add_update 0, 3, 0, 1, 2
		add_update 1, 0, 1, 2, 3
		add_update 0, 1, 2, 3, 0
		add_update 1, 2, 3, 0, 1
		add_update 0, 3, 0, 1, 2
		add_update 1, 0, 1, 2, 3
		add_update 0, 1, 2, 3, 0
		add_update 1, 2, 3, 0, 1
		add_update 0, 3, 0, 1, 2

		add_only 1, 1
		add_only 0, 2
		add_only 1, 3
		add_only 0

		/* update state */
		vadd.u32 dga, dga, dg0
		vadd.u32 dgb, dgb, dg1
		bne 0b

		/* store new state */
		vst1.32 {dga-dgb}, [r2]
		bx lr
		ENDPROC(sha2_ce_transform)

arch/arm/crypto/sha2-ce-glue.c

0 → 100644

+203 −0

Original line number	Diff line number	Diff line
		/*
		* sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
		*
		* Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*/

		#include <crypto/internal/hash.h>
		#include <crypto/sha.h>
		#include <linux/crypto.h>
		#include <linux/module.h>

		#include <asm/hwcap.h>
		#include <asm/simd.h>
		#include <asm/neon.h>
		#include <asm/unaligned.h>

		MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
		MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
		MODULE_LICENSE("GPL v2");

		asmlinkage void sha2_ce_transform(int blocks, u8 const src, u32 state,
		u8 *head);

		static int sha224_init(struct shash_desc *desc)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);

		*sctx = (struct sha256_state){
		.state = {
		SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
		SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
		}
		};
		return 0;
		}

		static int sha256_init(struct shash_desc *desc)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);

		*sctx = (struct sha256_state){
		.state = {
		SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
		SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
		}
		};
		return 0;
		}

		static int sha2_update(struct shash_desc desc, const u8 data,
		unsigned int len)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);
		unsigned int partial;

		if (!may_use_simd())
		return crypto_sha256_update(desc, data, len);

		partial = sctx->count % SHA256_BLOCK_SIZE;
		sctx->count += len;

		if ((partial + len) >= SHA256_BLOCK_SIZE) {
		int blocks;

		if (partial) {
		int p = SHA256_BLOCK_SIZE - partial;

		memcpy(sctx->buf + partial, data, p);
		data += p;
		len -= p;
		}

		blocks = len / SHA256_BLOCK_SIZE;
		len %= SHA256_BLOCK_SIZE;

		kernel_neon_begin();
		sha2_ce_transform(blocks, data, sctx->state,
		partial ? sctx->buf : NULL);
		kernel_neon_end();

		data += blocks * SHA256_BLOCK_SIZE;
		partial = 0;
		}
		if (len)
		memcpy(sctx->buf + partial, data, len);
		return 0;
		}

		static void sha2_final(struct shash_desc *desc)
		{
		static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, };

		struct sha256_state *sctx = shash_desc_ctx(desc);
		__be64 bits = cpu_to_be64(sctx->count << 3);
		u32 padlen = SHA256_BLOCK_SIZE
		- ((sctx->count + sizeof(bits)) % SHA256_BLOCK_SIZE);

		sha2_update(desc, padding, padlen);
		sha2_update(desc, (const u8 *)&bits, sizeof(bits));
		}

		static int sha224_final(struct shash_desc desc, u8 out)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);
		__be32 dst = (__be32 )out;
		int i;

		sha2_final(desc);

		for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(__be32); i++)
		put_unaligned_be32(sctx->state[i], dst++);

		*sctx = (struct sha256_state){};
		return 0;
		}

		static int sha256_final(struct shash_desc desc, u8 out)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);
		__be32 dst = (__be32 )out;
		int i;

		sha2_final(desc);

		for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(__be32); i++)
		put_unaligned_be32(sctx->state[i], dst++);

		*sctx = (struct sha256_state){};
		return 0;
		}

		static int sha2_export(struct shash_desc desc, void out)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);
		struct sha256_state *dst = out;

		dst = sctx;
		return 0;
		}

		static int sha2_import(struct shash_desc desc, const void in)
		{
		struct sha256_state *sctx = shash_desc_ctx(desc);
		struct sha256_state const *src = in;

		sctx = src;
		return 0;
		}

		static struct shash_alg algs[] = { {
		.init = sha224_init,
		.update = sha2_update,
		.final = sha224_final,
		.export = sha2_export,
		.import = sha2_import,
		.descsize = sizeof(struct sha256_state),
		.digestsize = SHA224_DIGEST_SIZE,
		.statesize = sizeof(struct sha256_state),
		.base = {
		.cra_name = "sha224",
		.cra_driver_name = "sha224-ce",
		.cra_priority = 300,
		.cra_flags = CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize = SHA256_BLOCK_SIZE,
		.cra_module = THIS_MODULE,
		}
		}, {
		.init = sha256_init,
		.update = sha2_update,
		.final = sha256_final,
		.export = sha2_export,
		.import = sha2_import,
		.descsize = sizeof(struct sha256_state),
		.digestsize = SHA256_DIGEST_SIZE,
		.statesize = sizeof(struct sha256_state),
		.base = {
		.cra_name = "sha256",
		.cra_driver_name = "sha256-ce",
		.cra_priority = 300,
		.cra_flags = CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize = SHA256_BLOCK_SIZE,
		.cra_module = THIS_MODULE,
		}
		} };

		static int __init sha2_ce_mod_init(void)
		{
		if (!(elf_hwcap2 & HWCAP2_SHA2))
		return -ENODEV;
		return crypto_register_shashes(algs, ARRAY_SIZE(algs));
		}

		static void __exit sha2_ce_mod_fini(void)
		{
		crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
		}

		module_init(sha2_ce_mod_init);
		module_exit(sha2_ce_mod_fini);

crypto/Kconfig

+9 −0

Original line number	Diff line number	Diff line
		@@ -611,6 +611,15 @@ config CRYPTO_SHA256_SPARC64
		SHA-256 secure hash standard (DFIPS 180-2) implemented
		using sparc64 crypto instructions, when available.

		config CRYPTO_SHA2_ARM_CE
		tristate "SHA-224/256 digest algorithm (ARM v8 Crypto Extensions)"
		depends on ARM && KERNEL_MODE_NEON
		select CRYPTO_SHA256
		select CRYPTO_HASH
		help
		SHA-256 secure hash standard (DFIPS 180-2) implemented
		using special ARMv8 Crypto Extensions.

		config CRYPTO_SHA512
		tristate "SHA384 and SHA512 digest algorithms"
		select CRYPTO_HASH