crypto: arm - add support for SHA1 using ARMv8 Crypto Instructions (864cbeed) · Commits · e / devices / android_kernel_oneplus_sm7250

arch/arm/crypto/Kconfig

+10 −0

Original line number	Diff line number	Diff line
		@@ -27,6 +27,16 @@ config CRYPTO_SHA1_ARM_NEON
		using optimized ARM NEON assembly, when NEON instructions are
		available.

		config CRYPTO_SHA1_ARM_CE
		tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)"
		depends on KERNEL_MODE_NEON
		select CRYPTO_SHA1_ARM
		select CRYPTO_SHA1
		select CRYPTO_HASH
		help
		SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
		using special ARMv8 Crypto Extensions.

		config CRYPTO_SHA512_ARM_NEON
		tristate "SHA384 and SHA512 digest algorithm (ARM NEON)"
		depends on KERNEL_MODE_NEON

arch/arm/crypto/Makefile

+2 −0

Original line number	Diff line number	Diff line
		@@ -7,12 +7,14 @@ obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
		obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
		obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
		obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o
		obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o

		aes-arm-y := aes-armv4.o aes_glue.o
		aes-arm-bs-y := aesbs-core.o aesbs-glue.o
		sha1-arm-y := sha1-armv4-large.o sha1_glue.o
		sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
		sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o
		sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o

		quiet_cmd_perl = PERL $@
		cmd_perl = $(PERL) $(<) > $(@)

arch/arm/crypto/sha1-ce-core.S

0 → 100644

+134 −0

Original line number	Diff line number	Diff line
		/*
		* sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
		*
		* Copyright (C) 2015 Linaro Ltd.
		* Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*/

		#include <linux/linkage.h>
		#include <asm/assembler.h>

		.text
		.fpu crypto-neon-fp-armv8

		k0 .req q0
		k1 .req q1
		k2 .req q2
		k3 .req q3

		ta0 .req q4
		ta1 .req q5
		tb0 .req q5
		tb1 .req q4

		dga .req q6
		dgb .req q7
		dgbs .req s28

		dg0 .req q12
		dg1a0 .req q13
		dg1a1 .req q14
		dg1b0 .req q14
		dg1b1 .req q13

		.macro add_only, op, ev, rc, s0, dg1
		.ifnb \s0
		vadd.u32 tb\ev, q\s0, \rc
		.endif
		sha1h.32 dg1b\ev, dg0
		.ifb \dg1
		sha1\op\().32 dg0, dg1a\ev, ta\ev
		.else
		sha1\op\().32 dg0, \dg1, ta\ev
		.endif
		.endm

		.macro add_update, op, ev, rc, s0, s1, s2, s3, dg1
		sha1su0.32 q\s0, q\s1, q\s2
		add_only \op, \ev, \rc, \s1, \dg1
		sha1su1.32 q\s0, q\s3
		.endm

		.align 6
		.Lsha1_rcon:
		.word 0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999
		.word 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1
		.word 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc
		.word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6

		/*
		* void sha1_ce_transform(int blocks, u8 const src, u32 state,
		* u8 *head);
		*/
		ENTRY(sha1_ce_transform)
		/* load round constants */
		adr ip, .Lsha1_rcon
		vld1.32 {k0-k1}, [ip, :128]!
		vld1.32 {k2-k3}, [ip, :128]

		/* load state */
		vld1.32 {dga}, [r2]
		vldr dgbs, [r2, #16]

		/* load partial input (if supplied) */
		teq r3, #0
		beq 0f
		vld1.32 {q8-q9}, [r3]!
		vld1.32 {q10-q11}, [r3]
		teq r0, #0
		b 1f

		/* load input */
		0: vld1.32 {q8-q9}, [r1]!
		vld1.32 {q10-q11}, [r1]!
		subs r0, r0, #1

		1:
		#ifndef CONFIG_CPU_BIG_ENDIAN
		vrev32.8 q8, q8
		vrev32.8 q9, q9
		vrev32.8 q10, q10
		vrev32.8 q11, q11
		#endif

		vadd.u32 ta0, q8, k0
		vmov dg0, dga

		add_update c, 0, k0, 8, 9, 10, 11, dgb
		add_update c, 1, k0, 9, 10, 11, 8
		add_update c, 0, k0, 10, 11, 8, 9
		add_update c, 1, k0, 11, 8, 9, 10
		add_update c, 0, k1, 8, 9, 10, 11

		add_update p, 1, k1, 9, 10, 11, 8
		add_update p, 0, k1, 10, 11, 8, 9
		add_update p, 1, k1, 11, 8, 9, 10
		add_update p, 0, k1, 8, 9, 10, 11
		add_update p, 1, k2, 9, 10, 11, 8

		add_update m, 0, k2, 10, 11, 8, 9
		add_update m, 1, k2, 11, 8, 9, 10
		add_update m, 0, k2, 8, 9, 10, 11
		add_update m, 1, k2, 9, 10, 11, 8
		add_update m, 0, k3, 10, 11, 8, 9

		add_update p, 1, k3, 11, 8, 9, 10
		add_only p, 0, k3, 9
		add_only p, 1, k3, 10
		add_only p, 0, k3, 11
		add_only p, 1

		/* update state */
		vadd.u32 dga, dga, dg0
		vadd.u32 dgb, dgb, dg1a0
		bne 0b

		/* store new state */
		vst1.32 {dga}, [r2]
		vstr dgbs, [r2, #16]
		bx lr
		ENDPROC(sha1_ce_transform)

arch/arm/crypto/sha1-ce-glue.c

0 → 100644

+150 −0

Original line number	Diff line number	Diff line
		/*
		* sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
		*
		* Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*/

		#include <crypto/internal/hash.h>
		#include <crypto/sha.h>
		#include <linux/crypto.h>
		#include <linux/module.h>

		#include <asm/crypto/sha1.h>
		#include <asm/hwcap.h>
		#include <asm/neon.h>
		#include <asm/simd.h>
		#include <asm/unaligned.h>

		MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
		MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
		MODULE_LICENSE("GPL v2");

		asmlinkage void sha1_ce_transform(int blocks, u8 const src, u32 state,
		u8 *head);

		static int sha1_init(struct shash_desc *desc)
		{
		struct sha1_state *sctx = shash_desc_ctx(desc);

		*sctx = (struct sha1_state){
		.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
		};
		return 0;
		}

		static int sha1_update(struct shash_desc desc, const u8 data,
		unsigned int len)
		{
		struct sha1_state *sctx = shash_desc_ctx(desc);
		unsigned int partial;

		if (!may_use_simd())
		return sha1_update_arm(desc, data, len);

		partial = sctx->count % SHA1_BLOCK_SIZE;
		sctx->count += len;

		if ((partial + len) >= SHA1_BLOCK_SIZE) {
		int blocks;

		if (partial) {
		int p = SHA1_BLOCK_SIZE - partial;

		memcpy(sctx->buffer + partial, data, p);
		data += p;
		len -= p;
		}

		blocks = len / SHA1_BLOCK_SIZE;
		len %= SHA1_BLOCK_SIZE;

		kernel_neon_begin();
		sha1_ce_transform(blocks, data, sctx->state,
		partial ? sctx->buffer : NULL);
		kernel_neon_end();

		data += blocks * SHA1_BLOCK_SIZE;
		partial = 0;
		}
		if (len)
		memcpy(sctx->buffer + partial, data, len);
		return 0;
		}

		static int sha1_final(struct shash_desc desc, u8 out)
		{
		static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };

		struct sha1_state *sctx = shash_desc_ctx(desc);
		__be64 bits = cpu_to_be64(sctx->count << 3);
		__be32 dst = (__be32 )out;
		int i;

		u32 padlen = SHA1_BLOCK_SIZE
		- ((sctx->count + sizeof(bits)) % SHA1_BLOCK_SIZE);

		sha1_update(desc, padding, padlen);
		sha1_update(desc, (const u8 *)&bits, sizeof(bits));

		for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++)
		put_unaligned_be32(sctx->state[i], dst++);

		*sctx = (struct sha1_state){};
		return 0;
		}

		static int sha1_export(struct shash_desc desc, void out)
		{
		struct sha1_state *sctx = shash_desc_ctx(desc);
		struct sha1_state *dst = out;

		dst = sctx;
		return 0;
		}

		static int sha1_import(struct shash_desc desc, const void in)
		{
		struct sha1_state *sctx = shash_desc_ctx(desc);
		struct sha1_state const *src = in;

		sctx = src;
		return 0;
		}

		static struct shash_alg alg = {
		.init = sha1_init,
		.update = sha1_update,
		.final = sha1_final,
		.export = sha1_export,
		.import = sha1_import,
		.descsize = sizeof(struct sha1_state),
		.digestsize = SHA1_DIGEST_SIZE,
		.statesize = sizeof(struct sha1_state),
		.base = {
		.cra_name = "sha1",
		.cra_driver_name = "sha1-ce",
		.cra_priority = 200,
		.cra_flags = CRYPTO_ALG_TYPE_SHASH,
		.cra_blocksize = SHA1_BLOCK_SIZE,
		.cra_module = THIS_MODULE,
		}
		};

		static int __init sha1_ce_mod_init(void)
		{
		if (!(elf_hwcap2 & HWCAP2_SHA1))
		return -ENODEV;
		return crypto_register_shash(&alg);
		}

		static void __exit sha1_ce_mod_fini(void)
		{
		crypto_unregister_shash(&alg);
		}

		module_init(sha1_ce_mod_init);
		module_exit(sha1_ce_mod_fini);