crypto: x86 - Add optimized MORUS implementations (6ecc9d9f) · Commits · e / devices / android_kernel_fairphone_FP4

arch/x86/crypto/Makefile

+10 −0

Original line number	Diff line number	Diff line
		@@ -42,6 +42,9 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
		obj-$(CONFIG_CRYPTO_AEGIS128L_AESNI_SSE2) += aegis128l-aesni.o
		obj-$(CONFIG_CRYPTO_AEGIS256_AESNI_SSE2) += aegis256-aesni.o

		obj-$(CONFIG_CRYPTO_MORUS640_SSE2) += morus640-sse2.o
		obj-$(CONFIG_CRYPTO_MORUS1280_SSE2) += morus1280-sse2.o

		# These modules require assembler to support AVX.
		ifeq ($(avx_supported),yes)
		obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += \
		@@ -59,6 +62,8 @@ ifeq ($(avx2_supported),yes)
		obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/
		obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/
		obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/

		obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o
		endif

		aes-i586-y := aes-i586-asm_32.o aes_glue.o
		@@ -80,6 +85,9 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
		aegis128l-aesni-y := aegis128l-aesni-asm.o aegis128l-aesni-glue.o
		aegis256-aesni-y := aegis256-aesni-asm.o aegis256-aesni-glue.o

		morus640-sse2-y := morus640-sse2-asm.o morus640-sse2-glue.o
		morus1280-sse2-y := morus1280-sse2-asm.o morus1280-sse2-glue.o

		ifeq ($(avx_supported),yes)
		camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \
		camellia_aesni_avx_glue.o
		@@ -95,6 +103,8 @@ ifeq ($(avx2_supported),yes)
		camellia-aesni-avx2-y := camellia-aesni-avx2-asm_64.o camellia_aesni_avx2_glue.o
		chacha20-x86_64-y += chacha20-avx2-x86_64.o
		serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o

		morus1280-avx2-y := morus1280-avx2-asm.o morus1280-avx2-glue.o
		endif

		aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o

arch/x86/crypto/morus1280-avx2-asm.S

0 → 100644

+621 −0

Original line number	Diff line number	Diff line
		/*
		* AVX2 implementation of MORUS-1280
		*
		* Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
		* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify it
		* under the terms of the GNU General Public License version 2 as published
		* by the Free Software Foundation.
		*/

		#include <linux/linkage.h>
		#include <asm/frame.h>

		#define SHUFFLE_MASK(i0, i1, i2, i3) \
		(i0 \| (i1 << 2) \| (i2 << 4) \| (i3 << 6))

		#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
		#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
		#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)

		#define STATE0 %ymm0
		#define STATE0_LOW %xmm0
		#define STATE1 %ymm1
		#define STATE2 %ymm2
		#define STATE3 %ymm3
		#define STATE4 %ymm4
		#define KEY %ymm5
		#define MSG %ymm5
		#define MSG_LOW %xmm5
		#define T0 %ymm6
		#define T0_LOW %xmm6
		#define T1 %ymm7

		.section .rodata.cst32.morus1280_const, "aM", @progbits, 32
		.align 32
		.Lmorus1280_const:
		.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
		.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
		.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
		.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd

		.section .rodata.cst32.morus1280_counter, "aM", @progbits, 32
		.align 32
		.Lmorus1280_counter:
		.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
		.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
		.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
		.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f

		.text

		.macro morus1280_round s0, s1, s2, s3, s4, b, w
		vpand \s1, \s2, T0
		vpxor T0, \s0, \s0
		vpxor \s3, \s0, \s0
		vpsllq $\b, \s0, T0
		vpsrlq $(64 - \b), \s0, \s0
		vpxor T0, \s0, \s0
		vpermq $\w, \s3, \s3
		.endm

		/*
		* __morus1280_update: internal ABI
		* input:
		* STATE[0-4] - input state
		* MSG - message block
		* output:
		* STATE[0-4] - output state
		* changed:
		* T0
		*/
		__morus1280_update:
		morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
		vpxor MSG, STATE1, STATE1
		morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
		vpxor MSG, STATE2, STATE2
		morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
		vpxor MSG, STATE3, STATE3
		morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2, 7, MASK2
		vpxor MSG, STATE4, STATE4
		morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3, 4, MASK1
		ret
		ENDPROC(__morus1280_update)

		/*
		* __morus1280_update_zero: internal ABI
		* input:
		* STATE[0-4] - input state
		* output:
		* STATE[0-4] - output state
		* changed:
		* T0
		*/
		__morus1280_update_zero:
		morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
		morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
		morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
		morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2, 7, MASK2
		morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3, 4, MASK1
		ret
		ENDPROC(__morus1280_update_zero)

		/*
		* __load_partial: internal ABI
		* input:
		* %rsi - src
		* %rcx - bytes
		* output:
		* MSG - message block
		* changed:
		* %r8
		* %r9
		*/
		__load_partial:
		xor %r9, %r9
		vpxor MSG, MSG, MSG

		mov %rcx, %r8
		and $0x1, %r8
		jz .Lld_partial_1

		mov %rcx, %r8
		and $0x1E, %r8
		add %rsi, %r8
		mov (%r8), %r9b

		.Lld_partial_1:
		mov %rcx, %r8
		and $0x2, %r8
		jz .Lld_partial_2

		mov %rcx, %r8
		and $0x1C, %r8
		add %rsi, %r8
		shl $16, %r9
		mov (%r8), %r9w

		.Lld_partial_2:
		mov %rcx, %r8
		and $0x4, %r8
		jz .Lld_partial_4

		mov %rcx, %r8
		and $0x18, %r8
		add %rsi, %r8
		shl $32, %r9
		mov (%r8), %r8d
		xor %r8, %r9

		.Lld_partial_4:
		movq %r9, MSG_LOW

		mov %rcx, %r8
		and $0x8, %r8
		jz .Lld_partial_8

		mov %rcx, %r8
		and $0x10, %r8
		add %rsi, %r8
		pshufd $MASK2, MSG_LOW, MSG_LOW
		pinsrq $0, (%r8), MSG_LOW

		.Lld_partial_8:
		mov %rcx, %r8
		and $0x10, %r8
		jz .Lld_partial_16

		vpermq $MASK2, MSG, MSG
		movdqu (%rsi), MSG_LOW

		.Lld_partial_16:
		ret
		ENDPROC(__load_partial)

		/*
		* __store_partial: internal ABI
		* input:
		* %rdx - dst
		* %rcx - bytes
		* output:
		* T0 - message block
		* changed:
		* %r8
		* %r9
		* %r10
		*/
		__store_partial:
		mov %rcx, %r8
		mov %rdx, %r9

		cmp $16, %r8
		jl .Lst_partial_16

		movdqu T0_LOW, (%r9)
		vpermq $MASK2, T0, T0

		sub $16, %r8
		add $16, %r9

		.Lst_partial_16:
		movq T0_LOW, %r10

		cmp $8, %r8
		jl .Lst_partial_8

		mov %r10, (%r9)
		pextrq $1, T0_LOW, %r10

		sub $8, %r8
		add $8, %r9

		.Lst_partial_8:
		cmp $4, %r8
		jl .Lst_partial_4

		mov %r10d, (%r9)
		shr $32, %r10

		sub $4, %r8
		add $4, %r9

		.Lst_partial_4:
		cmp $2, %r8
		jl .Lst_partial_2

		mov %r10w, (%r9)
		shr $16, %r10

		sub $2, %r8
		add $2, %r9

		.Lst_partial_2:
		cmp $1, %r8
		jl .Lst_partial_1

		mov %r10b, (%r9)

		.Lst_partial_1:
		ret
		ENDPROC(__store_partial)

		/*
		* void crypto_morus1280_avx2_init(void state, const void key,
		* const void *iv);
		*/
		ENTRY(crypto_morus1280_avx2_init)
		FRAME_BEGIN

		/* load IV: */
		vpxor STATE0, STATE0, STATE0
		movdqu (%rdx), STATE0_LOW
		/* load key: */
		vmovdqu (%rsi), KEY
		vmovdqa KEY, STATE1
		/* load all ones: */
		vpcmpeqd STATE2, STATE2, STATE2
		/* load all zeros: */
		vpxor STATE3, STATE3, STATE3
		/* load the constant: */
		vmovdqa .Lmorus1280_const, STATE4

		/* update 16 times with zero: */
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero
		call __morus1280_update_zero

		/* xor-in the key again after updates: */
		vpxor KEY, STATE1, STATE1

		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_init)

		/*
		* void crypto_morus1280_avx2_ad(void state, const void data,
		* unsigned int length);
		*/
		ENTRY(crypto_morus1280_avx2_ad)
		FRAME_BEGIN

		cmp $32, %rdx
		jb .Lad_out

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		mov %rsi, %r8
		and $0x1F, %r8
		jnz .Lad_u_loop

		.align 4
		.Lad_a_loop:
		vmovdqa (%rsi), MSG
		call __morus1280_update
		sub $32, %rdx
		add $32, %rsi
		cmp $32, %rdx
		jge .Lad_a_loop

		jmp .Lad_cont
		.align 4
		.Lad_u_loop:
		vmovdqu (%rsi), MSG
		call __morus1280_update
		sub $32, %rdx
		add $32, %rsi
		cmp $32, %rdx
		jge .Lad_u_loop

		.Lad_cont:
		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		.Lad_out:
		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_ad)

		/*
		* void crypto_morus1280_avx2_enc(void state, const void src, void *dst,
		* unsigned int length);
		*/
		ENTRY(crypto_morus1280_avx2_enc)
		FRAME_BEGIN

		cmp $32, %rcx
		jb .Lenc_out

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		mov %rsi, %r8
		or %rdx, %r8
		and $0x1F, %r8
		jnz .Lenc_u_loop

		.align 4
		.Lenc_a_loop:
		vmovdqa (%rsi), MSG
		vmovdqa MSG, T0
		vpxor STATE0, T0, T0
		vpermq $MASK3, STATE1, T1
		vpxor T1, T0, T0
		vpand STATE2, STATE3, T1
		vpxor T1, T0, T0
		vmovdqa T0, (%rdx)

		call __morus1280_update
		sub $32, %rcx
		add $32, %rsi
		add $32, %rdx
		cmp $32, %rcx
		jge .Lenc_a_loop

		jmp .Lenc_cont
		.align 4
		.Lenc_u_loop:
		vmovdqu (%rsi), MSG
		vmovdqa MSG, T0
		vpxor STATE0, T0, T0
		vpermq $MASK3, STATE1, T1
		vpxor T1, T0, T0
		vpand STATE2, STATE3, T1
		vpxor T1, T0, T0
		vmovdqu T0, (%rdx)

		call __morus1280_update
		sub $32, %rcx
		add $32, %rsi
		add $32, %rdx
		cmp $32, %rcx
		jge .Lenc_u_loop

		.Lenc_cont:
		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		.Lenc_out:
		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_enc)

		/*
		* void crypto_morus1280_avx2_enc_tail(void state, const void src, void *dst,
		* unsigned int length);
		*/
		ENTRY(crypto_morus1280_avx2_enc_tail)
		FRAME_BEGIN

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		/* encrypt message: */
		call __load_partial

		vmovdqa MSG, T0
		vpxor STATE0, T0, T0
		vpermq $MASK3, STATE1, T1
		vpxor T1, T0, T0
		vpand STATE2, STATE3, T1
		vpxor T1, T0, T0

		call __store_partial

		call __morus1280_update

		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		FRAME_END
		ENDPROC(crypto_morus1280_avx2_enc_tail)

		/*
		* void crypto_morus1280_avx2_dec(void state, const void src, void *dst,
		* unsigned int length);
		*/
		ENTRY(crypto_morus1280_avx2_dec)
		FRAME_BEGIN

		cmp $32, %rcx
		jb .Ldec_out

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		mov %rsi, %r8
		or %rdx, %r8
		and $0x1F, %r8
		jnz .Ldec_u_loop

		.align 4
		.Ldec_a_loop:
		vmovdqa (%rsi), MSG
		vpxor STATE0, MSG, MSG
		vpermq $MASK3, STATE1, T0
		vpxor T0, MSG, MSG
		vpand STATE2, STATE3, T0
		vpxor T0, MSG, MSG
		vmovdqa MSG, (%rdx)

		call __morus1280_update
		sub $32, %rcx
		add $32, %rsi
		add $32, %rdx
		cmp $32, %rcx
		jge .Ldec_a_loop

		jmp .Ldec_cont
		.align 4
		.Ldec_u_loop:
		vmovdqu (%rsi), MSG
		vpxor STATE0, MSG, MSG
		vpermq $MASK3, STATE1, T0
		vpxor T0, MSG, MSG
		vpand STATE2, STATE3, T0
		vpxor T0, MSG, MSG
		vmovdqu MSG, (%rdx)

		call __morus1280_update
		sub $32, %rcx
		add $32, %rsi
		add $32, %rdx
		cmp $32, %rcx
		jge .Ldec_u_loop

		.Ldec_cont:
		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		.Ldec_out:
		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_dec)

		/*
		* void crypto_morus1280_avx2_dec_tail(void state, const void src, void *dst,
		* unsigned int length);
		*/
		ENTRY(crypto_morus1280_avx2_dec_tail)
		FRAME_BEGIN

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		/* decrypt message: */
		call __load_partial

		vpxor STATE0, MSG, MSG
		vpermq $MASK3, STATE1, T0
		vpxor T0, MSG, MSG
		vpand STATE2, STATE3, T0
		vpxor T0, MSG, MSG
		vmovdqa MSG, T0

		call __store_partial

		/* mask with byte count: */
		movq %rcx, T0_LOW
		vpbroadcastb T0_LOW, T0
		vmovdqa .Lmorus1280_counter, T1
		vpcmpgtb T1, T0, T0
		vpand T0, MSG, MSG

		call __morus1280_update

		/* store the state: */
		vmovdqu STATE0, (0 * 32)(%rdi)
		vmovdqu STATE1, (1 * 32)(%rdi)
		vmovdqu STATE2, (2 * 32)(%rdi)
		vmovdqu STATE3, (3 * 32)(%rdi)
		vmovdqu STATE4, (4 * 32)(%rdi)

		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_dec_tail)

		/*
		* void crypto_morus1280_avx2_final(void state, void tag_xor,
		* u64 assoclen, u64 cryptlen);
		*/
		ENTRY(crypto_morus1280_avx2_final)
		FRAME_BEGIN

		/* load the state: */
		vmovdqu (0 * 32)(%rdi), STATE0
		vmovdqu (1 * 32)(%rdi), STATE1
		vmovdqu (2 * 32)(%rdi), STATE2
		vmovdqu (3 * 32)(%rdi), STATE3
		vmovdqu (4 * 32)(%rdi), STATE4

		/* xor state[0] into state[4]: */
		vpxor STATE0, STATE4, STATE4

		/* prepare length block: */
		vpxor MSG, MSG, MSG
		vpinsrq $0, %rdx, MSG_LOW, MSG_LOW
		vpinsrq $1, %rcx, MSG_LOW, MSG_LOW
		vpsllq $3, MSG, MSG /* multiply by 8 (to get bit count) */

		/* update state: */
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update
		call __morus1280_update

		/* xor tag: */
		vmovdqu (%rsi), MSG

		vpxor STATE0, MSG, MSG
		vpermq $MASK3, STATE1, T0
		vpxor T0, MSG, MSG
		vpand STATE2, STATE3, T0
		vpxor T0, MSG, MSG
		vmovdqu MSG, (%rsi)

		FRAME_END
		ret
		ENDPROC(crypto_morus1280_avx2_final)

arch/x86/crypto/morus1280-avx2-glue.c

0 → 100644

+68 −0

Original line number	Diff line number	Diff line
		/*
		* The MORUS-1280 Authenticated-Encryption Algorithm
		* Glue for AVX2 implementation
		*
		* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
		* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify it
		* under the terms of the GNU General Public License as published by the Free
		* Software Foundation; either version 2 of the License, or (at your option)
		* any later version.
		*/

		#include <crypto/internal/aead.h>
		#include <crypto/morus1280_glue.h>
		#include <linux/module.h>
		#include <asm/fpu/api.h>
		#include <asm/cpu_device_id.h>

		asmlinkage void crypto_morus1280_avx2_init(void state, const void key,
		const void *iv);
		asmlinkage void crypto_morus1280_avx2_ad(void state, const void data,
		unsigned int length);

		asmlinkage void crypto_morus1280_avx2_enc(void state, const void src,
		void *dst, unsigned int length);
		asmlinkage void crypto_morus1280_avx2_dec(void state, const void src,
		void *dst, unsigned int length);

		asmlinkage void crypto_morus1280_avx2_enc_tail(void state, const void src,
		void *dst, unsigned int length);
		asmlinkage void crypto_morus1280_avx2_dec_tail(void state, const void src,
		void *dst, unsigned int length);

		asmlinkage void crypto_morus1280_avx2_final(void state, void tag_xor,
		u64 assoclen, u64 cryptlen);

		MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400);

		static const struct x86_cpu_id avx2_cpu_id[] = {
		X86_FEATURE_MATCH(X86_FEATURE_AVX2),
		{}
		};
		MODULE_DEVICE_TABLE(x86cpu, avx2_cpu_id);

		static int __init crypto_morus1280_avx2_module_init(void)
		{
		if (!x86_match_cpu(avx2_cpu_id))
		return -ENODEV;

		return crypto_register_aeads(crypto_morus1280_avx2_algs,
		ARRAY_SIZE(crypto_morus1280_avx2_algs));
		}

		static void __exit crypto_morus1280_avx2_module_exit(void)
		{
		crypto_unregister_aeads(crypto_morus1280_avx2_algs,
		ARRAY_SIZE(crypto_morus1280_avx2_algs));
		}

		module_init(crypto_morus1280_avx2_module_init);
		module_exit(crypto_morus1280_avx2_module_exit);

		MODULE_LICENSE("GPL");
		MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
		MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- AVX2 implementation");
		MODULE_ALIAS_CRYPTO("morus1280");
		MODULE_ALIAS_CRYPTO("morus1280-avx2");

arch/x86/crypto/morus1280-sse2-asm.S

0 → 100644

+895 −0

File added.

Preview size limit exceeded, changes collapsed.

arch/x86/crypto/morus1280-sse2-glue.c

0 → 100644

+68 −0

Original line number	Diff line number	Diff line
		/*
		* The MORUS-1280 Authenticated-Encryption Algorithm
		* Glue for SSE2 implementation
		*
		* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
		* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify it
		* under the terms of the GNU General Public License as published by the Free
		* Software Foundation; either version 2 of the License, or (at your option)
		* any later version.
		*/

		#include <crypto/internal/aead.h>
		#include <crypto/morus1280_glue.h>
		#include <linux/module.h>
		#include <asm/fpu/api.h>
		#include <asm/cpu_device_id.h>

		asmlinkage void crypto_morus1280_sse2_init(void state, const void key,
		const void *iv);
		asmlinkage void crypto_morus1280_sse2_ad(void state, const void data,
		unsigned int length);

		asmlinkage void crypto_morus1280_sse2_enc(void state, const void src,
		void *dst, unsigned int length);
		asmlinkage void crypto_morus1280_sse2_dec(void state, const void src,
		void *dst, unsigned int length);

		asmlinkage void crypto_morus1280_sse2_enc_tail(void state, const void src,
		void *dst, unsigned int length);
		asmlinkage void crypto_morus1280_sse2_dec_tail(void state, const void src,
		void *dst, unsigned int length);

		asmlinkage void crypto_morus1280_sse2_final(void state, void tag_xor,
		u64 assoclen, u64 cryptlen);

		MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350);

		static const struct x86_cpu_id sse2_cpu_id[] = {
		X86_FEATURE_MATCH(X86_FEATURE_XMM2),
		{}
		};
		MODULE_DEVICE_TABLE(x86cpu, sse2_cpu_id);

		static int __init crypto_morus1280_sse2_module_init(void)
		{
		if (!x86_match_cpu(sse2_cpu_id))
		return -ENODEV;

		return crypto_register_aeads(crypto_morus1280_sse2_algs,
		ARRAY_SIZE(crypto_morus1280_sse2_algs));
		}

		static void __exit crypto_morus1280_sse2_module_exit(void)
		{
		crypto_unregister_aeads(crypto_morus1280_sse2_algs,
		ARRAY_SIZE(crypto_morus1280_sse2_algs));
		}

		module_init(crypto_morus1280_sse2_module_init);
		module_exit(crypto_morus1280_sse2_module_exit);

		MODULE_LICENSE("GPL");
		MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
		MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- SSE2 implementation");
		MODULE_ALIAS_CRYPTO("morus1280");
		MODULE_ALIAS_CRYPTO("morus1280-sse2");