Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e069efb6 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6 

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6:
  hwrng: core - Prevent too-small buffer sizes
  hwrng: virtio-rng - Convert to new API
  hwrng: core - Replace u32 in driver API with byte array
  crypto: ansi_cprng - Move FIPS functions under CONFIG_CRYPTO_FIPS
  crypto: testmgr - Add ghash algorithm test before provide to users
  crypto: ghash-clmulni-intel - Put proper .data section in place
  crypto: ghash-clmulni-intel - Use gas macro for PCLMULQDQ-NI and PSHUFB
  crypto: aesni-intel - Use gas macro for AES-NI instructions
  x86: Generate .byte code for some new instructions via gas macro
  crypto: ghash-intel - Fix irq_fpu_usable usage
  crypto: ghash-intel - Add PSHUFB macros
  crypto: ghash-intel - Hard-code pshufb
  crypto: ghash-intel - Fix building failure on x86_32
  crypto: testmgr - Fix warning
  crypto: ansi_cprng - Fix test in get_prng_bytes
  crypto: hash - Remove cra_u.{digest,hash}
  crypto: api - Remove digest case from procfs show handler
  crypto: hash - Remove legacy hash/digest code
  crypto: ansi_cprng - Add FIPS wrapper
  crypto: ghash - Add PCLMULQDQ accelerated implementation
parents 324889b6 eed89d0f

arch/x86/crypto/Makefile

+3 −0
Original line number Original line Diff line number Diff line
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o 
obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o

obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o

obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o

obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o

obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o

obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o

obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o





obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o

obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
@@ -24,3 +25,5 @@ twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o 
salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o

salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o





aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o

aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o



ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o

arch/x86/crypto/aesni-intel_asm.S

+173 −344
Original line number Original line Diff line number Diff line
@@ -16,6 +16,7 @@ 
 */

 */





#include <linux/linkage.h>

#include <linux/linkage.h>

#include <asm/inst.h>





.text

.text
@@ -122,103 +123,72 @@ ENTRY(aesni_set_key) 
	movups 0x10(%rsi), %xmm2	# other user key

	movups 0x10(%rsi), %xmm2	# other user key

	movaps %xmm2, (%rcx)

	movaps %xmm2, (%rcx)

	add $0x10, %rcx

	add $0x10, %rcx

	# aeskeygenassist $0x1, %xmm2, %xmm1	# round 1

	AESKEYGENASSIST 0x1 %xmm2 %xmm1		# round 1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x1, %xmm0, %xmm1

	AESKEYGENASSIST 0x1 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x2, %xmm2, %xmm1	# round 2

	AESKEYGENASSIST 0x2 %xmm2 %xmm1		# round 2

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x2, %xmm0, %xmm1

	AESKEYGENASSIST 0x2 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x4, %xmm2, %xmm1	# round 3

	AESKEYGENASSIST 0x4 %xmm2 %xmm1		# round 3

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x4, %xmm0, %xmm1

	AESKEYGENASSIST 0x4 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x8, %xmm2, %xmm1	# round 4

	AESKEYGENASSIST 0x8 %xmm2 %xmm1		# round 4

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x8, %xmm0, %xmm1

	AESKEYGENASSIST 0x8 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x10, %xmm2, %xmm1	# round 5

	AESKEYGENASSIST 0x10 %xmm2 %xmm1	# round 5

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x10, %xmm0, %xmm1

	AESKEYGENASSIST 0x10 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x20, %xmm2, %xmm1	# round 6

	AESKEYGENASSIST 0x20 %xmm2 %xmm1	# round 6

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20

	call _key_expansion_256a

	call _key_expansion_256a

	# aeskeygenassist $0x20, %xmm0, %xmm1

	AESKEYGENASSIST 0x20 %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20

	call _key_expansion_256b

	call _key_expansion_256b

	# aeskeygenassist $0x40, %xmm2, %xmm1	# round 7

	AESKEYGENASSIST 0x40 %xmm2 %xmm1	# round 7

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40

	call _key_expansion_256a

	call _key_expansion_256a

	jmp .Ldec_key

	jmp .Ldec_key

.Lenc_key192:

.Lenc_key192:

	movq 0x10(%rsi), %xmm2		# other user key

	movq 0x10(%rsi), %xmm2		# other user key

	# aeskeygenassist $0x1, %xmm2, %xmm1	# round 1

	AESKEYGENASSIST 0x1 %xmm2 %xmm1		# round 1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01

	call _key_expansion_192a

	call _key_expansion_192a

	# aeskeygenassist $0x2, %xmm2, %xmm1	# round 2

	AESKEYGENASSIST 0x2 %xmm2 %xmm1		# round 2

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02

	call _key_expansion_192b

	call _key_expansion_192b

	# aeskeygenassist $0x4, %xmm2, %xmm1	# round 3

	AESKEYGENASSIST 0x4 %xmm2 %xmm1		# round 3

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04

	call _key_expansion_192a

	call _key_expansion_192a

	# aeskeygenassist $0x8, %xmm2, %xmm1	# round 4

	AESKEYGENASSIST 0x8 %xmm2 %xmm1		# round 4

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08

	call _key_expansion_192b

	call _key_expansion_192b

	# aeskeygenassist $0x10, %xmm2, %xmm1	# round 5

	AESKEYGENASSIST 0x10 %xmm2 %xmm1	# round 5

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10

	call _key_expansion_192a

	call _key_expansion_192a

	# aeskeygenassist $0x20, %xmm2, %xmm1	# round 6

	AESKEYGENASSIST 0x20 %xmm2 %xmm1	# round 6

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20

	call _key_expansion_192b

	call _key_expansion_192b

	# aeskeygenassist $0x40, %xmm2, %xmm1	# round 7

	AESKEYGENASSIST 0x40 %xmm2 %xmm1	# round 7

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40

	call _key_expansion_192a

	call _key_expansion_192a

	# aeskeygenassist $0x80, %xmm2, %xmm1	# round 8

	AESKEYGENASSIST 0x80 %xmm2 %xmm1	# round 8

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x80

	call _key_expansion_192b

	call _key_expansion_192b

	jmp .Ldec_key

	jmp .Ldec_key

.Lenc_key128:

.Lenc_key128:

	# aeskeygenassist $0x1, %xmm0, %xmm1	# round 1

	AESKEYGENASSIST 0x1 %xmm0 %xmm1		# round 1

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x2, %xmm0, %xmm1	# round 2

	AESKEYGENASSIST 0x2 %xmm0 %xmm1		# round 2

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x4, %xmm0, %xmm1	# round 3

	AESKEYGENASSIST 0x4 %xmm0 %xmm1		# round 3

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x8, %xmm0, %xmm1	# round 4

	AESKEYGENASSIST 0x8 %xmm0 %xmm1		# round 4

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x10, %xmm0, %xmm1	# round 5

	AESKEYGENASSIST 0x10 %xmm0 %xmm1	# round 5

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x20, %xmm0, %xmm1	# round 6

	AESKEYGENASSIST 0x20 %xmm0 %xmm1	# round 6

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x40, %xmm0, %xmm1	# round 7

	AESKEYGENASSIST 0x40 %xmm0 %xmm1	# round 7

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x40

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x80, %xmm0, %xmm1	# round 8

	AESKEYGENASSIST 0x80 %xmm0 %xmm1	# round 8

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x80

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x1b, %xmm0, %xmm1	# round 9

	AESKEYGENASSIST 0x1b %xmm0 %xmm1	# round 9

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x1b

	call _key_expansion_128

	call _key_expansion_128

	# aeskeygenassist $0x36, %xmm0, %xmm1	# round 10

	AESKEYGENASSIST 0x36 %xmm0 %xmm1	# round 10

	.byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x36

	call _key_expansion_128

	call _key_expansion_128

.Ldec_key:

.Ldec_key:

	sub $0x10, %rcx

	sub $0x10, %rcx
@@ -231,8 +201,7 @@ ENTRY(aesni_set_key) 
.align 4

.align 4

.Ldec_key_loop:

.Ldec_key_loop:

	movaps (%rdi), %xmm0

	movaps (%rdi), %xmm0

	# aesimc %xmm0, %xmm1

	AESIMC %xmm0 %xmm1

	.byte 0x66, 0x0f, 0x38, 0xdb, 0xc8

	movaps %xmm1, (%rsi)

	movaps %xmm1, (%rsi)

	add $0x10, %rdi

	add $0x10, %rdi

	sub $0x10, %rsi

	sub $0x10, %rsi
@@ -274,51 +243,37 @@ _aesni_enc1: 
	je .Lenc192

	je .Lenc192

	add $0x20, TKEYP

	add $0x20, TKEYP

	movaps -0x60(TKEYP), KEY

	movaps -0x60(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps -0x50(TKEYP), KEY

	movaps -0x50(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

.align 4

.align 4

.Lenc192:

.Lenc192:

	movaps -0x40(TKEYP), KEY

	movaps -0x40(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps -0x30(TKEYP), KEY

	movaps -0x30(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

.align 4

.align 4

.Lenc128:

.Lenc128:

	movaps -0x20(TKEYP), KEY

	movaps -0x20(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps -0x10(TKEYP), KEY

	movaps -0x10(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps (TKEYP), KEY

	movaps (TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x10(TKEYP), KEY

	movaps 0x10(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x20(TKEYP), KEY

	movaps 0x20(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x30(TKEYP), KEY

	movaps 0x30(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x40(TKEYP), KEY

	movaps 0x40(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x50(TKEYP), KEY

	movaps 0x50(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x60(TKEYP), KEY

	movaps 0x60(TKEYP), KEY

	# aesenc KEY, STATE

	AESENC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	movaps 0x70(TKEYP), KEY

	movaps 0x70(TKEYP), KEY

	# aesenclast KEY, STATE	# last round

	AESENCLAST KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdd, 0xc2

	ret

	ret





/*

/*
@@ -353,135 +308,79 @@ _aesni_enc4: 
	je .L4enc192

	je .L4enc192

	add $0x20, TKEYP

	add $0x20, TKEYP

	movaps -0x60(TKEYP), KEY

	movaps -0x60(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps -0x50(TKEYP), KEY

	movaps -0x50(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

#.align 4

#.align 4

.L4enc192:

.L4enc192:

	movaps -0x40(TKEYP), KEY

	movaps -0x40(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps -0x30(TKEYP), KEY

	movaps -0x30(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

#.align 4

#.align 4

.L4enc128:

.L4enc128:

	movaps -0x20(TKEYP), KEY

	movaps -0x20(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps -0x10(TKEYP), KEY

	movaps -0x10(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps (TKEYP), KEY

	movaps (TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x10(TKEYP), KEY

	movaps 0x10(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x20(TKEYP), KEY

	movaps 0x20(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x30(TKEYP), KEY

	movaps 0x30(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x40(TKEYP), KEY

	movaps 0x40(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x50(TKEYP), KEY

	movaps 0x50(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x60(TKEYP), KEY

	movaps 0x60(TKEYP), KEY

	# aesenc KEY, STATE1

	AESENC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xc2

	AESENC KEY STATE2

	# aesenc KEY, STATE2

	AESENC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xe2

	AESENC KEY STATE4

	# aesenc KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xea

	# aesenc KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdc, 0xf2

	movaps 0x70(TKEYP), KEY

	movaps 0x70(TKEYP), KEY

	# aesenclast KEY, STATE1	# last round

	AESENCLAST KEY STATE1		# last round

	.byte 0x66, 0x0f, 0x38, 0xdd, 0xc2

	AESENCLAST KEY STATE2

	# aesenclast KEY, STATE2

	AESENCLAST KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdd, 0xe2

	AESENCLAST KEY STATE4

	# aesenclast KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdd, 0xea

	# aesenclast KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdd, 0xf2

	ret

	ret





/*

/*
@@ -518,51 +417,37 @@ _aesni_dec1: 
	je .Ldec192

	je .Ldec192

	add $0x20, TKEYP

	add $0x20, TKEYP

	movaps -0x60(TKEYP), KEY

	movaps -0x60(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps -0x50(TKEYP), KEY

	movaps -0x50(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

.align 4

.align 4

.Ldec192:

.Ldec192:

	movaps -0x40(TKEYP), KEY

	movaps -0x40(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps -0x30(TKEYP), KEY

	movaps -0x30(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

.align 4

.align 4

.Ldec128:

.Ldec128:

	movaps -0x20(TKEYP), KEY

	movaps -0x20(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps -0x10(TKEYP), KEY

	movaps -0x10(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps (TKEYP), KEY

	movaps (TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x10(TKEYP), KEY

	movaps 0x10(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x20(TKEYP), KEY

	movaps 0x20(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x30(TKEYP), KEY

	movaps 0x30(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x40(TKEYP), KEY

	movaps 0x40(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x50(TKEYP), KEY

	movaps 0x50(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x60(TKEYP), KEY

	movaps 0x60(TKEYP), KEY

	# aesdec KEY, STATE

	AESDEC KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	movaps 0x70(TKEYP), KEY

	movaps 0x70(TKEYP), KEY

	# aesdeclast KEY, STATE		# last round

	AESDECLAST KEY STATE

	.byte 0x66, 0x0f, 0x38, 0xdf, 0xc2

	ret

	ret





/*

/*
@@ -597,135 +482,79 @@ _aesni_dec4: 
	je .L4dec192

	je .L4dec192

	add $0x20, TKEYP

	add $0x20, TKEYP

	movaps -0x60(TKEYP), KEY

	movaps -0x60(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps -0x50(TKEYP), KEY

	movaps -0x50(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

.align 4

.align 4

.L4dec192:

.L4dec192:

	movaps -0x40(TKEYP), KEY

	movaps -0x40(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps -0x30(TKEYP), KEY

	movaps -0x30(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

.align 4

.align 4

.L4dec128:

.L4dec128:

	movaps -0x20(TKEYP), KEY

	movaps -0x20(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps -0x10(TKEYP), KEY

	movaps -0x10(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps (TKEYP), KEY

	movaps (TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x10(TKEYP), KEY

	movaps 0x10(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x20(TKEYP), KEY

	movaps 0x20(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x30(TKEYP), KEY

	movaps 0x30(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x40(TKEYP), KEY

	movaps 0x40(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x50(TKEYP), KEY

	movaps 0x50(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x60(TKEYP), KEY

	movaps 0x60(TKEYP), KEY

	# aesdec KEY, STATE1

	AESDEC KEY STATE1

	.byte 0x66, 0x0f, 0x38, 0xde, 0xc2

	AESDEC KEY STATE2

	# aesdec KEY, STATE2

	AESDEC KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xe2

	AESDEC KEY STATE4

	# aesdec KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xde, 0xea

	# aesdec KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xde, 0xf2

	movaps 0x70(TKEYP), KEY

	movaps 0x70(TKEYP), KEY

	# aesdeclast KEY, STATE1	# last round

	AESDECLAST KEY STATE1		# last round

	.byte 0x66, 0x0f, 0x38, 0xdf, 0xc2

	AESDECLAST KEY STATE2

	# aesdeclast KEY, STATE2

	AESDECLAST KEY STATE3

	.byte 0x66, 0x0f, 0x38, 0xdf, 0xe2

	AESDECLAST KEY STATE4

	# aesdeclast KEY, STATE3

	.byte 0x66, 0x0f, 0x38, 0xdf, 0xea

	# aesdeclast KEY, STATE4

	.byte 0x66, 0x0f, 0x38, 0xdf, 0xf2

	ret

	ret





/*

/*

arch/x86/crypto/ghash-clmulni-intel_asm.S

0 → 100644
+157 −0
Original line number Original line Diff line number Diff line 
/*

 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI

 * instructions. This file contains accelerated part of ghash

 * implementation. More information about PCLMULQDQ can be found at:

 *

 * http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/

 *

 * Copyright (c) 2009 Intel Corp.

 *   Author: Huang Ying <ying.huang@intel.com>

 *	     Vinodh Gopal

 *	     Erdinc Ozturk

 *	     Deniz Karakoyunlu

 *

 * This program is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 as published

 * by the Free Software Foundation.

 */



#include <linux/linkage.h>

#include <asm/inst.h>



.data



.align 16

.Lbswap_mask:

	.octa 0x000102030405060708090a0b0c0d0e0f

.Lpoly:

	.octa 0xc2000000000000000000000000000001

.Ltwo_one:

	.octa 0x00000001000000000000000000000001



#define DATA	%xmm0

#define SHASH	%xmm1

#define T1	%xmm2

#define T2	%xmm3

#define T3	%xmm4

#define BSWAP	%xmm5

#define IN1	%xmm6



.text



/*

 * __clmul_gf128mul_ble:	internal ABI

 * input:

 *	DATA:			operand1

 *	SHASH:			operand2, hash_key << 1 mod poly

 * output:

 *	DATA:			operand1 * operand2 mod poly

 * changed:

 *	T1

 *	T2

 *	T3

 */

__clmul_gf128mul_ble:

	movaps DATA, T1

	pshufd $0b01001110, DATA, T2

	pshufd $0b01001110, SHASH, T3

	pxor DATA, T2

	pxor SHASH, T3



	PCLMULQDQ 0x00 SHASH DATA	# DATA = a0 * b0

	PCLMULQDQ 0x11 SHASH T1		# T1 = a1 * b1

	PCLMULQDQ 0x00 T3 T2		# T2 = (a1 + a0) * (b1 + b0)

	pxor DATA, T2

	pxor T1, T2			# T2 = a0 * b1 + a1 * b0



	movaps T2, T3

	pslldq $8, T3

	psrldq $8, T2

	pxor T3, DATA

	pxor T2, T1			# <T1:DATA> is result of

					# carry-less multiplication



	# first phase of the reduction

	movaps DATA, T3

	psllq $1, T3

	pxor DATA, T3

	psllq $5, T3

	pxor DATA, T3

	psllq $57, T3

	movaps T3, T2

	pslldq $8, T2

	psrldq $8, T3

	pxor T2, DATA

	pxor T3, T1



	# second phase of the reduction

	movaps DATA, T2

	psrlq $5, T2

	pxor DATA, T2

	psrlq $1, T2

	pxor DATA, T2

	psrlq $1, T2

	pxor T2, T1

	pxor T1, DATA

	ret



/* void clmul_ghash_mul(char *dst, const be128 *shash) */

ENTRY(clmul_ghash_mul)

	movups (%rdi), DATA

	movups (%rsi), SHASH

	movaps .Lbswap_mask, BSWAP

	PSHUFB_XMM BSWAP DATA

	call __clmul_gf128mul_ble

	PSHUFB_XMM BSWAP DATA

	movups DATA, (%rdi)

	ret



/*

 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,

 *			   const be128 *shash);

 */

ENTRY(clmul_ghash_update)

	cmp $16, %rdx

	jb .Lupdate_just_ret	# check length

	movaps .Lbswap_mask, BSWAP

	movups (%rdi), DATA

	movups (%rcx), SHASH

	PSHUFB_XMM BSWAP DATA

.align 4

.Lupdate_loop:

	movups (%rsi), IN1

	PSHUFB_XMM BSWAP IN1

	pxor IN1, DATA

	call __clmul_gf128mul_ble

	sub $16, %rdx

	add $16, %rsi

	cmp $16, %rdx

	jge .Lupdate_loop

	PSHUFB_XMM BSWAP DATA

	movups DATA, (%rdi)

.Lupdate_just_ret:

	ret



/*

 * void clmul_ghash_setkey(be128 *shash, const u8 *key);

 *

 * Calculate hash_key << 1 mod poly

 */

ENTRY(clmul_ghash_setkey)

	movaps .Lbswap_mask, BSWAP

	movups (%rsi), %xmm0

	PSHUFB_XMM BSWAP %xmm0

	movaps %xmm0, %xmm1

	psllq $1, %xmm0

	psrlq $63, %xmm1

	movaps %xmm1, %xmm2

	pslldq $8, %xmm1

	psrldq $8, %xmm2

	por %xmm1, %xmm0

	# reduction

	pshufd $0b00100100, %xmm2, %xmm1

	pcmpeqd .Ltwo_one, %xmm1

	pand .Lpoly, %xmm1

	pxor %xmm1, %xmm0

	movups %xmm0, (%rdi)

	ret

arch/x86/crypto/ghash-clmulni-intel_glue.c

0 → 100644
+333 −0

File added.

Preview size limit exceeded, changes collapsed.

arch/x86/include/asm/cpufeature.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -248,6 +248,7 @@ extern const char * const x86_power_flags[32]; 
#define cpu_has_x2apic		boot_cpu_has(X86_FEATURE_X2APIC)

#define cpu_has_x2apic		boot_cpu_has(X86_FEATURE_X2APIC)

#define cpu_has_xsave		boot_cpu_has(X86_FEATURE_XSAVE)

#define cpu_has_xsave		boot_cpu_has(X86_FEATURE_XSAVE)

#define cpu_has_hypervisor	boot_cpu_has(X86_FEATURE_HYPERVISOR)

#define cpu_has_hypervisor	boot_cpu_has(X86_FEATURE_HYPERVISOR)

#define cpu_has_pclmulqdq	boot_cpu_has(X86_FEATURE_PCLMULQDQ)





#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)

#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)

# define cpu_has_invlpg		1

# define cpu_has_invlpg		1