Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c594c540 authored by Dave Watson's avatar Dave Watson Committed by Herbert Xu
Browse files

crypto: aesni - Split AAD hash calculation to separate macro 



AAD hash only needs to be calculated once for each scatter/gather operation.
Move it to its own macro, and call it from GCM_INIT instead of
INITIAL_BLOCKS.

Signed-off-by: default avatarDave Watson <davejwatson@fb.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 9ee4a5df

arch/x86/crypto/aesni-intel_asm.S

+43 −28
Original line number Diff line number Diff line
@@ -230,6 +230,10 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff 
	mov	%arg5, %r13		# %xmm13 holds HashKey<<1 (mod poly)

	and	$-16, %r13

	mov	%r13, %r12



	CALC_AAD_HASH %xmm13 %xmm0 %xmm1 %xmm2 %xmm3 %xmm4 \

	%xmm5 %xmm6

	mov %r13, %r12

.endm



# GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context
@@ -497,51 +501,62 @@ _read_next_byte_lt8_\@: 
_done_read_partial_block_\@:

.endm



/*

* if a = number of total plaintext bytes

* b = floor(a/16)

* num_initial_blocks = b mod 4

* encrypt the initial num_initial_blocks blocks and apply ghash on

* the ciphertext

* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers

* are clobbered

* arg1, %arg3, %arg4, %r14 are used as a pointer only, not modified

*/





.macro INITIAL_BLOCKS_ENC_DEC TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \

XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation

# CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted.

# clobbers r10-11, xmm14

.macro CALC_AAD_HASH HASHKEY TMP1 TMP2 TMP3 TMP4 TMP5 \

	TMP6 TMP7

	MOVADQ	   SHUF_MASK(%rip), %xmm14

	mov	   arg8, %r10		# %r10 = AAD

	mov	   arg9, %r11		# %r11 = aadLen

	pxor	   %xmm\i, %xmm\i

	pxor	   \XMM2, \XMM2

	pxor	   \TMP7, \TMP7

	pxor	   \TMP6, \TMP6



	cmp	   $16, %r11

	jl	   _get_AAD_rest\@

_get_AAD_blocks\@:

	movdqu	   (%r10), %xmm\i

	PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data

	pxor	   %xmm\i, \XMM2

	GHASH_MUL  \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1

	movdqu	   (%r10), \TMP7

	PSHUFB_XMM   %xmm14, \TMP7 # byte-reflect the AAD data

	pxor	   \TMP7, \TMP6

	GHASH_MUL  \TMP6, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5

	add	   $16, %r10

	sub	   $16, %r11

	cmp	   $16, %r11

	jge	   _get_AAD_blocks\@



	movdqu	   \XMM2, %xmm\i

	movdqu	   \TMP6, \TMP7



	/* read the last <16B of AAD */

_get_AAD_rest\@:

	cmp	   $0, %r11

	je	   _get_AAD_done\@



	READ_PARTIAL_BLOCK %r10, %r11, \TMP1, %xmm\i

	PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data

	pxor	   \XMM2, %xmm\i

	GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1

	READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7

	PSHUFB_XMM   %xmm14, \TMP7 # byte-reflect the AAD data

	pxor	   \TMP6, \TMP7

	GHASH_MUL  \TMP7, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5

	movdqu \TMP7, \TMP6



_get_AAD_done\@:

	movdqu \TMP6, AadHash(%arg2)

.endm



/*

* if a = number of total plaintext bytes

* b = floor(a/16)

* num_initial_blocks = b mod 4

* encrypt the initial num_initial_blocks blocks and apply ghash on

* the ciphertext

* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers

* are clobbered

* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified

*/





.macro INITIAL_BLOCKS_ENC_DEC TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \

	XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation



	movdqu AadHash(%arg2), %xmm\i		    # XMM0 = Y0



	xor	   %r11, %r11 # initialise the data pointer offset as zero

	# start AES for num_initial_blocks blocks