Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b4df50de authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6 

Pull crypto fixes from Herbert Xu:

 - Check for the right CPU feature bit in sm4-ce on arm64.

 - Fix scatterwalk WARN_ON in aes-gcm-ce on arm64.

 - Fix unaligned fault in aesni on x86.

 - Fix potential NULL pointer dereference on exit in chtls.

 - Fix DMA mapping direction for RSA in caam.

 - Fix error path return value for xts setkey in caam.

 - Fix address endianness when DMA unmapping in caam.

 - Fix sleep-in-atomic in vmx.

 - Fix command corruption when queue is full in cavium/nitrox.

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6:
  crypto: cavium/nitrox - fix for command corruption in queue full case with backlog submissions.
  crypto: vmx - Fix sleep-in-atomic bugs
  crypto: arm64/aes-gcm-ce - fix scatterwalk API violation
  crypto: aesni - Use unaligned loads from gcm_context_data
  crypto: chtls - fix null dereference chtls_free_uld()
  crypto: arm64/sm4-ce - check for the right CPU feature bit
  crypto: caam - fix DMA mapping direction for RSA forms 2 & 3
  crypto: caam/qi - fix error path in xts setkey
  crypto: caam/jr - fix descriptor DMA unmapping
parents 3f16503b 3d7c8206

arch/arm64/crypto/ghash-ce-glue.c

+23 −6
Original line number Diff line number Diff line
@@ -417,7 +417,7 @@ static int gcm_encrypt(struct aead_request *req) 
		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds);

		put_unaligned_be32(2, iv + GCM_IV_SIZE);



		while (walk.nbytes >= AES_BLOCK_SIZE) {

		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {

			int blocks = walk.nbytes / AES_BLOCK_SIZE;

			u8 *dst = walk.dst.virt.addr;

			u8 *src = walk.src.virt.addr;
@@ -437,11 +437,18 @@ static int gcm_encrypt(struct aead_request *req) 
					NULL);



			err = skcipher_walk_done(&walk,

						 walk.nbytes % AES_BLOCK_SIZE);

						 walk.nbytes % (2 * AES_BLOCK_SIZE));

		}

		if (walk.nbytes)

		if (walk.nbytes) {

			__aes_arm64_encrypt(ctx->aes_key.key_enc, ks, iv,

					    nrounds);

			if (walk.nbytes > AES_BLOCK_SIZE) {

				crypto_inc(iv, AES_BLOCK_SIZE);

				__aes_arm64_encrypt(ctx->aes_key.key_enc,

					            ks + AES_BLOCK_SIZE, iv,

						    nrounds);

			}

		}

	}



	/* handle the tail */
@@ -545,7 +552,7 @@ static int gcm_decrypt(struct aead_request *req) 
		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds);

		put_unaligned_be32(2, iv + GCM_IV_SIZE);



		while (walk.nbytes >= AES_BLOCK_SIZE) {

		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {

			int blocks = walk.nbytes / AES_BLOCK_SIZE;

			u8 *dst = walk.dst.virt.addr;

			u8 *src = walk.src.virt.addr;
@@ -564,12 +571,22 @@ static int gcm_decrypt(struct aead_request *req) 
			} while (--blocks > 0);



			err = skcipher_walk_done(&walk,

						 walk.nbytes % AES_BLOCK_SIZE);

						 walk.nbytes % (2 * AES_BLOCK_SIZE));

		}

		if (walk.nbytes) {

			if (walk.nbytes > AES_BLOCK_SIZE) {

				u8 *iv2 = iv + AES_BLOCK_SIZE;



				memcpy(iv2, iv, AES_BLOCK_SIZE);

				crypto_inc(iv2, AES_BLOCK_SIZE);



				__aes_arm64_encrypt(ctx->aes_key.key_enc, iv2,

						    iv2, nrounds);

			}

		if (walk.nbytes)

			__aes_arm64_encrypt(ctx->aes_key.key_enc, iv, iv,

					    nrounds);

		}

	}



	/* handle the tail */

	if (walk.nbytes) {

arch/arm64/crypto/sm4-ce-glue.c

+1 −1
Original line number Diff line number Diff line
@@ -69,5 +69,5 @@ static void __exit sm4_ce_mod_fini(void) 
	crypto_unregister_alg(&sm4_ce_alg);

}



module_cpu_feature_match(SM3, sm4_ce_mod_init);

module_cpu_feature_match(SM4, sm4_ce_mod_init);

module_exit(sm4_ce_mod_fini);

arch/x86/crypto/aesni-intel_asm.S

+33 −33
Original line number Diff line number Diff line
@@ -223,34 +223,34 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff 
	pcmpeqd TWOONE(%rip), \TMP2

	pand	POLY(%rip), \TMP2

	pxor	\TMP2, \TMP3

	movdqa	\TMP3, HashKey(%arg2)

	movdqu	\TMP3, HashKey(%arg2)



	movdqa	   \TMP3, \TMP5

	pshufd	   $78, \TMP3, \TMP1

	pxor	   \TMP3, \TMP1

	movdqa	   \TMP1, HashKey_k(%arg2)

	movdqu	   \TMP1, HashKey_k(%arg2)



	GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7

# TMP5 = HashKey^2<<1 (mod poly)

	movdqa	   \TMP5, HashKey_2(%arg2)

	movdqu	   \TMP5, HashKey_2(%arg2)

# HashKey_2 = HashKey^2<<1 (mod poly)

	pshufd	   $78, \TMP5, \TMP1

	pxor	   \TMP5, \TMP1

	movdqa	   \TMP1, HashKey_2_k(%arg2)

	movdqu	   \TMP1, HashKey_2_k(%arg2)



	GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7

# TMP5 = HashKey^3<<1 (mod poly)

	movdqa	   \TMP5, HashKey_3(%arg2)

	movdqu	   \TMP5, HashKey_3(%arg2)

	pshufd	   $78, \TMP5, \TMP1

	pxor	   \TMP5, \TMP1

	movdqa	   \TMP1, HashKey_3_k(%arg2)

	movdqu	   \TMP1, HashKey_3_k(%arg2)



	GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7

# TMP5 = HashKey^3<<1 (mod poly)

	movdqa	   \TMP5, HashKey_4(%arg2)

	movdqu	   \TMP5, HashKey_4(%arg2)

	pshufd	   $78, \TMP5, \TMP1

	pxor	   \TMP5, \TMP1

	movdqa	   \TMP1, HashKey_4_k(%arg2)

	movdqu	   \TMP1, HashKey_4_k(%arg2)

.endm



# GCM_INIT initializes a gcm_context struct to prepare for encoding/decoding.
@@ -271,7 +271,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff 
	movdqu %xmm0, CurCount(%arg2) # ctx_data.current_counter = iv



	PRECOMPUTE \SUBKEY, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,

	movdqa HashKey(%arg2), %xmm13

	movdqu HashKey(%arg2), %xmm13



	CALC_AAD_HASH %xmm13, \AAD, \AADLEN, %xmm0, %xmm1, %xmm2, %xmm3, \

	%xmm4, %xmm5, %xmm6
@@ -997,7 +997,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	pshufd	  $78, \XMM5, \TMP6

	pxor	  \XMM5, \TMP6

	paddd     ONE(%rip), \XMM0		# INCR CNT

	movdqa	  HashKey_4(%arg2), \TMP5

	movdqu	  HashKey_4(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1

	movdqa    \XMM0, \XMM1

	paddd     ONE(%rip), \XMM0		# INCR CNT
@@ -1016,7 +1016,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	pxor	  (%arg1), \XMM2

	pxor	  (%arg1), \XMM3

	pxor	  (%arg1), \XMM4

	movdqa	  HashKey_4_k(%arg2), \TMP5

	movdqu	  HashKey_4_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)

	movaps 0x10(%arg1), \TMP1

	AESENC	  \TMP1, \XMM1              # Round 1
@@ -1031,7 +1031,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM6, \TMP1

	pshufd	  $78, \XMM6, \TMP2

	pxor	  \XMM6, \TMP2

	movdqa	  HashKey_3(%arg2), \TMP5

	movdqu	  HashKey_3(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1

	movaps 0x30(%arg1), \TMP3

	AESENC    \TMP3, \XMM1              # Round 3
@@ -1044,7 +1044,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	AESENC	  \TMP3, \XMM2

	AESENC	  \TMP3, \XMM3

	AESENC	  \TMP3, \XMM4

	movdqa	  HashKey_3_k(%arg2), \TMP5

	movdqu	  HashKey_3_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)

	movaps 0x50(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1              # Round 5
@@ -1058,7 +1058,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM7, \TMP1

	pshufd	  $78, \XMM7, \TMP2

	pxor	  \XMM7, \TMP2

	movdqa	  HashKey_2(%arg2), \TMP5

	movdqu	  HashKey_2(%arg2), \TMP5



        # Multiply TMP5 * HashKey using karatsuba
@@ -1074,7 +1074,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	AESENC	  \TMP3, \XMM2

	AESENC	  \TMP3, \XMM3

	AESENC	  \TMP3, \XMM4

	movdqa	  HashKey_2_k(%arg2), \TMP5

	movdqu	  HashKey_2_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)

	movaps 0x80(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1             # Round 8
@@ -1092,7 +1092,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM8, \TMP1

	pshufd	  $78, \XMM8, \TMP2

	pxor	  \XMM8, \TMP2

	movdqa	  HashKey(%arg2), \TMP5

	movdqu	  HashKey(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1

	movaps 0x90(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1            # Round 9
@@ -1121,7 +1121,7 @@ aes_loop_par_enc_done\@: 
	AESENCLAST \TMP3, \XMM2

	AESENCLAST \TMP3, \XMM3

	AESENCLAST \TMP3, \XMM4

	movdqa    HashKey_k(%arg2), \TMP5

	movdqu    HashKey_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)

	movdqu	  (%arg4,%r11,1), \TMP3

	pxor	  \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
@@ -1205,7 +1205,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	pshufd	  $78, \XMM5, \TMP6

	pxor	  \XMM5, \TMP6

	paddd     ONE(%rip), \XMM0		# INCR CNT

	movdqa	  HashKey_4(%arg2), \TMP5

	movdqu	  HashKey_4(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1

	movdqa    \XMM0, \XMM1

	paddd     ONE(%rip), \XMM0		# INCR CNT
@@ -1224,7 +1224,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	pxor	  (%arg1), \XMM2

	pxor	  (%arg1), \XMM3

	pxor	  (%arg1), \XMM4

	movdqa	  HashKey_4_k(%arg2), \TMP5

	movdqu	  HashKey_4_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)

	movaps 0x10(%arg1), \TMP1

	AESENC	  \TMP1, \XMM1              # Round 1
@@ -1239,7 +1239,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM6, \TMP1

	pshufd	  $78, \XMM6, \TMP2

	pxor	  \XMM6, \TMP2

	movdqa	  HashKey_3(%arg2), \TMP5

	movdqu	  HashKey_3(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1

	movaps 0x30(%arg1), \TMP3

	AESENC    \TMP3, \XMM1              # Round 3
@@ -1252,7 +1252,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	AESENC	  \TMP3, \XMM2

	AESENC	  \TMP3, \XMM3

	AESENC	  \TMP3, \XMM4

	movdqa	  HashKey_3_k(%arg2), \TMP5

	movdqu	  HashKey_3_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)

	movaps 0x50(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1              # Round 5
@@ -1266,7 +1266,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM7, \TMP1

	pshufd	  $78, \XMM7, \TMP2

	pxor	  \XMM7, \TMP2

	movdqa	  HashKey_2(%arg2), \TMP5

	movdqu	  HashKey_2(%arg2), \TMP5



        # Multiply TMP5 * HashKey using karatsuba
@@ -1282,7 +1282,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	AESENC	  \TMP3, \XMM2

	AESENC	  \TMP3, \XMM3

	AESENC	  \TMP3, \XMM4

	movdqa	  HashKey_2_k(%arg2), \TMP5

	movdqu	  HashKey_2_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)

	movaps 0x80(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1             # Round 8
@@ -1300,7 +1300,7 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 
	movdqa	  \XMM8, \TMP1

	pshufd	  $78, \XMM8, \TMP2

	pxor	  \XMM8, \TMP2

	movdqa	  HashKey(%arg2), \TMP5

	movdqu	  HashKey(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1

	movaps 0x90(%arg1), \TMP3

	AESENC	  \TMP3, \XMM1            # Round 9
@@ -1329,7 +1329,7 @@ aes_loop_par_dec_done\@: 
	AESENCLAST \TMP3, \XMM2

	AESENCLAST \TMP3, \XMM3

	AESENCLAST \TMP3, \XMM4

	movdqa    HashKey_k(%arg2), \TMP5

	movdqu    HashKey_k(%arg2), \TMP5

	PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)

	movdqu	  (%arg4,%r11,1), \TMP3

	pxor	  \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
@@ -1405,10 +1405,10 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst 
	movdqa	  \XMM1, \TMP6

	pshufd	  $78, \XMM1, \TMP2

	pxor	  \XMM1, \TMP2

	movdqa	  HashKey_4(%arg2), \TMP5

	movdqu	  HashKey_4(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP6       # TMP6 = a1*b1

	PCLMULQDQ 0x00, \TMP5, \XMM1       # XMM1 = a0*b0

	movdqa	  HashKey_4_k(%arg2), \TMP4

	movdqu	  HashKey_4_k(%arg2), \TMP4

	PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)

	movdqa	  \XMM1, \XMMDst

	movdqa	  \TMP2, \XMM1              # result in TMP6, XMMDst, XMM1
@@ -1418,10 +1418,10 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst 
	movdqa	  \XMM2, \TMP1

	pshufd	  $78, \XMM2, \TMP2

	pxor	  \XMM2, \TMP2

	movdqa	  HashKey_3(%arg2), \TMP5

	movdqu	  HashKey_3(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1

	PCLMULQDQ 0x00, \TMP5, \XMM2       # XMM2 = a0*b0

	movdqa	  HashKey_3_k(%arg2), \TMP4

	movdqu	  HashKey_3_k(%arg2), \TMP4

	PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)

	pxor	  \TMP1, \TMP6

	pxor	  \XMM2, \XMMDst
@@ -1433,10 +1433,10 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst 
	movdqa	  \XMM3, \TMP1

	pshufd	  $78, \XMM3, \TMP2

	pxor	  \XMM3, \TMP2

	movdqa	  HashKey_2(%arg2), \TMP5

	movdqu	  HashKey_2(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1

	PCLMULQDQ 0x00, \TMP5, \XMM3       # XMM3 = a0*b0

	movdqa	  HashKey_2_k(%arg2), \TMP4

	movdqu	  HashKey_2_k(%arg2), \TMP4

	PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)

	pxor	  \TMP1, \TMP6

	pxor	  \XMM3, \XMMDst
@@ -1446,10 +1446,10 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst 
	movdqa	  \XMM4, \TMP1

	pshufd	  $78, \XMM4, \TMP2

	pxor	  \XMM4, \TMP2

	movdqa	  HashKey(%arg2), \TMP5

	movdqu	  HashKey(%arg2), \TMP5

	PCLMULQDQ 0x11, \TMP5, \TMP1	    # TMP1 = a1*b1

	PCLMULQDQ 0x00, \TMP5, \XMM4       # XMM4 = a0*b0

	movdqa	  HashKey_k(%arg2), \TMP4

	movdqu	  HashKey_k(%arg2), \TMP4

	PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)

	pxor	  \TMP1, \TMP6

	pxor	  \XMM4, \XMMDst

drivers/crypto/caam/caamalg_qi.c

+2 −4
Original line number Diff line number Diff line
@@ -679,10 +679,8 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 
	int ret = 0;



	if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {

		crypto_ablkcipher_set_flags(ablkcipher,

					    CRYPTO_TFM_RES_BAD_KEY_LEN);

		dev_err(jrdev, "key size mismatch\n");

		return -EINVAL;

		goto badkey;

	}



	ctx->cdata.keylen = keylen;
@@ -715,7 +713,7 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 
	return ret;

badkey:

	crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

	return 0;

	return -EINVAL;

}



/*

drivers/crypto/caam/caampkc.c

+10 −10
Original line number Diff line number Diff line
@@ -71,8 +71,8 @@ static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, 
	dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL);

	dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_BIDIRECTIONAL);

}



static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc,
@@ -90,8 +90,8 @@ static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc, 
	dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL);

	dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_BIDIRECTIONAL);

}



/* RSA Job Completion handler */
@@ -417,13 +417,13 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req, 
		goto unmap_p;

	}



	pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE);

	pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_BIDIRECTIONAL);

	if (dma_mapping_error(dev, pdb->tmp1_dma)) {

		dev_err(dev, "Unable to map RSA tmp1 memory\n");

		goto unmap_q;

	}



	pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE);

	pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_BIDIRECTIONAL);

	if (dma_mapping_error(dev, pdb->tmp2_dma)) {

		dev_err(dev, "Unable to map RSA tmp2 memory\n");

		goto unmap_tmp1;
@@ -451,7 +451,7 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req, 
	return 0;



unmap_tmp1:

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL);

unmap_q:

	dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);

unmap_p:
@@ -504,13 +504,13 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req, 
		goto unmap_dq;

	}



	pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE);

	pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_BIDIRECTIONAL);

	if (dma_mapping_error(dev, pdb->tmp1_dma)) {

		dev_err(dev, "Unable to map RSA tmp1 memory\n");

		goto unmap_qinv;

	}



	pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE);

	pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_BIDIRECTIONAL);

	if (dma_mapping_error(dev, pdb->tmp2_dma)) {

		dev_err(dev, "Unable to map RSA tmp2 memory\n");

		goto unmap_tmp1;
@@ -538,7 +538,7 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req, 
	return 0;



unmap_tmp1:

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);

	dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL);

unmap_qinv:

	dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE);

unmap_dq: