Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 58990986 authored by Jussi Kivilinna's avatar Jussi Kivilinna Committed by Herbert Xu
Browse files

crypto: x86/glue_helper - use le128 instead of u128 for CTR mode 



'u128' currently used for CTR mode is on little-endian 'long long' swapped
and would require extra swap operations by SSE/AVX code. Use of le128
instead of u128 allows IV calculations to be done with vector registers
easier.

Signed-off-by: default avatarJussi Kivilinna <jussi.kivilinna@mbnet.fi>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent e080b17a

arch/x86/crypto/camellia_glue.c

+8 −8
Original line number Diff line number Diff line
@@ -1317,21 +1317,21 @@ static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src) 
	u128_xor(&dst[1], &dst[1], &iv);

}



static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)

static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)

{

	be128 ctrblk;



	if (dst != src)

		*dst = *src;



	u128_to_be128(&ctrblk, iv);

	u128_inc(iv);

	le128_to_be128(&ctrblk, iv);

	le128_inc(iv);



	camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);

}



static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,

				    u128 *iv)

				    le128 *iv)

{

	be128 ctrblks[2];
@@ -1340,10 +1340,10 @@ static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, 
		dst[1] = src[1];

	}



	u128_to_be128(&ctrblks[0], iv);

	u128_inc(iv);

	u128_to_be128(&ctrblks[1], iv);

	u128_inc(iv);

	le128_to_be128(&ctrblks[0], iv);

	le128_inc(iv);

	le128_to_be128(&ctrblks[1], iv);

	le128_inc(iv);



	camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);

}

arch/x86/crypto/cast6_avx_glue.c

+6 −6
Original line number Diff line number Diff line
@@ -78,19 +78,19 @@ static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) 
		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);

}



static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)

static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)

{

	be128 ctrblk;



	u128_to_be128(&ctrblk, iv);

	u128_inc(iv);

	le128_to_be128(&ctrblk, iv);

	le128_inc(iv);



	__cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);

	u128_xor(dst, src, (u128 *)&ctrblk);

}



static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,

				   u128 *iv)

				 le128 *iv)

{

	be128 ctrblks[CAST6_PARALLEL_BLOCKS];

	unsigned int i;
@@ -99,8 +99,8 @@ static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, 
		if (dst != src)

			dst[i] = src[i];



		u128_to_be128(&ctrblks[i], iv);

		u128_inc(iv);

		le128_to_be128(&ctrblks[i], iv);

		le128_inc(iv);

	}



	cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);

arch/x86/crypto/glue_helper.c

+6 −6
Original line number Diff line number Diff line
@@ -221,16 +221,16 @@ static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr, 
	u8 *src = (u8 *)walk->src.virt.addr;

	u8 *dst = (u8 *)walk->dst.virt.addr;

	unsigned int nbytes = walk->nbytes;

	u128 ctrblk;

	le128 ctrblk;

	u128 tmp;



	be128_to_u128(&ctrblk, (be128 *)walk->iv);

	be128_to_le128(&ctrblk, (be128 *)walk->iv);



	memcpy(&tmp, src, nbytes);

	fn_ctr(ctx, &tmp, &tmp, &ctrblk);

	memcpy(dst, &tmp, nbytes);



	u128_to_be128((be128 *)walk->iv, &ctrblk);

	le128_to_be128((be128 *)walk->iv, &ctrblk);

}

EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit);
@@ -243,11 +243,11 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, 
	unsigned int nbytes = walk->nbytes;

	u128 *src = (u128 *)walk->src.virt.addr;

	u128 *dst = (u128 *)walk->dst.virt.addr;

	u128 ctrblk;

	le128 ctrblk;

	unsigned int num_blocks, func_bytes;

	unsigned int i;



	be128_to_u128(&ctrblk, (be128 *)walk->iv);

	be128_to_le128(&ctrblk, (be128 *)walk->iv);



	/* Process multi-block batch */

	for (i = 0; i < gctx->num_funcs; i++) {
@@ -269,7 +269,7 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, 
	}



done:

	u128_to_be128((be128 *)walk->iv, &ctrblk);

	le128_to_be128((be128 *)walk->iv, &ctrblk);

	return nbytes;

}

arch/x86/crypto/serpent_avx_glue.c

+6 −6
Original line number Diff line number Diff line
@@ -56,19 +56,19 @@ static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) 
		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);

}



static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)

static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)

{

	be128 ctrblk;



	u128_to_be128(&ctrblk, iv);

	u128_inc(iv);

	le128_to_be128(&ctrblk, iv);

	le128_inc(iv);



	__serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);

	u128_xor(dst, src, (u128 *)&ctrblk);

}



static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,

				   u128 *iv)

				   le128 *iv)

{

	be128 ctrblks[SERPENT_PARALLEL_BLOCKS];

	unsigned int i;
@@ -77,8 +77,8 @@ static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, 
		if (dst != src)

			dst[i] = src[i];



		u128_to_be128(&ctrblks[i], iv);

		u128_inc(iv);

		le128_to_be128(&ctrblks[i], iv);

		le128_inc(iv);

	}



	serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);

arch/x86/crypto/serpent_sse2_glue.c

+6 −6
Original line number Diff line number Diff line
@@ -59,19 +59,19 @@ static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) 
		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);

}



static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)

static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)

{

	be128 ctrblk;



	u128_to_be128(&ctrblk, iv);

	u128_inc(iv);

	le128_to_be128(&ctrblk, iv);

	le128_inc(iv);



	__serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);

	u128_xor(dst, src, (u128 *)&ctrblk);

}



static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,

				   u128 *iv)

				   le128 *iv)

{

	be128 ctrblks[SERPENT_PARALLEL_BLOCKS];

	unsigned int i;
@@ -80,8 +80,8 @@ static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, 
		if (dst != src)

			dst[i] = src[i];



		u128_to_be128(&ctrblks[i], iv);

		u128_inc(iv);

		le128_to_be128(&ctrblks[i], iv);

		le128_inc(iv);

	}



	serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);