Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5092fcf3 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

crypto: arm64/aes-ce-ccm: add non-SIMD generic fallback 



The arm64 kernel will shortly disallow nested kernel mode NEON.

So honour this in the ARMv8 Crypto Extensions implementation of
CCM-AES, and fall back to a scalar implementation using the generic
crypto helpers for AES, XOR and incrementing the CTR counter.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent b8fb993a

arch/arm64/crypto/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -59,6 +59,7 @@ config CRYPTO_AES_ARM64_CE_CCM 
	depends on ARM64 && KERNEL_MODE_NEON

	select CRYPTO_ALGAPI

	select CRYPTO_AES_ARM64_CE

	select CRYPTO_AES_ARM64

	select CRYPTO_AEAD



config CRYPTO_AES_ARM64_CE_BLK

arch/arm64/crypto/aes-ce-ccm-glue.c

+139 −35
Original line number Diff line number Diff line 
/*

 * aes-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions

 *

 * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>

 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as
@@ -9,6 +9,7 @@ 
 */



#include <asm/neon.h>

#include <asm/simd.h>

#include <asm/unaligned.h>

#include <crypto/aes.h>

#include <crypto/scatterwalk.h>
@@ -44,6 +45,8 @@ asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, 
asmlinkage void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u32 const rk[],

				 u32 rounds);



asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);



static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,

		      unsigned int key_len)

{
@@ -103,7 +106,45 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) 
	return 0;

}



static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])

static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[],

			   u32 abytes, u32 *macp, bool use_neon)

{

	if (likely(use_neon)) {

		ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc,

				     num_rounds(key));

	} else {

		if (*macp > 0 && *macp < AES_BLOCK_SIZE) {

			int added = min(abytes, AES_BLOCK_SIZE - *macp);



			crypto_xor(&mac[*macp], in, added);



			*macp += added;

			in += added;

			abytes -= added;

		}



		while (abytes > AES_BLOCK_SIZE) {

			__aes_arm64_encrypt(key->key_enc, mac, mac,

					    num_rounds(key));

			crypto_xor(mac, in, AES_BLOCK_SIZE);



			in += AES_BLOCK_SIZE;

			abytes -= AES_BLOCK_SIZE;

		}



		if (abytes > 0) {

			__aes_arm64_encrypt(key->key_enc, mac, mac,

					    num_rounds(key));

			crypto_xor(mac, in, abytes);

			*macp = abytes;

		} else {

			*macp = 0;

		}

	}

}



static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[],

				   bool use_neon)

{

	struct crypto_aead *aead = crypto_aead_reqtfm(req);

	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
@@ -122,8 +163,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 
		ltag.len = 6;

	}



	ce_aes_ccm_auth_data(mac, (u8 *)&ltag, ltag.len, &macp, ctx->key_enc,

			     num_rounds(ctx));

	ccm_update_mac(ctx, mac, (u8 *)&ltag, ltag.len, &macp, use_neon);

	scatterwalk_start(&walk, req->src);



	do {
@@ -135,8 +175,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 
			n = scatterwalk_clamp(&walk, len);

		}

		p = scatterwalk_map(&walk);

		ce_aes_ccm_auth_data(mac, p, n, &macp, ctx->key_enc,

				     num_rounds(ctx));

		ccm_update_mac(ctx, mac, p, n, &macp, use_neon);

		len -= n;



		scatterwalk_unmap(p);
@@ -145,6 +184,56 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 
	} while (len);

}



static int ccm_crypt_fallback(struct skcipher_walk *walk, u8 mac[], u8 iv0[],

			      struct crypto_aes_ctx *ctx, bool enc)

{

	u8 buf[AES_BLOCK_SIZE];

	int err = 0;



	while (walk->nbytes) {

		int blocks = walk->nbytes / AES_BLOCK_SIZE;

		u32 tail = walk->nbytes % AES_BLOCK_SIZE;

		u8 *dst = walk->dst.virt.addr;

		u8 *src = walk->src.virt.addr;

		u32 nbytes = walk->nbytes;



		if (nbytes == walk->total && tail > 0) {

			blocks++;

			tail = 0;

		}



		do {

			u32 bsize = AES_BLOCK_SIZE;



			if (nbytes < AES_BLOCK_SIZE)

				bsize = nbytes;



			crypto_inc(walk->iv, AES_BLOCK_SIZE);

			__aes_arm64_encrypt(ctx->key_enc, buf, walk->iv,

					    num_rounds(ctx));

			__aes_arm64_encrypt(ctx->key_enc, mac, mac,

					    num_rounds(ctx));

			if (enc)

				crypto_xor(mac, src, bsize);

			crypto_xor_cpy(dst, src, buf, bsize);

			if (!enc)

				crypto_xor(mac, dst, bsize);

			dst += bsize;

			src += bsize;

			nbytes -= bsize;

		} while (--blocks);



		err = skcipher_walk_done(walk, tail);

	}



	if (!err) {

		__aes_arm64_encrypt(ctx->key_enc, buf, iv0, num_rounds(ctx));

		__aes_arm64_encrypt(ctx->key_enc, mac, mac, num_rounds(ctx));

		crypto_xor(mac, buf, AES_BLOCK_SIZE);

	}

	return err;

}



static int ccm_encrypt(struct aead_request *req)

{

	struct crypto_aead *aead = crypto_aead_reqtfm(req);
@@ -153,39 +242,46 @@ static int ccm_encrypt(struct aead_request *req) 
	u8 __aligned(8) mac[AES_BLOCK_SIZE];

	u8 buf[AES_BLOCK_SIZE];

	u32 len = req->cryptlen;

	bool use_neon = may_use_simd();

	int err;



	err = ccm_init_mac(req, mac, len);

	if (err)

		return err;



	kernel_neon_begin_partial(6);

	if (likely(use_neon))

		kernel_neon_begin();



	if (req->assoclen)

		ccm_calculate_auth_mac(req, mac);

		ccm_calculate_auth_mac(req, mac, use_neon);



	/* preserve the original iv for the final round */

	memcpy(buf, req->iv, AES_BLOCK_SIZE);



	err = skcipher_walk_aead_encrypt(&walk, req, true);



	if (likely(use_neon)) {

		while (walk.nbytes) {

			u32 tail = walk.nbytes % AES_BLOCK_SIZE;



			if (walk.nbytes == walk.total)

				tail = 0;



		ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

			ce_aes_ccm_encrypt(walk.dst.virt.addr,

					   walk.src.virt.addr,

					   walk.nbytes - tail, ctx->key_enc,

					   num_rounds(ctx), mac, walk.iv);



			err = skcipher_walk_done(&walk, tail);

		}

		if (!err)

		ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));

			ce_aes_ccm_final(mac, buf, ctx->key_enc,

					 num_rounds(ctx));



		kernel_neon_end();



	} else {

		err = ccm_crypt_fallback(&walk, mac, buf, ctx, true);

	}

	if (err)

		return err;
@@ -205,38 +301,46 @@ static int ccm_decrypt(struct aead_request *req) 
	u8 __aligned(8) mac[AES_BLOCK_SIZE];

	u8 buf[AES_BLOCK_SIZE];

	u32 len = req->cryptlen - authsize;

	bool use_neon = may_use_simd();

	int err;



	err = ccm_init_mac(req, mac, len);

	if (err)

		return err;



	kernel_neon_begin_partial(6);

	if (likely(use_neon))

		kernel_neon_begin();



	if (req->assoclen)

		ccm_calculate_auth_mac(req, mac);

		ccm_calculate_auth_mac(req, mac, use_neon);



	/* preserve the original iv for the final round */

	memcpy(buf, req->iv, AES_BLOCK_SIZE);



	err = skcipher_walk_aead_decrypt(&walk, req, true);



	if (likely(use_neon)) {

		while (walk.nbytes) {

			u32 tail = walk.nbytes % AES_BLOCK_SIZE;



			if (walk.nbytes == walk.total)

				tail = 0;



		ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

			ce_aes_ccm_decrypt(walk.dst.virt.addr,

					   walk.src.virt.addr,

					   walk.nbytes - tail, ctx->key_enc,

					   num_rounds(ctx), mac, walk.iv);



			err = skcipher_walk_done(&walk, tail);

		}

		if (!err)

		ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));

			ce_aes_ccm_final(mac, buf, ctx->key_enc,

					 num_rounds(ctx));



		kernel_neon_end();

	} else {

		err = ccm_crypt_fallback(&walk, mac, buf, ctx, false);

	}



	if (err)

		return err;