Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 149e1225 authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: x86/aesni - convert to use AEAD SIMD helpers 



Convert the AES-NI implementations of "gcm(aes)" and "rfc4106(gcm(aes))"
to use the AEAD SIMD helpers, rather than hand-rolling the same
functionality.  This simplifies the code and also fixes the bug where
the user-provided aead_request is modified.

Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 8b56d348

arch/x86/crypto/aesni-intel_glue.c

+15 −146
Original line number Diff line number Diff line
@@ -25,7 +25,6 @@ 
#include <linux/err.h>

#include <crypto/algapi.h>

#include <crypto/aes.h>

#include <crypto/cryptd.h>

#include <crypto/ctr.h>

#include <crypto/b128ops.h>

#include <crypto/gcm.h>
@@ -643,29 +642,6 @@ static int xts_decrypt(struct skcipher_request *req) 
				   aes_ctx(ctx->raw_crypt_ctx));

}



static int rfc4106_init(struct crypto_aead *aead)

{

	struct cryptd_aead *cryptd_tfm;

	struct cryptd_aead **ctx = crypto_aead_ctx(aead);



	cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",

				       CRYPTO_ALG_INTERNAL,

				       CRYPTO_ALG_INTERNAL);

	if (IS_ERR(cryptd_tfm))

		return PTR_ERR(cryptd_tfm);



	*ctx = cryptd_tfm;

	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));

	return 0;

}



static void rfc4106_exit(struct crypto_aead *aead)

{

	struct cryptd_aead **ctx = crypto_aead_ctx(aead);



	cryptd_free_aead(*ctx);

}



static int

rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)

{
@@ -710,15 +686,8 @@ static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 
	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);

}



static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,

				  unsigned int key_len)

{

	struct cryptd_aead **ctx = crypto_aead_ctx(parent);

	struct cryptd_aead *cryptd_tfm = *ctx;



	return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);

}



/* This is the Integrity Check Value (aka the authentication tag) length and can

 * be 8, 12 or 16 bytes long. */

static int common_rfc4106_set_authsize(struct crypto_aead *aead,

				       unsigned int authsize)

{
@@ -734,17 +703,6 @@ static int common_rfc4106_set_authsize(struct crypto_aead *aead, 
	return 0;

}



/* This is the Integrity Check Value (aka the authentication tag length and can

 * be 8, 12 or 16 bytes long. */

static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,

				       unsigned int authsize)

{

	struct cryptd_aead **ctx = crypto_aead_ctx(parent);

	struct cryptd_aead *cryptd_tfm = *ctx;



	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);

}



static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,

				       unsigned int authsize)

{
@@ -964,38 +922,6 @@ static int helper_rfc4106_decrypt(struct aead_request *req) 
	return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,

			      aes_ctx);

}



static int gcmaes_wrapper_encrypt(struct aead_request *req)

{

	struct crypto_aead *tfm = crypto_aead_reqtfm(req);

	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);

	struct cryptd_aead *cryptd_tfm = *ctx;



	tfm = &cryptd_tfm->base;

	if (irq_fpu_usable() && (!in_atomic() ||

				 !cryptd_aead_queued(cryptd_tfm)))

		tfm = cryptd_aead_child(cryptd_tfm);



	aead_request_set_tfm(req, tfm);



	return crypto_aead_encrypt(req);

}



static int gcmaes_wrapper_decrypt(struct aead_request *req)

{

	struct crypto_aead *tfm = crypto_aead_reqtfm(req);

	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);

	struct cryptd_aead *cryptd_tfm = *ctx;



	tfm = &cryptd_tfm->base;

	if (irq_fpu_usable() && (!in_atomic() ||

				 !cryptd_aead_queued(cryptd_tfm)))

		tfm = cryptd_aead_child(cryptd_tfm);



	aead_request_set_tfm(req, tfm);



	return crypto_aead_decrypt(req);

}

#endif



static struct crypto_alg aesni_algs[] = { {
@@ -1148,31 +1074,7 @@ static int generic_gcmaes_decrypt(struct aead_request *req) 
			      aes_ctx);

}



static int generic_gcmaes_init(struct crypto_aead *aead)

{

	struct cryptd_aead *cryptd_tfm;

	struct cryptd_aead **ctx = crypto_aead_ctx(aead);



	cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",

				       CRYPTO_ALG_INTERNAL,

				       CRYPTO_ALG_INTERNAL);

	if (IS_ERR(cryptd_tfm))

		return PTR_ERR(cryptd_tfm);



	*ctx = cryptd_tfm;

	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));



	return 0;

}



static void generic_gcmaes_exit(struct crypto_aead *aead)

{

	struct cryptd_aead **ctx = crypto_aead_ctx(aead);



	cryptd_free_aead(*ctx);

}



static struct aead_alg aesni_aead_algs[] = { {

static struct aead_alg aesni_aeads[] = { {

	.setkey			= common_rfc4106_set_key,

	.setauthsize		= common_rfc4106_set_authsize,

	.encrypt		= helper_rfc4106_encrypt,
@@ -1180,32 +1082,15 @@ static struct aead_alg aesni_aead_algs[] = { { 
	.ivsize			= GCM_RFC4106_IV_SIZE,

	.maxauthsize		= 16,

	.base = {

		.cra_name		= "__gcm-aes-aesni",

		.cra_driver_name	= "__driver-gcm-aes-aesni",

		.cra_name		= "__rfc4106(gcm(aes))",

		.cra_driver_name	= "__rfc4106-gcm-aesni",

		.cra_priority		= 400,

		.cra_flags		= CRYPTO_ALG_INTERNAL,

		.cra_blocksize		= 1,

		.cra_ctxsize		= sizeof(struct aesni_rfc4106_gcm_ctx),

		.cra_alignmask		= AESNI_ALIGN - 1,

		.cra_module		= THIS_MODULE,

	},

}, {

	.init			= rfc4106_init,

	.exit			= rfc4106_exit,

	.setkey			= gcmaes_wrapper_set_key,

	.setauthsize		= gcmaes_wrapper_set_authsize,

	.encrypt		= gcmaes_wrapper_encrypt,

	.decrypt		= gcmaes_wrapper_decrypt,

	.ivsize			= GCM_RFC4106_IV_SIZE,

	.maxauthsize		= 16,

	.base = {

		.cra_name		= "rfc4106(gcm(aes))",

		.cra_driver_name	= "rfc4106-gcm-aesni",

		.cra_priority		= 400,

		.cra_flags		= CRYPTO_ALG_ASYNC,

		.cra_blocksize		= 1,

		.cra_ctxsize		= sizeof(struct cryptd_aead *),

		.cra_module		= THIS_MODULE,

	},

}, {

	.setkey			= generic_gcmaes_set_key,

	.setauthsize		= generic_gcmaes_set_authsize,
@@ -1214,38 +1099,21 @@ static struct aead_alg aesni_aead_algs[] = { { 
	.ivsize			= GCM_AES_IV_SIZE,

	.maxauthsize		= 16,

	.base = {

		.cra_name		= "__generic-gcm-aes-aesni",

		.cra_driver_name	= "__driver-generic-gcm-aes-aesni",

		.cra_priority		= 0,

		.cra_name		= "__gcm(aes)",

		.cra_driver_name	= "__generic-gcm-aesni",

		.cra_priority		= 400,

		.cra_flags		= CRYPTO_ALG_INTERNAL,

		.cra_blocksize		= 1,

		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),

		.cra_alignmask		= AESNI_ALIGN - 1,

		.cra_module		= THIS_MODULE,

	},

}, {

	.init			= generic_gcmaes_init,

	.exit			= generic_gcmaes_exit,

	.setkey			= gcmaes_wrapper_set_key,

	.setauthsize		= gcmaes_wrapper_set_authsize,

	.encrypt		= gcmaes_wrapper_encrypt,

	.decrypt		= gcmaes_wrapper_decrypt,

	.ivsize			= GCM_AES_IV_SIZE,

	.maxauthsize		= 16,

	.base = {

		.cra_name		= "gcm(aes)",

		.cra_driver_name	= "generic-gcm-aesni",

		.cra_priority		= 400,

		.cra_flags		= CRYPTO_ALG_ASYNC,

		.cra_blocksize		= 1,

		.cra_ctxsize		= sizeof(struct cryptd_aead *),

		.cra_module		= THIS_MODULE,

	},

} };

#else

static struct aead_alg aesni_aead_algs[0];

static struct aead_alg aesni_aeads[0];

#endif



static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];



static const struct x86_cpu_id aesni_cpu_id[] = {

	X86_FEATURE_MATCH(X86_FEATURE_AES),
@@ -1296,8 +1164,8 @@ static int __init aesni_init(void) 
	if (err)

		goto unregister_algs;



	err = crypto_register_aeads(aesni_aead_algs,

				    ARRAY_SIZE(aesni_aead_algs));

	err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),

					 aesni_simd_aeads);

	if (err)

		goto unregister_skciphers;
@@ -1313,7 +1181,8 @@ static int __init aesni_init(void) 


static void __exit aesni_exit(void)

{

	crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));

	simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),

			      aesni_simd_aeads);

	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),

				  aesni_simd_skciphers);

	crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));