crypto: qat - add support for ctr(aes) and xts(aes) (def14bfa) · Commits · e / devices / android_kernel_xiaomi_sm6125

drivers/crypto/qat/qat_common/qat_algs.c

+127 −39

Original line number	Original line	Diff line number	Diff line
	@@ -62,13 +62,13 @@
	#include "icp_qat_fw.h"		#include "icp_qat_fw.h"
	#include "icp_qat_fw_la.h"		#include "icp_qat_fw_la.h"

	#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \		#define QAT_AES_HW_CONFIG_ENC(alg, mode) \
	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \		ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
	ICP_QAT_HW_CIPHER_NO_CONVERT, \		ICP_QAT_HW_CIPHER_NO_CONVERT, \
	ICP_QAT_HW_CIPHER_ENCRYPT)		ICP_QAT_HW_CIPHER_ENCRYPT)

	#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \		#define QAT_AES_HW_CONFIG_DEC(alg, mode) \
	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \		ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
	ICP_QAT_HW_CIPHER_KEY_CONVERT, \		ICP_QAT_HW_CIPHER_KEY_CONVERT, \
	ICP_QAT_HW_CIPHER_DECRYPT)		ICP_QAT_HW_CIPHER_DECRYPT)

	@@ -271,7 +271,8 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)

	static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,		static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
	int alg,		int alg,
	struct crypto_authenc_keys *keys)		struct crypto_authenc_keys *keys,
			int mode)
	{		{
	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);		struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
	unsigned int digestsize = crypto_aead_authsize(aead_tfm);		unsigned int digestsize = crypto_aead_authsize(aead_tfm);
	@@ -288,7 +289,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
	struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;		struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;

	/* CD setup */		/* CD setup */
	cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);		cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
	memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);		memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
	hash->sha.inner_setup.auth_config.config =		hash->sha.inner_setup.auth_config.config =
	ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,		ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
	@@ -351,7 +352,8 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,

	static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,		static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
	int alg,		int alg,
	struct crypto_authenc_keys *keys)		struct crypto_authenc_keys *keys,
			int mode)
	{		{
	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);		struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
	unsigned int digestsize = crypto_aead_authsize(aead_tfm);		unsigned int digestsize = crypto_aead_authsize(aead_tfm);
	@@ -373,7 +375,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
	sizeof(struct icp_qat_fw_la_cipher_req_params));		sizeof(struct icp_qat_fw_la_cipher_req_params));

	/* CD setup */		/* CD setup */
	cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);		cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
	memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);		memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
	hash->sha.inner_setup.auth_config.config =		hash->sha.inner_setup.auth_config.config =
	ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,		ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
	@@ -464,7 +466,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,

	static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,		static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
	int alg, const uint8_t *key,		int alg, const uint8_t *key,
	unsigned int keylen)		unsigned int keylen, int mode)
	{		{
	struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;		struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
	struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;		struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
	@@ -472,12 +474,12 @@ static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,

	qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);		qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;		cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
	enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);		enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
	}		}

	static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,		static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
	int alg, const uint8_t *key,		int alg, const uint8_t *key,
	unsigned int keylen)		unsigned int keylen, int mode)
	{		{
	struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;		struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
	struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;		struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
	@@ -485,11 +487,18 @@ static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,

	qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);		qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;		cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
	dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
			if (mode != ICP_QAT_HW_CIPHER_CTR_MODE)
			dec_cd->aes.cipher_config.val =
			QAT_AES_HW_CONFIG_DEC(alg, mode);
			else
			dec_cd->aes.cipher_config.val =
			QAT_AES_HW_CONFIG_ENC(alg, mode);
	}		}

	static int qat_alg_validate_key(int key_len, int *alg)		static int qat_alg_validate_key(int key_len, int *alg, int mode)
	{		{
			if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
	switch (key_len) {		switch (key_len) {
	case AES_KEYSIZE_128:		case AES_KEYSIZE_128:
	*alg = ICP_QAT_HW_CIPHER_ALGO_AES128;		*alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
	@@ -503,11 +512,23 @@ static int qat_alg_validate_key(int key_len, int *alg)
	default:		default:
	return -EINVAL;		return -EINVAL;
	}		}
			} else {
			switch (key_len) {
			case AES_KEYSIZE_128 << 1:
			*alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
			break;
			case AES_KEYSIZE_256 << 1:
			*alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
			break;
			default:
			return -EINVAL;
			}
			}
	return 0;		return 0;
	}		}

	static int qat_alg_aead_init_sessions(struct crypto_aead *tfm,		static int qat_alg_aead_init_sessions(struct crypto_aead tfm, const u8 key,
	const uint8_t *key, unsigned int keylen)		unsigned int keylen, int mode)
	{		{
	struct crypto_authenc_keys keys;		struct crypto_authenc_keys keys;
	int alg;		int alg;
	@@ -515,13 +536,13 @@ static int qat_alg_aead_init_sessions(struct crypto_aead *tfm,
	if (crypto_authenc_extractkeys(&keys, key, keylen))		if (crypto_authenc_extractkeys(&keys, key, keylen))
	goto bad_key;		goto bad_key;

	if (qat_alg_validate_key(keys.enckeylen, &alg))		if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
	goto bad_key;		goto bad_key;

	if (qat_alg_aead_init_enc_session(tfm, alg, &keys))		if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
	goto error;		goto error;

	if (qat_alg_aead_init_dec_session(tfm, alg, &keys))		if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
	goto error;		goto error;

	return 0;		return 0;
	@@ -534,15 +555,16 @@ static int qat_alg_aead_init_sessions(struct crypto_aead *tfm,

	static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,		static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
	const uint8_t *key,		const uint8_t *key,
	unsigned int keylen)		unsigned int keylen,
			int mode)
	{		{
	int alg;		int alg;

	if (qat_alg_validate_key(keylen, &alg))		if (qat_alg_validate_key(keylen, &alg, mode))
	goto bad_key;		goto bad_key;

	qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);		qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen, mode);
	qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);		qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen, mode);
	return 0;		return 0;
	bad_key:		bad_key:
	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);		crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
	@@ -586,7 +608,8 @@ static int qat_alg_aead_setkey(struct crypto_aead tfm, const uint8_t key,
	goto out_free_enc;		goto out_free_enc;
	}		}
	}		}
	if (qat_alg_aead_init_sessions(tfm, key, keylen))		if (qat_alg_aead_init_sessions(tfm, key, keylen,
			ICP_QAT_HW_CIPHER_CBC_MODE))
	goto out_free_all;		goto out_free_all;

	return 0;		return 0;
	@@ -876,8 +899,8 @@ static int qat_alg_aead_enc(struct aead_request *areq)
	}		}

	static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,		static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
	const uint8_t *key,		const u8 *key, unsigned int keylen,
	unsigned int keylen)		int mode)
	{		{
	struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);		struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
	struct device *dev;		struct device *dev;
	@@ -918,7 +941,7 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
	}		}
	}		}
	spin_unlock(&ctx->lock);		spin_unlock(&ctx->lock);
	if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))		if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode))
	goto out_free_all;		goto out_free_all;

	return 0;		return 0;
	@@ -936,6 +959,27 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
	return -ENOMEM;		return -ENOMEM;
	}		}

			static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm,
			const u8 *key, unsigned int keylen)
			{
			return qat_alg_ablkcipher_setkey(tfm, key, keylen,
			ICP_QAT_HW_CIPHER_CBC_MODE);
			}

			static int qat_alg_ablkcipher_ctr_setkey(struct crypto_ablkcipher *tfm,
			const u8 *key, unsigned int keylen)
			{
			return qat_alg_ablkcipher_setkey(tfm, key, keylen,
			ICP_QAT_HW_CIPHER_CTR_MODE);
			}

			static int qat_alg_ablkcipher_xts_setkey(struct crypto_ablkcipher *tfm,
			const u8 *key, unsigned int keylen)
			{
			return qat_alg_ablkcipher_setkey(tfm, key, keylen,
			ICP_QAT_HW_CIPHER_XTS_MODE);
			}

	static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)		static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
	{		{
	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);		struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
	@@ -1171,7 +1215,51 @@ static struct crypto_alg qat_algs[] = { {
	.cra_exit = qat_alg_ablkcipher_exit,		.cra_exit = qat_alg_ablkcipher_exit,
	.cra_u = {		.cra_u = {
	.ablkcipher = {		.ablkcipher = {
	.setkey = qat_alg_ablkcipher_setkey,		.setkey = qat_alg_ablkcipher_cbc_setkey,
			.decrypt = qat_alg_ablkcipher_decrypt,
			.encrypt = qat_alg_ablkcipher_encrypt,
			.min_keysize = AES_MIN_KEY_SIZE,
			.max_keysize = AES_MAX_KEY_SIZE,
			.ivsize = AES_BLOCK_SIZE,
			},
			},
			}, {
			.cra_name = "ctr(aes)",
			.cra_driver_name = "qat_aes_ctr",
			.cra_priority = 4001,
			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
			.cra_alignmask = 0,
			.cra_type = &crypto_ablkcipher_type,
			.cra_module = THIS_MODULE,
			.cra_init = qat_alg_ablkcipher_init,
			.cra_exit = qat_alg_ablkcipher_exit,
			.cra_u = {
			.ablkcipher = {
			.setkey = qat_alg_ablkcipher_ctr_setkey,
			.decrypt = qat_alg_ablkcipher_decrypt,
			.encrypt = qat_alg_ablkcipher_encrypt,
			.min_keysize = AES_MIN_KEY_SIZE,
			.max_keysize = AES_MAX_KEY_SIZE,
			.ivsize = AES_BLOCK_SIZE,
			},
			},
			}, {
			.cra_name = "xts(aes)",
			.cra_driver_name = "qat_aes_xts",
			.cra_priority = 4001,
			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
			.cra_alignmask = 0,
			.cra_type = &crypto_ablkcipher_type,
			.cra_module = THIS_MODULE,
			.cra_init = qat_alg_ablkcipher_init,
			.cra_exit = qat_alg_ablkcipher_exit,
			.cra_u = {
			.ablkcipher = {
			.setkey = qat_alg_ablkcipher_xts_setkey,
	.decrypt = qat_alg_ablkcipher_decrypt,		.decrypt = qat_alg_ablkcipher_decrypt,
	.encrypt = qat_alg_ablkcipher_encrypt,		.encrypt = qat_alg_ablkcipher_encrypt,
	.min_keysize = AES_MIN_KEY_SIZE,		.min_keysize = AES_MIN_KEY_SIZE,