Merge "crypto: msm: add dynamic engine assignment support" (f1b37554) · Commits · e / devices / android_kernel_sony_msm8994

drivers/crypto/msm/qcrypto.c

+302 −139

Original line number	Diff line number	Diff line
		@@ -59,58 +59,60 @@
		#define QCRYPTO_HIGH_BANDWIDTH_TIMEOUT 1000

		struct crypto_stat {
		u32 aead_sha1_aes_enc;
		u32 aead_sha1_aes_dec;
		u32 aead_sha1_des_enc;
		u32 aead_sha1_des_dec;
		u32 aead_sha1_3des_enc;
		u32 aead_sha1_3des_dec;
		u32 aead_ccm_aes_enc;
		u32 aead_ccm_aes_dec;
		u32 aead_rfc4309_ccm_aes_enc;
		u32 aead_rfc4309_ccm_aes_dec;
		u32 aead_op_success;
		u32 aead_op_fail;
		u32 aead_bad_msg;
		u32 ablk_cipher_aes_enc;
		u32 ablk_cipher_aes_dec;
		u32 ablk_cipher_des_enc;
		u32 ablk_cipher_des_dec;
		u32 ablk_cipher_3des_enc;
		u32 ablk_cipher_3des_dec;
		u32 ablk_cipher_op_success;
		u32 ablk_cipher_op_fail;
		u32 sha1_digest;
		u32 sha256_digest;
		u32 sha_op_success;
		u32 sha_op_fail;
		u32 sha1_hmac_digest;
		u32 sha256_hmac_digest;
		u32 sha_hmac_op_success;
		u32 sha_hmac_op_fail;
		u64 aead_sha1_aes_enc;
		u64 aead_sha1_aes_dec;
		u64 aead_sha1_des_enc;
		u64 aead_sha1_des_dec;
		u64 aead_sha1_3des_enc;
		u64 aead_sha1_3des_dec;
		u64 aead_ccm_aes_enc;
		u64 aead_ccm_aes_dec;
		u64 aead_rfc4309_ccm_aes_enc;
		u64 aead_rfc4309_ccm_aes_dec;
		u64 aead_op_success;
		u64 aead_op_fail;
		u64 aead_bad_msg;
		u64 ablk_cipher_aes_enc;
		u64 ablk_cipher_aes_dec;
		u64 ablk_cipher_des_enc;
		u64 ablk_cipher_des_dec;
		u64 ablk_cipher_3des_enc;
		u64 ablk_cipher_3des_dec;
		u64 ablk_cipher_op_success;
		u64 ablk_cipher_op_fail;
		u64 sha1_digest;
		u64 sha256_digest;
		u64 sha_op_success;
		u64 sha_op_fail;
		u64 sha1_hmac_digest;
		u64 sha256_hmac_digest;
		u64 sha_hmac_op_success;
		u64 sha_hmac_op_fail;
		};
		static struct crypto_stat _qcrypto_stat;
		static struct dentry *_debug_dent;
		static char _debug_read_buf[DEBUG_MAX_RW_BUF];
		static bool _qcrypto_init_assign;
		struct crypto_priv;
		struct crypto_engine {
		struct list_head elist;
		void qce; / qce handle */
		struct platform_device pdev; / platform device */
		struct crypto_async_request req; / current active request */
		struct qcrypto_resp_ctx arsp; / rsp associcated with req */
		int res; /* execution result */
		struct crypto_priv *pcp;
		struct tasklet_struct done_tasklet;
		uint32_t bus_scale_handle;
		struct crypto_queue req_queue; /*
		* request queue for those requests
		* that have this engine assgined
		* that have this engine assigned
		* waiting to be executed
		*/
		u32 total_req;
		u32 err_req;
		u64 total_req;
		u64 err_req;
		u32 unit;
		u32 ce_device;
		int res; /* execution result */
		unsigned int signature;
		uint32_t high_bw_req_count;
		bool high_bw_req;
		@@ -140,6 +142,12 @@ struct crypto_priv {
		int32_t total_units; /* total units of engines */
		struct mutex engine_lock;
		struct crypto_engine next_engine; / next assign engine */
		struct crypto_queue req_queue; /*
		* request queue for those requests
		* that waiting for an available
		* engine.
		*/

		};
		static struct crypto_priv qcrypto_dev;
		static struct crypto_engine *_qcrypto_static_assign_engine(
		@@ -261,6 +269,11 @@ struct qcrypto_alg {
		#define QCRYPTO_CCM4309_NONCE_LEN 3

		struct qcrypto_cipher_ctx {
		struct list_head rsp_queue; /* response queue */
		struct crypto_engine pengine; / fixed engine assigned to this tfm */
		struct crypto_priv *cp;
		unsigned int flags;

		u8 auth_key[QCRYPTO_MAX_KEY_SIZE];
		u8 iv[QCRYPTO_MAX_IV_LENGTH];

		@@ -270,13 +283,18 @@ struct qcrypto_cipher_ctx {
		unsigned int authsize;
		unsigned int auth_key_len;

		struct crypto_priv *cp;
		unsigned int flags;
		struct crypto_engine pengine; / fixed engine assigned */
		u8 ccm4309_nonce[QCRYPTO_CCM4309_NONCE_LEN];
		};

		struct qcrypto_resp_ctx {
		struct list_head list;
		struct crypto_async_request async_req; / async req */
		int res; /* execution result */
		};

		struct qcrypto_cipher_req_ctx {
		struct qcrypto_resp_ctx rsp_entry;/* rsp entry. */
		struct crypto_engine pengine; / engine assigned to this request */
		u8 *iv;
		u8 rfc4309_iv[QCRYPTO_MAX_IV_LENGTH];
		unsigned int ivsize;
		@@ -301,6 +319,8 @@ struct qcrypto_cipher_req_ctx {
		#define SHA_MAX_STATE_SIZE (SHA256_DIGEST_SIZE / sizeof(u32))
		#define SHA_MAX_DIGEST_SIZE SHA256_DIGEST_SIZE

		#define MSM_QCRYPTO_REQ_QUEUE_LENGTH 50

		static uint8_t _std_init_vector_sha1_uint8[] = {
		0x67, 0x45, 0x23, 0x01, 0xEF, 0xCD, 0xAB, 0x89,
		0x98, 0xBA, 0xDC, 0xFE, 0x10, 0x32, 0x54, 0x76,
		@@ -316,18 +336,21 @@ static uint8_t _std_init_vector_sha256_uint8[] = {
		};

		struct qcrypto_sha_ctx {
		struct list_head rsp_queue; /* response queue */
		struct crypto_engine pengine; / fixed engine assigned to this tfm */
		struct crypto_priv *cp;
		unsigned int flags;
		enum qce_hash_alg_enum alg;
		uint32_t diglen;
		uint32_t authkey_in_len;
		uint8_t authkey[SHA_MAX_BLOCK_SIZE];
		struct ahash_request *ahash_req;
		struct completion ahash_req_complete;
		struct crypto_priv *cp;
		unsigned int flags;
		struct crypto_engine pengine; / fixed engine assigned */
		};

		struct qcrypto_sha_req_ctx {
		struct qcrypto_resp_ctx rsp_entry;/* rsp entry. */
		struct crypto_engine pengine; / engine assigned to this request */

		struct scatterlist *src;
		uint32_t nbytes;
		@@ -411,11 +434,10 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine,
		int ret = 0;

		if (high_bw_req && pengine->high_bw_req == false) {
		pm_stay_awake(&pengine->pdev->dev);
		ret = qce_enable_clk(pengine->qce);
		if (ret) {
		pr_err("%s Unable enable clk\n", __func__);
		goto clk_err;
		return;
		}
		ret = msm_bus_scale_client_update_request(
		pengine->bus_scale_handle, 1);
		@@ -423,7 +445,7 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine,
		pr_err("%s Unable to set to high bandwidth\n",
		__func__);
		qce_disable_clk(pengine->qce);
		goto clk_err;
		return;
		}
		pengine->high_bw_req = true;
		} else if (high_bw_req == false && pengine->high_bw_req == true) {
		@@ -432,7 +454,7 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine,
		if (ret) {
		pr_err("%s Unable to set to low bandwidth\n",
		__func__);
		goto clk_err;
		return;
		}
		ret = qce_disable_clk(pengine->qce);
		if (ret) {
		@@ -442,16 +464,10 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine,
		if (ret)
		pr_err("%s Unable to set to high bandwidth\n",
		__func__);
		goto clk_err;
		return;
		}
		pengine->high_bw_req = false;
		pm_relax(&pengine->pdev->dev);
		}
		return;
		clk_err:
		pm_relax(&pengine->pdev->dev);
		return;

		}

		static void qcrypto_bw_scale_down_timer_callback(unsigned long data)
		@@ -473,10 +489,15 @@ static void qcrypto_bw_set_timeout(struct crypto_engine *pengine)
		add_timer(&(pengine->bw_scale_down_timer));
		}

		static void qcrypto_ce_bw_scaling_req(struct crypto_engine *pengine,
		static void qcrypto_ce_bw_scaling_req(struct crypto_priv *cp,
		bool high_bw_req)
		{
		mutex_lock(&pengine->pcp->engine_lock);
		struct crypto_engine *pengine;

		if (cp->platform_support.bus_scale_table == NULL)
		return;
		mutex_lock(&cp->engine_lock);
		list_for_each_entry(pengine, &cp->engine_list, elist) {
		if (high_bw_req) {
		if (pengine->high_bw_req_count == 0)
		qcrypto_ce_set_bus(pengine, true);
		@@ -486,7 +507,8 @@ static void qcrypto_ce_bw_scaling_req(struct crypto_engine *pengine,
		if (pengine->high_bw_req_count == 0)
		qcrypto_bw_set_timeout(pengine);
		}
		mutex_unlock(&pengine->pcp->engine_lock);
		}
		mutex_unlock(&cp->engine_lock);
		}

		static void qcrypto_low_bw_req_work(struct work_struct *work)
		@@ -597,11 +619,14 @@ static int _qcrypto_cipher_cra_init(struct crypto_tfm *tfm)

		/* random first IV */
		get_random_bytes(ctx->iv, QCRYPTO_MAX_IV_LENGTH);
		if (_qcrypto_init_assign) {
		ctx->pengine = _qcrypto_static_assign_engine(ctx->cp);
		if (ctx->pengine == NULL)
		return -ENODEV;
		if (ctx->cp->platform_support.bus_scale_table != NULL)
		qcrypto_ce_bw_scaling_req(ctx->pengine, true);
		} else
		ctx->pengine = NULL;
		qcrypto_ce_bw_scaling_req(ctx->cp, true);
		INIT_LIST_HEAD(&ctx->rsp_queue);
		return 0;
		};

		@@ -619,11 +644,14 @@ static int _qcrypto_ahash_cra_init(struct crypto_tfm *tfm)
		sha_ctx->cp = q_alg->cp;
		sha_ctx->flags = 0;
		sha_ctx->ahash_req = NULL;
		if (_qcrypto_init_assign) {
		sha_ctx->pengine = _qcrypto_static_assign_engine(sha_ctx->cp);
		if (sha_ctx->pengine == NULL)
		return -ENODEV;
		if (sha_ctx->cp->platform_support.bus_scale_table != NULL)
		qcrypto_ce_bw_scaling_req(sha_ctx->pengine, true);
		} else
		sha_ctx->pengine = NULL;
		qcrypto_ce_bw_scaling_req(sha_ctx->cp, true);
		INIT_LIST_HEAD(&sha_ctx->rsp_queue);
		return 0;
		};

		@@ -631,13 +659,13 @@ static void _qcrypto_ahash_cra_exit(struct crypto_tfm *tfm)
		{
		struct qcrypto_sha_ctx *sha_ctx = crypto_tfm_ctx(tfm);

		if (!list_empty(&sha_ctx->rsp_queue))
		pr_err("_qcrypto_ahash_cra_exit: requests still outstanding");
		if (sha_ctx->ahash_req != NULL) {
		ahash_request_free(sha_ctx->ahash_req);
		sha_ctx->ahash_req = NULL;
		}
		if (sha_ctx->pengine &&
		sha_ctx->cp->platform_support.bus_scale_table != NULL)
		qcrypto_ce_bw_scaling_req(sha_ctx->pengine, false);
		qcrypto_ce_bw_scaling_req(sha_ctx->cp, false);
		};


		@@ -686,16 +714,18 @@ static void _qcrypto_cra_ablkcipher_exit(struct crypto_tfm *tfm)
		{
		struct qcrypto_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

		if (ctx->pengine && ctx->cp->platform_support.bus_scale_table != NULL)
		qcrypto_ce_bw_scaling_req(ctx->pengine, false);
		if (!list_empty(&ctx->rsp_queue))
		pr_err("_qcrypto__cra_ablkcipher_exit: requests still outstanding");
		qcrypto_ce_bw_scaling_req(ctx->cp, false);
		};

		static void _qcrypto_cra_aead_exit(struct crypto_tfm *tfm)
		{
		struct qcrypto_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

		if (ctx->pengine && ctx->cp->platform_support.bus_scale_table != NULL)
		qcrypto_ce_bw_scaling_req(ctx->pengine, false);
		if (!list_empty(&ctx->rsp_queue))
		pr_err("_qcrypto__cra_aead_exit: requests still outstanding");
		qcrypto_ce_bw_scaling_req(ctx->cp, false);
		};

		static int _disp_stats(int id)
		@@ -708,117 +738,117 @@ static int _disp_stats(int id)

		pstat = &_qcrypto_stat;
		len = scnprintf(_debug_read_buf, DEBUG_MAX_RW_BUF - 1,
		"\nQualcomm crypto accelerator %d Statistics:\n",
		"\nQualcomm crypto accelerator %d Statistics\n",
		id + 1);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK AES CIPHER encryption : %d\n",
		" ABLK AES CIPHER encryption : %llu\n",
		pstat->ablk_cipher_aes_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK AES CIPHER decryption : %d\n",
		" ABLK AES CIPHER decryption : %llu\n",
		pstat->ablk_cipher_aes_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK DES CIPHER encryption : %d\n",
		" ABLK DES CIPHER encryption : %llu\n",
		pstat->ablk_cipher_des_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK DES CIPHER decryption : %d\n",
		" ABLK DES CIPHER decryption : %llu\n",
		pstat->ablk_cipher_des_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK 3DES CIPHER encryption : %d\n",
		" ABLK 3DES CIPHER encryption : %llu\n",
		pstat->ablk_cipher_3des_enc);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK 3DES CIPHER decryption : %d\n",
		" ABLK 3DES CIPHER decryption : %llu\n",
		pstat->ablk_cipher_3des_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK CIPHER operation success: %d\n",
		" ABLK CIPHER operation success : %llu\n",
		pstat->ablk_cipher_op_success);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" ABLK CIPHER operation fail : %d\n",
		" ABLK CIPHER operation fail : %llu\n",
		pstat->ablk_cipher_op_fail);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-AES encryption : %d\n",
		" AEAD SHA1-AES encryption : %llu\n",
		pstat->aead_sha1_aes_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-AES decryption : %d\n",
		" AEAD SHA1-AES decryption : %llu\n",
		pstat->aead_sha1_aes_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-DES encryption : %d\n",
		" AEAD SHA1-DES encryption : %llu\n",
		pstat->aead_sha1_des_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-DES decryption : %d\n",
		" AEAD SHA1-DES decryption : %llu\n",
		pstat->aead_sha1_des_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-3DES encryption : %d\n",
		" AEAD SHA1-3DES encryption : %llu\n",
		pstat->aead_sha1_3des_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD SHA1-3DES decryption : %d\n",
		" AEAD SHA1-3DES decryption : %llu\n",
		pstat->aead_sha1_3des_dec);

		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD CCM-AES encryption : %d\n",
		" AEAD CCM-AES encryption : %llu\n",
		pstat->aead_ccm_aes_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD CCM-AES decryption : %d\n",
		" AEAD CCM-AES decryption : %llu\n",
		pstat->aead_ccm_aes_dec);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD RFC4309-CCM-AES encryption : %d\n",
		" AEAD RFC4309-CCM-AES encryption : %llu\n",
		pstat->aead_rfc4309_ccm_aes_enc);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD RFC4309-CCM-AES decryption : %d\n",
		" AEAD RFC4309-CCM-AES decryption : %llu\n",
		pstat->aead_rfc4309_ccm_aes_dec);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD operation success : %d\n",
		" AEAD operation success : %llu\n",
		pstat->aead_op_success);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD operation fail : %d\n",
		" AEAD operation fail : %llu\n",
		pstat->aead_op_fail);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" AEAD bad message : %d\n",
		" AEAD bad message : %llu\n",
		pstat->aead_bad_msg);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA1 digest : %d\n",
		" SHA1 digest : %llu\n",
		pstat->sha1_digest);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA256 digest : %d\n",
		" SHA256 digest : %llu\n",
		pstat->sha256_digest);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA operation fail : %d\n",
		" SHA operation fail : %llu\n",
		pstat->sha_op_fail);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA operation success : %d\n",
		" SHA operation success : %llu\n",
		pstat->sha_op_success);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA1 HMAC digest : %d\n",
		" SHA1 HMAC digest : %llu\n",
		pstat->sha1_hmac_digest);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA256 HMAC digest : %d\n",
		" SHA256 HMAC digest : %llu\n",
		pstat->sha256_hmac_digest);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA HMAC operation fail : %d\n",
		" SHA HMAC operation fail : %llu\n",
		pstat->sha_hmac_op_fail);
		len += scnprintf(_debug_read_buf + len, DEBUG_MAX_RW_BUF - len - 1,
		" SHA HMAC operation success : %d\n",
		" SHA HMAC operation success : %llu\n",
		pstat->sha_hmac_op_success);
		spin_lock_irqsave(&cp->lock, flags);
		list_for_each_entry(pe, &cp->engine_list, elist) {
		len += scnprintf(
		_debug_read_buf + len,
		DEBUG_MAX_RW_BUF - len - 1,
		" Engine %d Req : %d\n",
		" Engine %4d Req : %llu\n",
		pe->unit,
		pe->total_req
		);
		len += scnprintf(
		_debug_read_buf + len,
		DEBUG_MAX_RW_BUF - len - 1,
		" Engine %d Req Error : %d\n",
		" Engine %4d Req Error : %llu\n",
		pe->unit,
		pe->err_req
		);
		@@ -847,7 +877,6 @@ static void _qcrypto_remove_engine(struct crypto_engine *pengine)
		tasklet_kill(&pengine->done_tasklet);
		cancel_work_sync(&pengine->low_bw_req_ws);
		del_timer_sync(&pengine->bw_scale_down_timer);
		device_init_wakeup(&pengine->pdev->dev, false);

		if (pengine->bus_scale_handle != 0)
		msm_bus_scale_unregister_client(pengine->bus_scale_handle);
		@@ -1020,27 +1049,73 @@ static int _qcrypto_setkey_3des(struct crypto_ablkcipher cipher, const u8 key,
		return 0;
		};

		static void _qcrypto_tfm_complete(struct crypto_priv *cp, u32 type,
		void *tfm_ctx)
		{
		unsigned long flags;
		struct qcrypto_resp_ctx *arsp;
		struct list_head *plist;
		struct crypto_async_request *areq;

		switch (type) {
		case CRYPTO_ALG_TYPE_AHASH:
		plist = &((struct qcrypto_sha_ctx *) tfm_ctx)->rsp_queue;
		break;
		case CRYPTO_ALG_TYPE_ABLKCIPHER:
		case CRYPTO_ALG_TYPE_AEAD:
		default:
		plist = &((struct qcrypto_cipher_ctx *) tfm_ctx)->rsp_queue;
		break;
		}
		again:
		spin_lock_irqsave(&cp->lock, flags);
		if (list_empty(plist)) {
		arsp = NULL; /* nothing to do */
		} else {
		arsp = list_first_entry(plist,
		struct qcrypto_resp_ctx, list);
		if (arsp->res == -EINPROGRESS)
		arsp = NULL; /* still in progress */
		else
		list_del(&arsp->list); /* request is complete */
		}
		spin_unlock_irqrestore(&cp->lock, flags);
		if (arsp) {
		areq = arsp->async_req;
		areq->complete(areq, arsp->res);
		goto again;
		}
		}

		static void req_done(unsigned long data)
		{
		struct crypto_async_request *areq;
		struct crypto_engine pengine = (struct crypto_engine )data;
		struct crypto_priv *cp;
		unsigned long flags;
		struct qcrypto_resp_ctx *arsp;
		int res;
		u32 type = 0;
		void *tfm_ctx = NULL;

		cp = pengine->pcp;
		spin_lock_irqsave(&cp->lock, flags);
		areq = pengine->req;
		pengine->req = NULL;

		arsp = pengine->arsp;
		res = pengine->res;
		pengine->req = NULL;
		pengine->arsp = NULL;
		if (areq) {
		type = crypto_tfm_alg_type(areq->tfm);
		tfm_ctx = crypto_tfm_ctx(areq->tfm);
		arsp->res = res;
		}
		spin_unlock_irqrestore(&cp->lock, flags);
		if (areq)
		areq->complete(areq, res);
		if (res)
		pengine->err_req++;
		_start_qcrypto_process(cp, pengine);
		};

		if (areq)
		_qcrypto_tfm_complete(cp, type, tfm_ctx);
		}

		static void _qce_ahash_complete(void cookie, unsigned char digest,
		unsigned char *authdata, int ret)
		@@ -1057,7 +1132,7 @@ static void _qce_ahash_complete(void cookie, unsigned char digest,

		pstat = &_qcrypto_stat;

		pengine = sha_ctx->pengine;
		pengine = rctx->pengine;
		#ifdef QCRYPTO_DEBUG
		dev_info(&pengine->pdev->dev, "_qce_ahash_complete: %p ret %d\n",
		areq, ret);
		@@ -1103,10 +1178,12 @@ static void _qce_ablk_cipher_complete(void cookie, unsigned char icb,
		struct qcrypto_cipher_ctx *ctx = crypto_tfm_ctx(areq->base.tfm);
		struct crypto_priv *cp = ctx->cp;
		struct crypto_stat *pstat;
		struct qcrypto_cipher_req_ctx *rctx;
		struct crypto_engine *pengine;

		pstat = &_qcrypto_stat;
		pengine = ctx->pengine;
		rctx = ablkcipher_request_ctx(areq);
		pengine = rctx->pengine;
		#ifdef QCRYPTO_DEBUG
		dev_info(&pengine->pdev->dev, "_qce_ablk_cipher_complete: %p ret %d\n",
		areq, ret);
		@@ -1158,8 +1235,8 @@ static void _qce_aead_complete(void cookie, unsigned char icv,
		struct crypto_engine *pengine;

		pstat = &_qcrypto_stat;
		pengine = ctx->pengine;
		rctx = aead_request_ctx(areq);
		pengine = rctx->pengine;

		if (rctx->mode == QCE_MODE_CCM) {
		if (cp->ce_support.aligned_only) {
		@@ -1363,6 +1440,7 @@ static int _qcrypto_process_ablkcipher(struct crypto_engine *pengine,
		req = container_of(async_req, struct ablkcipher_request, base);
		cipher_ctx = crypto_tfm_ctx(async_req->tfm);
		rctx = ablkcipher_request_ctx(req);
		rctx->pengine = pengine;
		tfm = crypto_ablkcipher_reqtfm(req);
		if (pengine->pcp->ce_support.aligned_only) {
		uint32_t bytes = 0;
		@@ -1426,6 +1504,7 @@ static int _qcrypto_process_ahash(struct crypto_engine *pengine,
		struct ahash_request, base);
		rctx = ahash_request_ctx(req);
		sha_ctx = crypto_tfm_ctx(async_req->tfm);
		rctx->pengine = pengine;

		sreq.qce_cb = _qce_ahash_complete;
		sreq.digest = &rctx->digest[0];
		@@ -1481,6 +1560,7 @@ static int _qcrypto_process_aead(struct crypto_engine *pengine,
		struct crypto_aead *aead = crypto_aead_reqtfm(req);

		rctx = aead_request_ctx(req);
		rctx->pengine = pengine;
		cipher_ctx = crypto_tfm_ctx(async_req->tfm);

		qreq.op = QCE_REQ_AEAD;
		@@ -1695,28 +1775,93 @@ static int _start_qcrypto_process(struct crypto_priv *cp,
		struct crypto_engine *pengine)
		{
		struct crypto_async_request *async_req = NULL;
		struct crypto_async_request *backlog = NULL;
		struct crypto_async_request *backlog_eng = NULL;
		struct crypto_async_request *backlog_cp = NULL;
		unsigned long flags;
		u32 type;
		int ret = 0;
		struct crypto_stat *pstat;
		void *tfm_ctx;
		struct qcrypto_cipher_req_ctx *cipher_rctx;
		struct qcrypto_sha_req_ctx *ahash_rctx;
		struct ablkcipher_request *ablkcipher_req;
		struct ahash_request *ahash_req;
		struct aead_request *aead_req;
		struct qcrypto_resp_ctx *arsp;

		pstat = &_qcrypto_stat;

		again:
		spin_lock_irqsave(&cp->lock, flags);
		if (pengine->req == NULL) {
		backlog = crypto_get_backlog(&pengine->req_queue);
		async_req = crypto_dequeue_request(&pengine->req_queue);
		pengine->req = async_req;
		if (pengine->req) {
		spin_unlock_irqrestore(&cp->lock, flags);
		return 0;
		}

		backlog_eng = crypto_get_backlog(&pengine->req_queue);

		/* try to get request from request queue of the engine first */
		async_req = crypto_dequeue_request(&pengine->req_queue);
		if (!async_req) {
		/*
		* if no request from the engine,
		* try to get from request queue of driver
		*/
		backlog_cp = crypto_get_backlog(&cp->req_queue);
		async_req = crypto_dequeue_request(&cp->req_queue);
		if (!async_req) {
		spin_unlock_irqrestore(&cp->lock, flags);
		if (!async_req)
		return ret;
		if (backlog)
		backlog->complete(backlog, -EINPROGRESS);
		return 0;
		}
		}

		/* add associated rsp entry to tfm response queue */
		type = crypto_tfm_alg_type(async_req->tfm);
		tfm_ctx = crypto_tfm_ctx(async_req->tfm);
		switch (type) {
		case CRYPTO_ALG_TYPE_AHASH:
		ahash_req = container_of(async_req,
		struct ahash_request, base);
		ahash_rctx = ahash_request_ctx(ahash_req);
		arsp = &ahash_rctx->rsp_entry;
		list_add_tail(
		&arsp->list,
		&((struct qcrypto_sha_ctx *)tfm_ctx)
		->rsp_queue);
		break;
		case CRYPTO_ALG_TYPE_ABLKCIPHER:
		ablkcipher_req = container_of(async_req,
		struct ablkcipher_request, base);
		cipher_rctx = ablkcipher_request_ctx(ablkcipher_req);
		arsp = &cipher_rctx->rsp_entry;
		list_add_tail(
		&arsp->list,
		&((struct qcrypto_sha_ctx *)tfm_ctx)
		->rsp_queue);
		break;
		case CRYPTO_ALG_TYPE_AEAD:
		default:
		aead_req = container_of(async_req,
		struct aead_request, base);
		cipher_rctx = aead_request_ctx(aead_req);
		arsp = &cipher_rctx->rsp_entry;
		list_add_tail(
		&arsp->list,
		&((struct qcrypto_sha_ctx *)tfm_ctx)
		->rsp_queue);
		break;
		}

		arsp->res = -EINPROGRESS;
		arsp->async_req = async_req;
		pengine->req = async_req;
		pengine->arsp = arsp;

		spin_unlock_irqrestore(&cp->lock, flags);
		if (backlog_eng)
		backlog_eng->complete(backlog_eng, -EINPROGRESS);
		if (backlog_cp)
		backlog_cp->complete(backlog_cp, -EINPROGRESS);
		switch (type) {
		case CRYPTO_ALG_TYPE_ABLKCIPHER:
		ret = _qcrypto_process_ablkcipher(pengine, async_req);
		@@ -1732,9 +1877,11 @@ again:
		};
		pengine->total_req++;
		if (ret) {
		arsp->res = ret;
		pengine->err_req++;
		spin_lock_irqsave(&cp->lock, flags);
		pengine->req = NULL;
		pengine->arsp = NULL;
		spin_unlock_irqrestore(&cp->lock, flags);

		if (type == CRYPTO_ALG_TYPE_ABLKCIPHER)
		@@ -1745,11 +1892,22 @@ again:
		else
		pstat->aead_op_fail++;

		async_req->complete(async_req, ret);
		_qcrypto_tfm_complete(cp, type, tfm_ctx);
		goto again;
		};
		return ret;
		};
		}

		static struct crypto_engine _avail_eng(struct crypto_priv cp)
		{
		struct crypto_engine *pe = NULL;

		list_for_each_entry(pe, &cp->engine_list, elist) {
		if (pe->req == NULL)
		return pe;
		}
		return NULL;
		}

		static int _qcrypto_queue_req(struct crypto_priv *cp,
		struct crypto_engine *pengine,
		@@ -1765,8 +1923,14 @@ static int _qcrypto_queue_req(struct crypto_priv *cp,
		}

		spin_lock_irqsave(&cp->lock, flags);
		if (pengine) {
		ret = crypto_enqueue_request(&pengine->req_queue, req);
		} else {
		ret = crypto_enqueue_request(&cp->req_queue, req);
		pengine = _avail_eng(cp);
		}
		spin_unlock_irqrestore(&cp->lock, flags);
		if (pengine)
		_start_qcrypto_process(cp, pengine);

		return ret;
		@@ -3997,6 +4161,7 @@ static int _qcrypto_probe(struct platform_device *pdev)
		pengine->pcp = cp;
		pengine->pdev = pdev;
		pengine->req = NULL;
		pengine->signature = 0xdeadbeef;

		pengine->high_bw_req_count = 0;
		pengine->high_bw_req = false;
		@@ -4004,11 +4169,8 @@ static int _qcrypto_probe(struct platform_device *pdev)
		INIT_WORK(&pengine->low_bw_req_ws, qcrypto_low_bw_req_work);
		pengine->bw_scale_down_timer.function =
		qcrypto_bw_scale_down_timer_callback;

		device_init_wakeup(&pengine->pdev->dev, true);

		tasklet_init(&pengine->done_tasklet, req_done, (unsigned long)pengine);
		crypto_init_queue(&pengine->req_queue, 50);
		crypto_init_queue(&pengine->req_queue, MSM_QCRYPTO_REQ_QUEUE_LENGTH);

		mutex_lock(&cp->engine_lock);
		cp->total_units++;
		@@ -4529,6 +4691,7 @@ static int __init _qcrypto_init(void)
		pcp->ce_lock_count = 0;
		pcp->platform_support.bus_scale_table = NULL;
		pcp->next_engine = NULL;
		crypto_init_queue(&pcp->req_queue, MSM_QCRYPTO_REQ_QUEUE_LENGTH);
		return platform_driver_register(&_qualcomm_crypto);
		}