Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5b6885dc authored by Zhen Kong's avatar Zhen Kong
Browse files

crypto: msm: re-work qcrypto driver clk and bus management 



This patch re-works bus scaling and clk management to respond to
the actual crypto engine usage. The driver suspend/resume are
integrated with overall driver clk and bus management.

Furthermore, two functions are added to the low level crypto driver. Low
level crypto driver is informed to save and restored hardware context if
necessary across PM suspend/resume.

Change-Id: Ic906e0c7e96dee847253d6ef57341d1a38e294cf
Acked-by: default avatarChemin Hsieh <cheminh@qti.qualcomm.com>
Signed-off-by: default avatarRohit Vaswani <rvaswani@codeaurora.org>
Signed-off-by: default avatarZhen Kong <zkong@codeaurora.org>
parent 5698cb80

drivers/crypto/msm/qce.c

+12 −9
Original line number Diff line number Diff line 
/* Qualcomm Crypto Engine driver.

 *

 * Copyright (c) 2010-2013, The Linux Foundation. All rights reserved.

 * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -1932,6 +1932,9 @@ static int _ce_f8_setup(struct qce_device *pce_dev, struct qce_f8_req *req, 
	return 0;

};



struct qce_pm_table qce_pm_table = {NULL, NULL};

EXPORT_SYMBOL(qce_pm_table);



int qce_aead_req(void *handle, struct qce_req *q_req)

{

	struct qce_device *pce_dev = (struct qce_device *) handle;

drivers/crypto/msm/qce.h

+7 −0
Original line number Diff line number Diff line
@@ -166,6 +166,13 @@ struct qce_req { 
	unsigned int  flags;

};



struct qce_pm_table {

	int (*suspend)(void *handle);

	int (*resume)(void *handle);

};



extern struct qce_pm_table qce_pm_table;



void *qce_open(struct platform_device *pdev, int *rc);

int qce_close(void *handle);

int qce_aead_req(void *handle, struct qce_req *req);

drivers/crypto/msm/qce50.c

+66 −6
Original line number Diff line number Diff line
@@ -4387,6 +4387,65 @@ bad: 
	return rc;

}



static int _qce_suspend(void *handle)

{

	struct qce_device *pce_dev = (struct qce_device *)handle;

	struct sps_pipe *sps_pipe_info;



	if (handle == NULL)

		return -ENODEV;



	qce_enable_clk(pce_dev);



	sps_pipe_info = pce_dev->ce_sps.consumer.pipe;

	sps_disconnect(sps_pipe_info);



	sps_pipe_info = pce_dev->ce_sps.producer.pipe;

	sps_disconnect(sps_pipe_info);



	qce_disable_clk(pce_dev);

	return 0;

}



static int _qce_resume(void *handle)

{

	struct qce_device *pce_dev = (struct qce_device *)handle;

	struct sps_pipe *sps_pipe_info;

	struct sps_connect *sps_connect_info;

	int rc;



	if (handle == NULL)

		return -ENODEV;



	qce_enable_clk(pce_dev);



	sps_pipe_info = pce_dev->ce_sps.consumer.pipe;

	sps_connect_info = &pce_dev->ce_sps.consumer.connect;

	memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size);

	rc = sps_connect(sps_pipe_info, sps_connect_info);

	if (rc) {

		pr_err("sps_connect() fail pipe_handle=0x%x, rc = %d\n",

			(u32)sps_pipe_info, rc);

		return rc;

	}

	sps_pipe_info = pce_dev->ce_sps.producer.pipe;

	sps_connect_info = &pce_dev->ce_sps.producer.connect;

	memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size);

	rc = sps_connect(sps_pipe_info, sps_connect_info);

	if (rc)

		pr_err("sps_connect() fail pipe_handle=0x%x, rc = %d\n",

			(u32)sps_pipe_info, rc);



	pce_dev->ce_sps.out_transfer.user = pce_dev->ce_sps.producer.pipe;

	pce_dev->ce_sps.in_transfer.user = pce_dev->ce_sps.consumer.pipe;



	qce_disable_clk(pce_dev);

	return rc;

}



struct qce_pm_table qce_pm_table  = {_qce_suspend, _qce_resume};

EXPORT_SYMBOL(qce_pm_table);



int qce_aead_req(void *handle, struct qce_req *q_req)

{

	struct qce_device *pce_dev;
@@ -5442,5 +5501,6 @@ int qce_hw_support(void *handle, struct ce_hw_support *ce_support) 
}

EXPORT_SYMBOL(qce_hw_support);





MODULE_LICENSE("GPL v2");

MODULE_DESCRIPTION("Crypto Engine driver");

drivers/crypto/msm/qcrypto.c

+210 −116
Original line number Diff line number Diff line
@@ -58,6 +58,14 @@ 


#define QCRYPTO_HIGH_BANDWIDTH_TIMEOUT 1000



enum qcrypto_bus_state {

	BUS_NO_BANDWIDTH = 0,

	BUS_HAS_BANDWIDTH,

	BUS_BANDWIDTH_RELEASING,

	BUS_BANDWIDTH_ALLOCATING,

	BUS_SUSPENDED,

};



struct crypto_stat {

	u64 aead_sha1_aes_enc;

	u64 aead_sha1_aes_dec;
@@ -114,10 +122,19 @@ struct crypto_engine { 
	u32 unit;

	u32 ce_device;

	unsigned int signature;

	uint32_t high_bw_req_count;



	enum qcrypto_bus_state bw_state;

	bool   high_bw_req;

	struct timer_list bw_scale_down_timer;

	struct work_struct low_bw_req_ws;

	struct timer_list bw_reaper_timer;

	struct work_struct bw_reaper_ws;

	struct work_struct bw_allocate_ws;



	/* engine execution sequence number */

	u32    active_seq;

	/* last QCRYPTO_HIGH_BANDWIDTH_TIMEOUT active_seq */

	u32    last_active_seq;



	bool   check_flag;

};



struct crypto_priv {
@@ -127,7 +144,7 @@ struct crypto_priv { 
	/* CE features/algorithms supported by HW engine*/

	struct ce_hw_support ce_support;



	/* the lock protects queue and req*/

	/* the lock protects crypto queue and req */

	spinlock_t lock;



	/* list of  registered algorithms */
@@ -141,13 +158,13 @@ struct crypto_priv { 
	struct list_head engine_list; /* list of  qcrypto engines */

	int32_t total_units;   /* total units of engines */

	struct mutex engine_lock;



	struct crypto_engine *next_engine; /* next assign engine */

	struct crypto_queue req_queue;	/*

					 * request queue for those requests

					 * that waiting for an available

					 * engine.

					 */



};

static struct crypto_priv qcrypto_dev;

static struct crypto_engine *_qcrypto_static_assign_engine(
@@ -433,11 +450,12 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine, 
{

	int ret = 0;



	if (high_bw_req && pengine->high_bw_req == false) {

	if (high_bw_req) {

		pm_stay_awake(&pengine->pdev->dev);

		ret = qce_enable_clk(pengine->qce);

		if (ret) {

			pr_err("%s Unable enable clk\n", __func__);

			return;

			goto clk_err;

		}

		ret = msm_bus_scale_client_update_request(

				pengine->bus_scale_handle, 1);
@@ -445,16 +463,18 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine, 
			pr_err("%s Unable to set to high bandwidth\n",

						__func__);

			qce_disable_clk(pengine->qce);

			return;

			goto clk_err;

		}

		pengine->high_bw_req = true;

	} else if (high_bw_req == false && pengine->high_bw_req == true) {





	} else {



		ret = msm_bus_scale_client_update_request(

				pengine->bus_scale_handle, 0);

		if (ret) {

			pr_err("%s Unable to set to low bandwidth\n",

						__func__);

			return;

			goto clk_err;

		}

		ret = qce_disable_clk(pengine->qce);

		if (ret) {
@@ -464,66 +484,118 @@ static void qcrypto_ce_set_bus(struct crypto_engine *pengine, 
			if (ret)

				pr_err("%s Unable to set to high bandwidth\n",

						__func__);

			return;

			goto clk_err;

		}

		pengine->high_bw_req = false;

		pm_relax(&pengine->pdev->dev);

	}

	return;

clk_err:

	pm_relax(&pengine->pdev->dev);

	return;



}



static void qcrypto_bw_scale_down_timer_callback(unsigned long data)

static void qcrypto_bw_reaper_timer_callback(unsigned long data)

{

	struct crypto_engine *pengine = (struct crypto_engine *)data;



	schedule_work(&pengine->low_bw_req_ws);

	schedule_work(&pengine->bw_reaper_ws);



	return;

}



static void qcrypto_bw_set_timeout(struct crypto_engine *pengine)

{

	del_timer_sync(&(pengine->bw_scale_down_timer));

	pengine->bw_scale_down_timer.data =

	pengine->bw_reaper_timer.data =

			(unsigned long)(pengine);

	pengine->bw_scale_down_timer.expires = jiffies +

	pengine->bw_reaper_timer.expires = jiffies +

			msecs_to_jiffies(QCRYPTO_HIGH_BANDWIDTH_TIMEOUT);

	add_timer(&(pengine->bw_scale_down_timer));

	add_timer(&(pengine->bw_reaper_timer));

}



static void qcrypto_ce_bw_scaling_req(struct crypto_priv *cp,

				 bool high_bw_req)

static void qcrypto_ce_bw_allocate_req(struct crypto_engine *pengine)

{

	struct crypto_engine *pengine;

	schedule_work(&pengine->bw_allocate_ws);

}



static int _start_qcrypto_process(struct crypto_priv *cp,

					struct crypto_engine *pengine);



static void qcrypto_bw_allocate_work(struct work_struct *work)

{

	struct  crypto_engine *pengine = container_of(work,

				struct crypto_engine, bw_allocate_ws);

	unsigned long flags;

	struct crypto_priv *cp = pengine->pcp;



	spin_lock_irqsave(&cp->lock, flags);

	pengine->bw_state = BUS_BANDWIDTH_ALLOCATING;

	spin_unlock_irqrestore(&cp->lock, flags);



	if (cp->platform_support.bus_scale_table == NULL)

		return;

	mutex_lock(&cp->engine_lock);

	list_for_each_entry(pengine, &cp->engine_list, elist) {

		if (high_bw_req) {

			if (pengine->high_bw_req_count == 0)

	qcrypto_ce_set_bus(pengine, true);

			pengine->high_bw_req_count++;

		} else {

			pengine->high_bw_req_count--;

			if (pengine->high_bw_req_count == 0)

				qcrypto_bw_set_timeout(pengine);

		}

	}

	mutex_unlock(&cp->engine_lock);

}



static void qcrypto_low_bw_req_work(struct work_struct *work)

	spin_lock_irqsave(&cp->lock, flags);

	pengine->bw_state = BUS_HAS_BANDWIDTH;

	pengine->high_bw_req = false;

	pengine->active_seq++;

	pengine->check_flag = true;

	spin_unlock_irqrestore(&cp->lock, flags);

	_start_qcrypto_process(cp, pengine);

};



static void qcrypto_bw_reaper_work(struct work_struct *work)

{

	struct  crypto_engine *pengine = container_of(work,

				struct crypto_engine, low_bw_req_ws);

				struct crypto_engine, bw_reaper_ws);

	struct crypto_priv *cp = pengine->pcp;

	unsigned long flags;

	u32    active_seq;

	bool restart = false;



	mutex_lock(&pengine->pcp->engine_lock);

	if (pengine->high_bw_req_count == 0)

		qcrypto_ce_set_bus(pengine, false);

	mutex_unlock(&pengine->pcp->engine_lock);

	spin_lock_irqsave(&cp->lock, flags);

	active_seq = pengine->active_seq;

	if (pengine->bw_state == BUS_HAS_BANDWIDTH &&

		(active_seq == pengine->last_active_seq)) {



		/* check if engine is stuck */

		if (pengine->req) {

			if (pengine->check_flag)

				dev_err(&pengine->pdev->dev,

				"The engine appears to be stuck seq %d req %p.\n",

				active_seq, pengine->req);

			pengine->check_flag = false;

			goto ret;

		}

		if (cp->platform_support.bus_scale_table == NULL)

			goto ret;

		pengine->bw_state = BUS_BANDWIDTH_RELEASING;

		spin_unlock_irqrestore(&cp->lock, flags);



static int _start_qcrypto_process(struct crypto_priv *cp,

					struct crypto_engine *pengine);

		qcrypto_ce_set_bus(pengine, false);



		spin_lock_irqsave(&cp->lock, flags);



		if (pengine->high_bw_req == true) {

			/* we got request while we are disabling clock */

			pengine->bw_state = BUS_BANDWIDTH_ALLOCATING;

			spin_unlock_irqrestore(&cp->lock, flags);



			qcrypto_ce_set_bus(pengine, true);



			spin_lock_irqsave(&cp->lock, flags);

			pengine->bw_state = BUS_HAS_BANDWIDTH;

			pengine->high_bw_req = false;

			restart = true;

		} else

			pengine->bw_state = BUS_NO_BANDWIDTH;

	}

ret:

	pengine->last_active_seq = active_seq;

	spin_unlock_irqrestore(&cp->lock, flags);

	if (restart)

		_start_qcrypto_process(cp, pengine);

	qcrypto_bw_set_timeout(pengine);

}



static int qcrypto_count_sg(struct scatterlist *sg, int nbytes)

{
@@ -625,7 +697,6 @@ static int _qcrypto_cipher_cra_init(struct crypto_tfm *tfm) 
			return -ENODEV;

	} else

		ctx->pengine = NULL;

	qcrypto_ce_bw_scaling_req(ctx->cp, true);

	INIT_LIST_HEAD(&ctx->rsp_queue);

	return 0;

};
@@ -650,7 +721,6 @@ static int _qcrypto_ahash_cra_init(struct crypto_tfm *tfm) 
			return -ENODEV;

	} else

		sha_ctx->pengine = NULL;

	qcrypto_ce_bw_scaling_req(sha_ctx->cp, true);

	INIT_LIST_HEAD(&sha_ctx->rsp_queue);

	return 0;

};
@@ -665,7 +735,6 @@ static void _qcrypto_ahash_cra_exit(struct crypto_tfm *tfm) 
		ahash_request_free(sha_ctx->ahash_req);

		sha_ctx->ahash_req = NULL;

	}

	qcrypto_ce_bw_scaling_req(sha_ctx->cp, false);

};
@@ -716,7 +785,6 @@ static void _qcrypto_cra_ablkcipher_exit(struct crypto_tfm *tfm) 


	if (!list_empty(&ctx->rsp_queue))

		pr_err("_qcrypto__cra_ablkcipher_exit: requests still outstanding");

	qcrypto_ce_bw_scaling_req(ctx->cp, false);

};



static void _qcrypto_cra_aead_exit(struct crypto_tfm *tfm)
@@ -725,7 +793,6 @@ static void _qcrypto_cra_aead_exit(struct crypto_tfm *tfm) 


	if (!list_empty(&ctx->rsp_queue))

		pr_err("_qcrypto__cra_aead_exit: requests still outstanding");

	qcrypto_ce_bw_scaling_req(ctx->cp, false);

};



static int _disp_stats(int id)
@@ -875,8 +942,9 @@ static void _qcrypto_remove_engine(struct crypto_engine *pengine) 
	cp->total_units--;



	tasklet_kill(&pengine->done_tasklet);

	cancel_work_sync(&pengine->low_bw_req_ws);

	del_timer_sync(&pengine->bw_scale_down_timer);

	cancel_work_sync(&pengine->bw_reaper_ws);

	cancel_work_sync(&pengine->bw_allocate_ws);

	del_timer_sync(&pengine->bw_reaper_timer);



	if (pengine->bus_scale_handle != 0)

		msm_bus_scale_unregister_client(pengine->bus_scale_handle);
@@ -1856,6 +1924,8 @@ again: 
	arsp->async_req = async_req;

	pengine->req = async_req;

	pengine->arsp = arsp;

	pengine->active_seq++;

	pengine->check_flag = true;



	spin_unlock_irqrestore(&cp->lock, flags);

	if (backlog_eng)
@@ -1923,16 +1993,40 @@ static int _qcrypto_queue_req(struct crypto_priv *cp, 
	}



	spin_lock_irqsave(&cp->lock, flags);



	if (pengine) {

		ret = crypto_enqueue_request(&pengine->req_queue, req);

	} else {

		ret = crypto_enqueue_request(&cp->req_queue, req);

		pengine = _avail_eng(cp);

	}

	if (pengine) {

		switch (pengine->bw_state) {

		case BUS_NO_BANDWIDTH:

			if (pengine->high_bw_req == false) {

				qcrypto_ce_bw_allocate_req(pengine);

				pengine->high_bw_req = true;

			}

			pengine = NULL;

			break;

		case BUS_HAS_BANDWIDTH:

			break;

		case BUS_BANDWIDTH_RELEASING:

			pengine->high_bw_req = true;

			pengine = NULL;

			break;

		case BUS_BANDWIDTH_ALLOCATING:

			pengine = NULL;

			break;

		case BUS_SUSPENDED:

		default:

			pengine = NULL;

			break;

		}

	}

	spin_unlock_irqrestore(&cp->lock, flags);

	if (pengine)

		_start_qcrypto_process(cp, pengine);



	return ret;

}
@@ -4163,12 +4257,17 @@ static int _qcrypto_probe(struct platform_device *pdev) 
	pengine->req = NULL;

	pengine->signature = 0xdeadbeef;



	pengine->high_bw_req_count = 0;

	init_timer(&(pengine->bw_reaper_timer));

	INIT_WORK(&pengine->bw_reaper_ws, qcrypto_bw_reaper_work);

	pengine->bw_reaper_timer.function =

			qcrypto_bw_reaper_timer_callback;

	INIT_WORK(&pengine->bw_allocate_ws, qcrypto_bw_allocate_work);

	pengine->high_bw_req = false;

	init_timer(&(pengine->bw_scale_down_timer));

	INIT_WORK(&pengine->low_bw_req_ws, qcrypto_low_bw_req_work);

	pengine->bw_scale_down_timer.function =

			qcrypto_bw_scale_down_timer_callback;

	pengine->active_seq = 0;

	pengine->last_active_seq = 0;

	pengine->check_flag = false;

	qcrypto_bw_set_timeout(pengine);



	tasklet_init(&pengine->done_tasklet, req_done, (unsigned long)pengine);

	crypto_init_queue(&pengine->req_queue, MSM_QCRYPTO_REQ_QUEUE_LENGTH);
@@ -4208,7 +4307,9 @@ static int _qcrypto_probe(struct platform_device *pdev) 
				platform_support->bus_scale_table;

		cp->platform_support.sha_hmac = platform_support->sha_hmac;

	}



	pengine->bus_scale_handle = 0;



	if (cp->platform_support.bus_scale_table != NULL) {

		pengine->bus_scale_handle =

			msm_bus_scale_register_client(
@@ -4220,6 +4321,9 @@ static int _qcrypto_probe(struct platform_device *pdev) 
			rc =  -ENOMEM;

			goto err;

		}

		pengine->bw_state = BUS_NO_BANDWIDTH;

	} else {

		pengine->bw_state = BUS_HAS_BANDWIDTH;

	}



	if (cp->total_units != 1) {
@@ -4486,12 +4590,12 @@ err: 
	return rc;

};





static int  _qcrypto_suspend(struct platform_device *pdev, pm_message_t state)

{

	int ret = 0;

	struct crypto_engine *pengine;

	struct crypto_priv *cp;

	unsigned long flags;



	pengine = platform_get_drvdata(pdev);

	if (!pengine)
@@ -4504,42 +4608,39 @@ static int _qcrypto_suspend(struct platform_device *pdev, pm_message_t state) 
	cp = pengine->pcp;

	if (!cp->ce_support.clk_mgmt_sus_res)

		return 0;



	mutex_lock(&cp->engine_lock);



	if (pengine->high_bw_req) {

		del_timer_sync(&(pengine->bw_scale_down_timer));

		ret = msm_bus_scale_client_update_request(

				pengine->bus_scale_handle, 0);

		if (ret) {

			dev_err(&pdev->dev, "%s Unable to set to low bandwidth\n",

					__func__);

			mutex_unlock(&cp->engine_lock);

			return ret;

	spin_lock_irqsave(&cp->lock, flags);

	switch (pengine->bw_state) {

	case BUS_NO_BANDWIDTH:

		if (pengine->high_bw_req == false)

			pengine->bw_state = BUS_SUSPENDED;

		else

			ret = -EBUSY;

		break;

	case BUS_HAS_BANDWIDTH:

	case BUS_BANDWIDTH_RELEASING:

	case BUS_BANDWIDTH_ALLOCATING:

	case BUS_SUSPENDED:

	default:

			ret = -EBUSY;

			break;

	}

		ret = qce_disable_clk(pengine->qce);

		if (ret) {

			pr_err("%s Unable disable clk\n", __func__);

			ret = msm_bus_scale_client_update_request(

				pengine->bus_scale_handle, 1);



	spin_unlock_irqrestore(&cp->lock, flags);

	if (ret)

				dev_err(&pdev->dev,

					"%s Unable to set to high bandwidth\n",

					__func__);

			mutex_unlock(&cp->engine_lock);

		return ret;

		}

	}



	mutex_unlock(&cp->engine_lock);

	else {

		if (qce_pm_table.suspend)

			qce_pm_table.suspend(pengine->qce);

		return 0;

	}

}



static int  _qcrypto_resume(struct platform_device *pdev)

{

	int ret = 0;

	struct crypto_engine *pengine;

	struct crypto_priv *cp;

	unsigned long flags;

	bool restart = false;



	pengine = platform_get_drvdata(pdev);
@@ -4550,33 +4651,26 @@ static int _qcrypto_resume(struct platform_device *pdev) 
	if (!cp->ce_support.clk_mgmt_sus_res)

		return 0;



	mutex_lock(&cp->engine_lock);

	if (pengine->high_bw_req) {

		ret = qce_enable_clk(pengine->qce);

		if (ret) {

			dev_err(&pdev->dev, "%s Unable to enable clk\n",

				__func__);

			mutex_unlock(&cp->engine_lock);

			return ret;

		}

		ret = msm_bus_scale_client_update_request(

				pengine->bus_scale_handle, 1);

		if (ret) {

			dev_err(&pdev->dev,

				"%s Unable to set to high bandwidth\n",

				__func__);

			qce_disable_clk(pengine->qce);

			mutex_unlock(&cp->engine_lock);

			return ret;

		}

		pengine->bw_scale_down_timer.data =

					(unsigned long)(pengine);

		pengine->bw_scale_down_timer.expires = jiffies +

			msecs_to_jiffies(QCRYPTO_HIGH_BANDWIDTH_TIMEOUT);

		add_timer(&(pengine->bw_scale_down_timer));

	}

	spin_lock_irqsave(&cp->lock, flags);

	if (pengine->bw_state == BUS_SUSPENDED) {

		pengine->bw_state = BUS_BANDWIDTH_ALLOCATING;

		spin_unlock_irqrestore(&cp->lock, flags);



	mutex_unlock(&cp->engine_lock);

		if (qce_pm_table.resume)

			qce_pm_table.resume(pengine->qce);



		qcrypto_ce_set_bus(pengine, true);



		spin_lock_irqsave(&cp->lock, flags);

		pengine->bw_state = BUS_HAS_BANDWIDTH;

		pengine->high_bw_req = false;

		restart = true;

		pengine->active_seq++;

		pengine->check_flag = true;

	}

	spin_unlock_irqrestore(&cp->lock, flags);

	if (restart)

		_start_qcrypto_process(cp, pengine);



	return 0;

}