Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bc385447 authored by Tom Lendacky's avatar Tom Lendacky Committed by Herbert Xu
Browse files

crypto: ccp - Use a single queue for proper ordering of tfm requests 



Move to a single queue to serialize requests within a tfm. When
testing using IPSec with a large number of network connections
the per cpu tfm queuing logic was not working properly.

Signed-off-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent c11baa02

drivers/crypto/ccp/ccp-crypto-main.c

+48 −116
Original line number Diff line number Diff line
@@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); 
static LIST_HEAD(hash_algs);

static LIST_HEAD(cipher_algs);



/* For any tfm, requests for that tfm on the same CPU must be returned

 * in the order received.  With multiple queues available, the CCP can

 * process more than one cmd at a time.  Therefore we must maintain

 * a cmd list to insure the proper ordering of requests on a given tfm/cpu

 * combination.

/* For any tfm, requests for that tfm must be returned on the order

 * received.  With multiple queues available, the CCP can process more

 * than one cmd at a time.  Therefore we must maintain a cmd list to insure

 * the proper ordering of requests on a given tfm.

 */

struct ccp_crypto_cpu_queue {

struct ccp_crypto_queue {

	struct list_head cmds;

	struct list_head *backlog;

	unsigned int cmd_count;

};

#define CCP_CRYPTO_MAX_QLEN	50

#define CCP_CRYPTO_MAX_QLEN	100



struct ccp_crypto_percpu_queue {

	struct ccp_crypto_cpu_queue __percpu *cpu_queue;

};

static struct ccp_crypto_percpu_queue req_queue;

static struct ccp_crypto_queue req_queue;

static spinlock_t req_queue_lock;



struct ccp_crypto_cmd {

	struct list_head entry;
@@ -71,8 +68,6 @@ struct ccp_crypto_cmd { 


	/* Used for held command processing to determine state */

	int ret;



	int cpu;

};



struct ccp_crypto_cpu {
@@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err) 
	return true;

}



/*

 * ccp_crypto_cmd_complete must be called while running on the appropriate

 * cpu and the caller must have done a get_cpu to disable preemption

 */

static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(

	struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)

{

	struct ccp_crypto_cpu_queue *cpu_queue;

	struct ccp_crypto_cmd *held = NULL, *tmp;

	unsigned long flags;



	*backlog = NULL;



	cpu_queue = this_cpu_ptr(req_queue.cpu_queue);

	spin_lock_irqsave(&req_queue_lock, flags);



	/* Held cmds will be after the current cmd in the queue so start

	 * searching for a cmd with a matching tfm for submission.

	 */

	tmp = crypto_cmd;

	list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) {

	list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {

		if (crypto_cmd->tfm != tmp->tfm)

			continue;

		held = tmp;
@@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( 
	 *   Because cmds can be executed from any point in the cmd list

	 *   special precautions have to be taken when handling the backlog.

	 */

	if (cpu_queue->backlog != &cpu_queue->cmds) {

	if (req_queue.backlog != &req_queue.cmds) {

		/* Skip over this cmd if it is the next backlog cmd */

		if (cpu_queue->backlog == &crypto_cmd->entry)

			cpu_queue->backlog = crypto_cmd->entry.next;

		if (req_queue.backlog == &crypto_cmd->entry)

			req_queue.backlog = crypto_cmd->entry.next;



		*backlog = container_of(cpu_queue->backlog,

		*backlog = container_of(req_queue.backlog,

					struct ccp_crypto_cmd, entry);

		cpu_queue->backlog = cpu_queue->backlog->next;

		req_queue.backlog = req_queue.backlog->next;



		/* Skip over this cmd if it is now the next backlog cmd */

		if (cpu_queue->backlog == &crypto_cmd->entry)

			cpu_queue->backlog = crypto_cmd->entry.next;

		if (req_queue.backlog == &crypto_cmd->entry)

			req_queue.backlog = crypto_cmd->entry.next;

	}



	/* Remove the cmd entry from the list of cmds */

	cpu_queue->cmd_count--;

	req_queue.cmd_count--;

	list_del(&crypto_cmd->entry);



	spin_unlock_irqrestore(&req_queue_lock, flags);



	return held;

}



static void ccp_crypto_complete_on_cpu(struct work_struct *work)

static void ccp_crypto_complete(void *data, int err)

{

	struct ccp_crypto_cpu *cpu_work =

		container_of(work, struct ccp_crypto_cpu, work);

	struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd;

	struct ccp_crypto_cmd *crypto_cmd = data;

	struct ccp_crypto_cmd *held, *next, *backlog;

	struct crypto_async_request *req = crypto_cmd->req;

	struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);

	int cpu, ret;



	cpu = get_cpu();

	int ret;



	if (cpu_work->err == -EINPROGRESS) {

	if (err == -EINPROGRESS) {

		/* Only propogate the -EINPROGRESS if necessary */

		if (crypto_cmd->ret == -EBUSY) {

			crypto_cmd->ret = -EINPROGRESS;

			req->complete(req, -EINPROGRESS);

		}



		goto e_cpu;

		return;

	}



	/* Operation has completed - update the queue before invoking
@@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) 
		req->complete(req, -EINPROGRESS);



	/* Completion callbacks */

	ret = cpu_work->err;

	ret = err;

	if (ctx->complete)

		ret = ctx->complete(req, ret);

	req->complete(req, ret);
@@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) 
	}



	kfree(crypto_cmd);



e_cpu:

	put_cpu();



	complete(&cpu_work->completion);

}



static void ccp_crypto_complete(void *data, int err)

{

	struct ccp_crypto_cmd *crypto_cmd = data;

	struct ccp_crypto_cpu cpu_work;



	INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu);

	init_completion(&cpu_work.completion);

	cpu_work.crypto_cmd = crypto_cmd;

	cpu_work.err = err;



	schedule_work_on(crypto_cmd->cpu, &cpu_work.work);



	/* Keep the completion call synchronous */

	wait_for_completion(&cpu_work.completion);

}



static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)

{

	struct ccp_crypto_cpu_queue *cpu_queue;

	struct ccp_crypto_cmd *active = NULL, *tmp;

	int cpu, ret;



	cpu = get_cpu();

	crypto_cmd->cpu = cpu;

	unsigned long flags;

	int ret;



	cpu_queue = this_cpu_ptr(req_queue.cpu_queue);

	spin_lock_irqsave(&req_queue_lock, flags);



	/* Check if the cmd can/should be queued */

	if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {

	if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {

		ret = -EBUSY;

		if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))

			goto e_cpu;

			goto e_lock;

	}



	/* Look for an entry with the same tfm.  If there is a cmd

	 * with the same tfm in the list for this cpu then the current

	 * cmd cannot be submitted to the CCP yet.

	 * with the same tfm in the list then the current cmd cannot

	 * be submitted to the CCP yet.

	 */

	list_for_each_entry(tmp, &cpu_queue->cmds, entry) {

	list_for_each_entry(tmp, &req_queue.cmds, entry) {

		if (crypto_cmd->tfm != tmp->tfm)

			continue;

		active = tmp;
@@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) 
	if (!active) {

		ret = ccp_enqueue_cmd(crypto_cmd->cmd);

		if (!ccp_crypto_success(ret))

			goto e_cpu;

			goto e_lock;

	}



	if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {

	if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {

		ret = -EBUSY;

		if (cpu_queue->backlog == &cpu_queue->cmds)

			cpu_queue->backlog = &crypto_cmd->entry;

		if (req_queue.backlog == &req_queue.cmds)

			req_queue.backlog = &crypto_cmd->entry;

	}

	crypto_cmd->ret = ret;



	cpu_queue->cmd_count++;

	list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds);

	req_queue.cmd_count++;

	list_add_tail(&crypto_cmd->entry, &req_queue.cmds);



e_cpu:

	put_cpu();

e_lock:

	spin_unlock_irqrestore(&req_queue_lock, flags);



	return ret;

}
@@ -387,50 +352,18 @@ static void ccp_unregister_algs(void) 
	}

}



static int ccp_init_queues(void)

{

	struct ccp_crypto_cpu_queue *cpu_queue;

	int cpu;



	req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue);

	if (!req_queue.cpu_queue)

		return -ENOMEM;



	for_each_possible_cpu(cpu) {

		cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);

		INIT_LIST_HEAD(&cpu_queue->cmds);

		cpu_queue->backlog = &cpu_queue->cmds;

		cpu_queue->cmd_count = 0;

	}



	return 0;

}



static void ccp_fini_queue(void)

{

	struct ccp_crypto_cpu_queue *cpu_queue;

	int cpu;



	for_each_possible_cpu(cpu) {

		cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);

		BUG_ON(!list_empty(&cpu_queue->cmds));

	}

	free_percpu(req_queue.cpu_queue);

}



static int ccp_crypto_init(void)

{

	int ret;



	ret = ccp_init_queues();

	if (ret)

		return ret;

	spin_lock_init(&req_queue_lock);

	INIT_LIST_HEAD(&req_queue.cmds);

	req_queue.backlog = &req_queue.cmds;

	req_queue.cmd_count = 0;



	ret = ccp_register_algs();

	if (ret) {

	if (ret)

		ccp_unregister_algs();

		ccp_fini_queue();

	}



	return ret;

}
@@ -438,7 +371,6 @@ static int ccp_crypto_init(void) 
static void ccp_crypto_exit(void)

{

	ccp_unregister_algs();

	ccp_fini_queue();

}



module_init(ccp_crypto_init);