Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 14c0ea65 authored by Jordan Crouse's avatar Jordan Crouse
Browse files

msm: kgsl: Replace the sorted dispatcher pending list 



Since the beginning of the dispatcher all pending draw contexts have been
attached to one big sorted priority list complete with locks. This is
overweight, requires a number of not so pretty workarounds for requeuing,
and (most importantly) plist is not exported for modules.

Replace the master plist with a set of 16 lists, one for each priority.
By using llist we can avoid the lock and easily solve the circular
requeuing problems we had before. The result should be cleaner, faster
and module friendly.

Change-Id: Ic0dedbad195a58d791a3583d244e39a14f2de941
Signed-off-by: default avatarJordan Crouse <jcrouse@codeaurora.org>
parent 2f23b00d

drivers/gpu/msm/adreno_dispatch.c

+73 −108
Original line number Diff line number Diff line
@@ -47,6 +47,9 @@ unsigned int adreno_drawobj_timeout = 2000; 
/* Interval for reading and comparing fault detection registers */

static unsigned int _fault_timer_interval = 200;



/* Use a kmem cache to speed up allocations for dispatcher jobs */

static struct kmem_cache *jobs_cache;



#define DRAWQUEUE_RB(_drawqueue) \

	((struct adreno_ringbuffer *) \

		container_of((_drawqueue),\
@@ -485,26 +488,32 @@ static inline int adreno_dispatcher_requeue_cmdobj( 
 *

 * Add a context to the dispatcher pending list.

 */

static void  dispatcher_queue_context(struct adreno_device *adreno_dev,

static int dispatcher_queue_context(struct adreno_device *adreno_dev,

		struct adreno_context *drawctxt)

{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct adreno_dispatch_job *job;



	/* Refuse to queue a detached context */

	if (kgsl_context_detached(&drawctxt->base))

		return;

		return 0;



	spin_lock(&dispatcher->plist_lock);

	if (!_kgsl_context_get(&drawctxt->base))

		return 0;



	if (plist_node_empty(&drawctxt->pending)) {

		/* Get a reference to the context while it sits on the list */

		if (_kgsl_context_get(&drawctxt->base)) {

			trace_dispatch_queue_context(drawctxt);

			plist_add(&drawctxt->pending, &dispatcher->pending);

		}

	/* This function can be called in an atomic context */

	job = kmem_cache_alloc(jobs_cache, GFP_ATOMIC);

	if (!job) {

		kgsl_context_put(&drawctxt->base);

		return -ENOMEM;

	}



	spin_unlock(&dispatcher->plist_lock);

	job->drawctxt = drawctxt;



	trace_dispatch_queue_context(drawctxt);

	llist_add(&job->node, &dispatcher->jobs[drawctxt->base.priority]);



	return 0;

}



/**
@@ -844,115 +853,65 @@ static int dispatcher_context_sendcmds(struct adreno_device *adreno_dev, 
	return ret;

}



/**

 * _adreno_dispatcher_issuecmds() - Issue commmands from pending contexts

 * @adreno_dev: Pointer to the adreno device struct

 *

 * Issue as many commands as possible (up to inflight) from the pending contexts

 * This function assumes the dispatcher mutex has been locked.

 */

static void _adreno_dispatcher_issuecmds(struct adreno_device *adreno_dev)

static bool adreno_drawctxt_bad(struct adreno_context *drawctxt)

{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct adreno_context *drawctxt, *next;

	struct plist_head requeue, busy_list;

	int ret;



	/* Leave early if the dispatcher isn't in a happy state */

	if (adreno_gpu_fault(adreno_dev) != 0)

		return;



	plist_head_init(&requeue);

	plist_head_init(&busy_list);



	/* Try to fill the ringbuffers as much as possible */

	while (1) {



		/* Stop doing things if the dispatcher is paused or faulted */

		if (adreno_gpu_fault(adreno_dev) != 0)

			break;



		if (adreno_gpu_halt(adreno_dev) != 0)

			break;



		spin_lock(&dispatcher->plist_lock);



		if (plist_head_empty(&dispatcher->pending)) {

			spin_unlock(&dispatcher->plist_lock);

			break;

	return (kgsl_context_detached(&drawctxt->base) ||

		kgsl_context_invalid(&drawctxt->base));

}



		/* Get the next entry on the list */

		drawctxt = plist_first_entry(&dispatcher->pending,

			struct adreno_context, pending);

static bool adreno_gpu_stopped(struct adreno_device *adreno_dev)

{

	return (adreno_gpu_fault(adreno_dev) || adreno_gpu_halt(adreno_dev));

}



		plist_del(&drawctxt->pending, &dispatcher->pending);

static void _adreno_dispatcher_handle_jobs(struct adreno_device *adreno_dev,

		struct llist_head *head)

{

	struct llist_node *list = llist_del_all(head);

	struct adreno_dispatch_job *job, *next;



		spin_unlock(&dispatcher->plist_lock);

	llist_for_each_entry_safe(job, next, list, node) {

		int ret;



		if (kgsl_context_detached(&drawctxt->base) ||

			kgsl_context_invalid(&drawctxt->base)) {

			kgsl_context_put(&drawctxt->base);

		if (adreno_gpu_stopped(adreno_dev) ||

			adreno_drawctxt_bad(job->drawctxt)) {

			kgsl_context_put(&job->drawctxt->base);

			kmem_cache_free(jobs_cache, job);

			continue;

		}



		ret = dispatcher_context_sendcmds(adreno_dev, drawctxt);



		/* Don't bother requeuing on -ENOENT - context is detached */

		if (ret != 0 && ret != -ENOENT) {

			spin_lock(&dispatcher->plist_lock);



			/*

			 * Check to seen if the context had been requeued while

			 * we were processing it (probably by another thread

			 * pushing commands). If it has then shift it to the

			 * requeue list if it was not able to submit commands

			 * due to the dispatch_q being full. Also, do a put to

			 * make sure the reference counting stays accurate.

			 * If the node is empty then we will put it on the

			 * requeue list and not touch the refcount since we

			 * already hold it from the first time it went on the

			 * list.

			 */

		ret = dispatcher_context_sendcmds(adreno_dev, job->drawctxt);



			if (!plist_node_empty(&drawctxt->pending)) {

				plist_del(&drawctxt->pending,

						&dispatcher->pending);

				kgsl_context_put(&drawctxt->base);

		if (!ret || ret == -ENOENT) {

			kgsl_context_put(&job->drawctxt->base);

			kmem_cache_free(jobs_cache, job);

			continue;

		}



			if (ret == -EBUSY)

				/* Inflight queue is full */

				plist_add(&drawctxt->pending, &busy_list);

			else

				plist_add(&drawctxt->pending, &requeue);



			spin_unlock(&dispatcher->plist_lock);

		} else {

			/*

			 * If the context doesn't need be requeued put back the

			 * refcount

			 */



			kgsl_context_put(&drawctxt->base);

		/* The job didn't finish, put it back on the job queue */

		llist_add(&job->node, head);

	}

}



	spin_lock(&dispatcher->plist_lock);



	/* Put the contexts that couldn't submit back on the pending list */

	plist_for_each_entry_safe(drawctxt, next, &busy_list, pending) {

		plist_del(&drawctxt->pending, &busy_list);

		plist_add(&drawctxt->pending, &dispatcher->pending);

	}

/**

 * _adreno_dispatcher_issuecmds() - Issue commmands from pending contexts

 * @adreno_dev: Pointer to the adreno device struct

 *

 * Issue as many commands as possible (up to inflight) from the pending contexts

 * This function assumes the dispatcher mutex has been locked.

 */

static void _adreno_dispatcher_issuecmds(struct adreno_device *adreno_dev)

{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	int i;



	/* Now put the contexts that need to be requeued back on the list */

	plist_for_each_entry_safe(drawctxt, next, &requeue, pending) {

		plist_del(&drawctxt->pending, &requeue);

		plist_add(&drawctxt->pending, &dispatcher->pending);

	}

	/* Leave early if the dispatcher isn't in a happy state */

	if (adreno_gpu_fault(adreno_dev) != 0)

		return;



	spin_unlock(&dispatcher->plist_lock);

	for (i = 0; i < ARRAY_SIZE(dispatcher->jobs); i++)

		_adreno_dispatcher_handle_jobs(adreno_dev,

			&dispatcher->jobs[i]);

}



static inline void _decrement_submit_now(struct kgsl_device *device)
@@ -1452,7 +1411,9 @@ int adreno_dispatcher_queue_cmds(struct kgsl_device_private *dev_priv, 
	spin_unlock(&drawctxt->lock);



	/* Add the context to the dispatcher pending list */

	dispatcher_queue_context(adreno_dev, drawctxt);

	ret = dispatcher_queue_context(adreno_dev, drawctxt);

	if (ret)

		return ret;



	/*

	 * Only issue commands if inflight is less than burst -this prevents us
@@ -2700,6 +2661,8 @@ void adreno_dispatcher_close(struct adreno_device *adreno_dev) 
	mutex_unlock(&dispatcher->mutex);



	kobject_put(&dispatcher->kobj);



	kmem_cache_destroy(jobs_cache);

}



struct dispatcher_attribute {
@@ -2837,7 +2800,7 @@ int adreno_dispatcher_init(struct adreno_device *adreno_dev) 
{

	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	int ret;

	int ret, i;



	memset(dispatcher, 0, sizeof(*dispatcher));
@@ -2852,8 +2815,10 @@ int adreno_dispatcher_init(struct adreno_device *adreno_dev) 
	init_completion(&dispatcher->idle_gate);

	complete_all(&dispatcher->idle_gate);



	plist_head_init(&dispatcher->pending);

	spin_lock_init(&dispatcher->plist_lock);

	jobs_cache = KMEM_CACHE(adreno_dispatch_job, 0);



	for (i = 0; i < ARRAY_SIZE(dispatcher->jobs); i++)

		init_llist_head(&dispatcher->jobs[i]);



	ret = kobject_init_and_add(&dispatcher->kobj, &ktype_dispatcher,

		&device->dev->kobj, "dispatch");

drivers/gpu/msm/adreno_dispatch.h

+18 −3
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 


#include <linux/kobject.h>

#include <linux/kthread.h>

#include <linux/llist.h>



extern unsigned int adreno_drawobj_timeout;
@@ -39,6 +40,21 @@ struct adreno_dispatcher_drawqueue { 
	unsigned long expires;

};



/**

 * struct adreno_dispatch_job - An instance of work for the dispatcher

 * @node: llist node for the list of jobs

 * @drawctxt: A pointer to an adreno draw context

 *

 * This struct defines work for the dispatcher. When a drawctxt is ready to send

 * commands it will attach itself to the appropriate list for it's priority.

 * The dispatcher will process all jobs on each priority every time it goes

 * through a dispatch cycle

 */

struct adreno_dispatch_job {

	struct llist_node node;

	struct adreno_context *drawctxt;

};



/**

 * struct adreno_dispatcher - container for the adreno GPU dispatcher

 * @mutex: Mutex to protect the structure
@@ -47,7 +63,6 @@ struct adreno_dispatcher_drawqueue { 
 * @inflight: Number of drawobj operations pending in the ringbuffer

 * @fault: Non-zero if a fault was detected.

 * @pending: Priority list of contexts waiting to submit drawobjs

 * @plist_lock: Spin lock to protect the pending queue

 * @work: work_struct to put the dispatcher in a work queue

 * @kobj: kobject for the dispatcher directory in the device sysfs node

 * @idle_gate: Gate to wait on for dispatcher to idle
@@ -59,8 +74,8 @@ struct adreno_dispatcher { 
	struct timer_list fault_timer;

	unsigned int inflight;

	atomic_t fault;

	struct plist_head pending;

	spinlock_t plist_lock;

	/** @jobs - Array of dispatch job lists for each priority level */

	struct llist_head jobs[16];

	struct kthread_work work;

	struct kobject kobj;

	struct completion idle_gate;

drivers/gpu/msm/adreno_drawctxt.c

+0 −6
Original line number Diff line number Diff line
@@ -384,12 +384,6 @@ adreno_drawctxt_create(struct kgsl_device_private *dev_priv, 
	/* set the context ringbuffer */

	drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt);



	/*

	 * Set up the plist node for the dispatcher.  Insert the node into the

	 * drawctxt pending list based on priority.

	 */

	plist_node_init(&drawctxt->pending, drawctxt->base.priority);



	/*

	 * Now initialize the common part of the context. This allocates the

	 * context id, and then possibly another thread could look it up.

drivers/gpu/msm/adreno_drawctxt.h

+0 −2
Original line number Diff line number Diff line
@@ -32,7 +32,6 @@ struct kgsl_device_private; 
 *			context

 * @drawqueue_head: Head of the drawqueue queue

 * @drawqueue_tail: Tail of the drawqueue queue

 * @pending: Priority list node for the dispatcher list of pending contexts

 * @wq: Workqueue structure for contexts to sleep pending room in the queue

 * @waiting: Workqueue structure for contexts waiting for a timestamp or event

 * @timeout: Workqueue structure for contexts waiting to invalidate
@@ -61,7 +60,6 @@ struct adreno_context { 
	unsigned int drawqueue_head;

	unsigned int drawqueue_tail;



	struct plist_node pending;

	wait_queue_head_t wq;

	wait_queue_head_t waiting;

	wait_queue_head_t timeout;