async_tx: fix multiple dependency submission (19242d72) · Commits · e / devices / android_kernel_samsung_smdk4412

crypto/async_tx/async_tx.c

+162 −35

Original line number	Original line	Diff line number	Diff line
	@@ -89,13 +89,19 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
	iter = tx;		iter = tx;

	/* find the root of the unsubmitted dependency chain */		/* find the root of the unsubmitted dependency chain */
	while (iter->cookie == -EBUSY) {		do {
	parent = iter->parent;		parent = iter->parent;
	if (parent && parent->cookie == -EBUSY)		if (!parent)
	iter = iter->parent;
	else
	break;		break;
	}		else
			iter = parent;
			} while (parent);

			/* there is a small window for ->parent == NULL and
			* ->cookie == -EBUSY
			*/
			while (iter->cookie == -EBUSY)
			cpu_relax();

	status = dma_sync_wait(iter->chan, iter->cookie);		status = dma_sync_wait(iter->chan, iter->cookie);
	} while (status == DMA_IN_PROGRESS \|\| (iter != tx));		} while (status == DMA_IN_PROGRESS \|\| (iter != tx));
	@@ -111,24 +117,33 @@ EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
	void		void
	async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)		async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)
	{		{
	struct dma_async_tx_descriptor dep_tx, _dep_tx;		struct dma_async_tx_descriptor *next = tx->next;
	struct dma_device *dev;
	struct dma_chan *chan;		struct dma_chan *chan;

	list_for_each_entry_safe(dep_tx, _dep_tx, &tx->depend_list,		if (!next)
	depend_node) {		return;
	chan = dep_tx->chan;
	dev = chan->device;		tx->next = NULL;
	/* we can't depend on ourselves */		chan = next->chan;
	BUG_ON(chan == tx->chan);
	list_del(&dep_tx->depend_node);		/* keep submitting up until a channel switch is detected
	tx->tx_submit(dep_tx);		* in that case we will be called again as a result of
			* processing the interrupt from async_tx_channel_switch
	/* we need to poke the engine as client code does not
	* know about dependency submission events
	*/		*/
	dev->device_issue_pending(chan);		while (next && next->chan == chan) {
			struct dma_async_tx_descriptor *_next;

			spin_lock_bh(&next->lock);
			next->parent = NULL;
			_next = next->next;
			next->next = NULL;
			spin_unlock_bh(&next->lock);

			next->tx_submit(next);
			next = _next;
	}		}

			chan->device->device_issue_pending(chan);
	}		}
	EXPORT_SYMBOL_GPL(async_tx_run_dependencies);		EXPORT_SYMBOL_GPL(async_tx_run_dependencies);

	@@ -397,6 +412,92 @@ static void __exit async_tx_exit(void)
	}		}
	#endif		#endif


			/**
			* async_tx_channel_switch - queue an interrupt descriptor with a dependency
			* pre-attached.
			* @depend_tx: the operation that must finish before the new operation runs
			* @tx: the new operation
			*/
			static void
			async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
			struct dma_async_tx_descriptor *tx)
			{
			struct dma_chan *chan;
			struct dma_device *device;
			struct dma_async_tx_descriptor intr_tx = (void ) ~0;

			/* first check to see if we can still append to depend_tx */
			spin_lock_bh(&depend_tx->lock);
			if (depend_tx->parent && depend_tx->chan == tx->chan) {
			tx->parent = depend_tx;
			depend_tx->next = tx;
			intr_tx = NULL;
			}
			spin_unlock_bh(&depend_tx->lock);

			if (!intr_tx)
			return;

			chan = depend_tx->chan;
			device = chan->device;

			/* see if we can schedule an interrupt
			* otherwise poll for completion
			*/
			if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
			intr_tx = device->device_prep_dma_interrupt(chan);
			else
			intr_tx = NULL;

			if (intr_tx) {
			intr_tx->callback = NULL;
			intr_tx->callback_param = NULL;
			tx->parent = intr_tx;
			/* safe to set ->next outside the lock since we know we are
			* not submitted yet
			*/
			intr_tx->next = tx;

			/* check if we need to append */
			spin_lock_bh(&depend_tx->lock);
			if (depend_tx->parent) {
			intr_tx->parent = depend_tx;
			depend_tx->next = intr_tx;
			async_tx_ack(intr_tx);
			intr_tx = NULL;
			}
			spin_unlock_bh(&depend_tx->lock);

			if (intr_tx) {
			intr_tx->parent = NULL;
			intr_tx->tx_submit(intr_tx);
			async_tx_ack(intr_tx);
			}
			} else {
			if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
			panic("%s: DMA_ERROR waiting for depend_tx\n",
			__func__);
			tx->tx_submit(tx);
			}
			}


			/**
			* submit_disposition - while holding depend_tx->lock we must avoid submitting
			* new operations to prevent a circular locking dependency with
			* drivers that already hold a channel lock when calling
			* async_tx_run_dependencies.
			* @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
			* @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
			* @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
			*/
			enum submit_disposition {
			ASYNC_TX_SUBMITTED,
			ASYNC_TX_CHANNEL_SWITCH,
			ASYNC_TX_DIRECT_SUBMIT,
			};

	void		void
	async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,		async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,
	enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,		enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
	@@ -405,28 +506,54 @@ async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,
	tx->callback = cb_fn;		tx->callback = cb_fn;
	tx->callback_param = cb_param;		tx->callback_param = cb_param;

	/* set this new tx to run after depend_tx if:		if (depend_tx) {
	* 1/ a dependency exists (depend_tx is !NULL)		enum submit_disposition s;
	* 2/ the tx can not be submitted to the current channel
	*/		/* sanity check the dependency chain:
	if (depend_tx && depend_tx->chan != chan) {		* 1/ if ack is already set then we cannot be sure
	/* if ack is already set then we cannot be sure
	* we are referring to the correct operation		* we are referring to the correct operation
			* 2/ dependencies are 1:1 i.e. two transactions can
			* not depend on the same parent
	*/		*/
	BUG_ON(depend_tx->ack);		BUG_ON(depend_tx->ack \|\| depend_tx->next \|\| tx->parent);

	tx->parent = depend_tx;		/* the lock prevents async_tx_run_dependencies from missing
			* the setting of ->next when ->parent != NULL
			*/
	spin_lock_bh(&depend_tx->lock);		spin_lock_bh(&depend_tx->lock);
	list_add_tail(&tx->depend_node, &depend_tx->depend_list);		if (depend_tx->parent) {
	if (depend_tx->cookie == 0) {		/* we have a parent so we can not submit directly
	struct dma_chan *dep_chan = depend_tx->chan;		* if we are staying on the same channel: append
	struct dma_device *dep_dev = dep_chan->device;		* else: channel switch
	dep_dev->device_dependency_added(dep_chan);		*/
			if (depend_tx->chan == chan) {
			tx->parent = depend_tx;
			depend_tx->next = tx;
			s = ASYNC_TX_SUBMITTED;
			} else
			s = ASYNC_TX_CHANNEL_SWITCH;
			} else {
			/* we do not have a parent so we may be able to submit
			* directly if we are staying on the same channel
			*/
			if (depend_tx->chan == chan)
			s = ASYNC_TX_DIRECT_SUBMIT;
			else
			s = ASYNC_TX_CHANNEL_SWITCH;
	}		}
	spin_unlock_bh(&depend_tx->lock);		spin_unlock_bh(&depend_tx->lock);

	/* schedule an interrupt to trigger the channel switch */		switch (s) {
	async_trigger_callback(ASYNC_TX_ACK, depend_tx, NULL, NULL);		case ASYNC_TX_SUBMITTED:
			break;
			case ASYNC_TX_CHANNEL_SWITCH:
			async_tx_channel_switch(depend_tx, tx);
			break;
			case ASYNC_TX_DIRECT_SUBMIT:
			tx->parent = NULL;
			tx->tx_submit(tx);
			break;
			}
	} else {		} else {
	tx->parent = NULL;		tx->parent = NULL;
	tx->tx_submit(tx);		tx->tx_submit(tx);

drivers/dma/dmaengine.c

+0 −2

Original line number	Original line	Diff line number	Diff line
	@@ -600,8 +600,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	{		{
	tx->chan = chan;		tx->chan = chan;
	spin_lock_init(&tx->lock);		spin_lock_init(&tx->lock);
	INIT_LIST_HEAD(&tx->depend_node);
	INIT_LIST_HEAD(&tx->depend_list);
	}		}
	EXPORT_SYMBOL(dma_async_tx_descriptor_init);		EXPORT_SYMBOL(dma_async_tx_descriptor_init);

drivers/dma/iop-adma.c

+5 −4

Original line number	Original line	Diff line number	Diff line
	@@ -63,7 +63,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
	struct iop_adma_chan *iop_chan, dma_cookie_t cookie)		struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
	{		{
	BUG_ON(desc->async_tx.cookie < 0);		BUG_ON(desc->async_tx.cookie < 0);
	spin_lock_bh(&desc->async_tx.lock);
	if (desc->async_tx.cookie > 0) {		if (desc->async_tx.cookie > 0) {
	cookie = desc->async_tx.cookie;		cookie = desc->async_tx.cookie;
	desc->async_tx.cookie = 0;		desc->async_tx.cookie = 0;
	@@ -101,7 +100,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,

	/* run dependent operations */		/* run dependent operations */
	async_tx_run_dependencies(&desc->async_tx);		async_tx_run_dependencies(&desc->async_tx);
	spin_unlock_bh(&desc->async_tx.lock);

	return cookie;		return cookie;
	}		}
	@@ -275,8 +273,11 @@ iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)

	static void iop_adma_tasklet(unsigned long data)		static void iop_adma_tasklet(unsigned long data)
	{		{
	struct iop_adma_chan chan = (struct iop_adma_chan ) data;		struct iop_adma_chan iop_chan = (struct iop_adma_chan ) data;
	__iop_adma_slot_cleanup(chan);
			spin_lock(&iop_chan->lock);
			__iop_adma_slot_cleanup(iop_chan);
			spin_unlock(&iop_chan->lock);
	}		}

	static struct iop_adma_desc_slot *		static struct iop_adma_desc_slot *

include/linux/dmaengine.h

+3 −6

Original line number	Original line	Diff line number	Diff line
	@@ -221,11 +221,9 @@ typedef void (dma_async_tx_callback)(void dma_async_param);
	* @callback: routine to call after this operation is complete		* @callback: routine to call after this operation is complete
	* @callback_param: general parameter to pass to the callback routine		* @callback_param: general parameter to pass to the callback routine
	* ---async_tx api specific fields---		* ---async_tx api specific fields---
	* @depend_list: at completion this list of transactions are submitted		* @next: at completion submit this descriptor
	* @depend_node: allow this transaction to be executed after another
	* transaction has completed, possibly on another channel
	* @parent: pointer to the next level up in the dependency chain		* @parent: pointer to the next level up in the dependency chain
	* @lock: protect the dependency list		* @lock: protect the parent and next pointers
	*/		*/
	struct dma_async_tx_descriptor {		struct dma_async_tx_descriptor {
	dma_cookie_t cookie;		dma_cookie_t cookie;
	@@ -236,8 +234,7 @@ struct dma_async_tx_descriptor {
	dma_cookie_t (tx_submit)(struct dma_async_tx_descriptor tx);		dma_cookie_t (tx_submit)(struct dma_async_tx_descriptor tx);
	dma_async_tx_callback callback;		dma_async_tx_callback callback;
	void *callback_param;		void *callback_param;
	struct list_head depend_list;		struct dma_async_tx_descriptor *next;
	struct list_head depend_node;
	struct dma_async_tx_descriptor *parent;		struct dma_async_tx_descriptor *parent;
	spinlock_t lock;		spinlock_t lock;
	};		};