Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a6e3f6f0 authored by Joerg Roedel's avatar Joerg Roedel
Browse files

iommu/amd: Add flush counters to struct dma_ops_domain 



The counters are increased every time the TLB for a given
domain is flushed. We also store the current value of that
counter into newly added entries of the flush-queue, so that
we can tell whether this entry is already flushed.

Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent e241f8e7

drivers/iommu/amd_iommu.c

+52 −0
Original line number Diff line number Diff line
@@ -141,6 +141,7 @@ static void detach_device(struct device *dev); 
struct flush_queue_entry {

	unsigned long iova_pfn;

	unsigned long pages;

	u64 counter; /* Flush counter when this entry was added to the queue */

};



struct flush_queue {
@@ -160,6 +161,27 @@ struct dma_ops_domain { 
	struct iova_domain iovad;



	struct flush_queue __percpu *flush_queue;



	/*

	 * We need two counter here to be race-free wrt. IOTLB flushing and

	 * adding entries to the flush queue.

	 *

	 * The flush_start_cnt is incremented _before_ the IOTLB flush starts.

	 * New entries added to the flush ring-buffer get their 'counter' value

	 * from here. This way we can make sure that entries added to the queue

	 * (or other per-cpu queues of the same domain) while the TLB is about

	 * to be flushed are not considered to be flushed already.

	 */

	atomic64_t flush_start_cnt;



	/*

	 * The flush_finish_cnt is incremented when an IOTLB flush is complete.

	 * This value is always smaller than flush_start_cnt. The queue_add

	 * function frees all IOVAs that have a counter value smaller than

	 * flush_finish_cnt. This makes sure that we only free IOVAs that are

	 * flushed out of the IOTLB of the domain.

	 */

	atomic64_t flush_finish_cnt;

};



static struct iova_domain reserved_iova_ranges;
@@ -1777,6 +1799,9 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom) 
{

	int cpu;



	atomic64_set(&dom->flush_start_cnt,  0);

	atomic64_set(&dom->flush_finish_cnt, 0);



	dom->flush_queue = alloc_percpu(struct flush_queue);

	if (!dom->flush_queue)

		return -ENOMEM;
@@ -1844,22 +1869,48 @@ static inline unsigned queue_ring_add(struct flush_queue *queue) 
	return idx;

}



static inline void queue_ring_remove_head(struct flush_queue *queue)

{

	assert_spin_locked(&queue->lock);

	queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;

}



static void queue_add(struct dma_ops_domain *dom,

		      unsigned long address, unsigned long pages)

{

	struct flush_queue *queue;

	unsigned long flags;

	u64 counter;

	int idx;



	pages     = __roundup_pow_of_two(pages);

	address >>= PAGE_SHIFT;



	counter = atomic64_read(&dom->flush_finish_cnt);



	queue = get_cpu_ptr(dom->flush_queue);

	spin_lock_irqsave(&queue->lock, flags);



	queue_ring_for_each(idx, queue) {

		/*

		 * This assumes that counter values in the ring-buffer are

		 * monotonously rising.

		 */

		if (queue->entries[idx].counter >= counter)

			break;



		free_iova_fast(&dom->iovad,

			       queue->entries[idx].iova_pfn,

			       queue->entries[idx].pages);



		queue_ring_remove_head(queue);

	}



	if (queue_ring_full(queue)) {

		atomic64_inc(&dom->flush_start_cnt);

		domain_flush_tlb(&dom->domain);

		domain_flush_complete(&dom->domain);

		atomic64_inc(&dom->flush_finish_cnt);

		queue_release(dom, queue);

	}
@@ -1867,6 +1918,7 @@ static void queue_add(struct dma_ops_domain *dom, 


	queue->entries[idx].iova_pfn = address;

	queue->entries[idx].pages    = pages;

	queue->entries[idx].counter  = atomic64_read(&dom->flush_start_cnt);



	spin_unlock_irqrestore(&queue->lock, flags);

	put_cpu_ptr(dom->flush_queue);