Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fca6af6a authored by Joerg Roedel's avatar Joerg Roedel
Browse files

iommu/amd: Add per-domain timer to flush per-cpu queues



Add a timer to each dma_ops domain so that we flush unused
IOTLB entries regularily, even if the queues don't get full
all the time.

Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent a6e3f6f0
Loading
Loading
Loading
Loading
+67 −17
Original line number Diff line number Diff line
@@ -182,6 +182,13 @@ struct dma_ops_domain {
	 * flushed out of the IOTLB of the domain.
	 */
	atomic64_t flush_finish_cnt;

	/*
	 * Timer to make sure we don't keep IOVAs around unflushed
	 * for too long
	 */
	struct timer_list flush_timer;
	atomic_t flush_timer_on;
};

static struct iova_domain reserved_iova_ranges;
@@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
	return 0;
}

static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
{
	atomic64_inc(&dom->flush_start_cnt);
	domain_flush_tlb(&dom->domain);
	domain_flush_complete(&dom->domain);
	atomic64_inc(&dom->flush_finish_cnt);
}

static inline bool queue_ring_full(struct flush_queue *queue)
{
	assert_spin_locked(&queue->lock);
@@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue)
	queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
}

static void queue_add(struct dma_ops_domain *dom,
		      unsigned long address, unsigned long pages)
static void queue_ring_free_flushed(struct dma_ops_domain *dom,
				    struct flush_queue *queue)
{
	struct flush_queue *queue;
	unsigned long flags;
	u64 counter;
	u64 counter = atomic64_read(&dom->flush_finish_cnt);
	int idx;

	pages     = __roundup_pow_of_two(pages);
	address >>= PAGE_SHIFT;

	counter = atomic64_read(&dom->flush_finish_cnt);

	queue = get_cpu_ptr(dom->flush_queue);
	spin_lock_irqsave(&queue->lock, flags);

	queue_ring_for_each(idx, queue) {
		/*
		 * This assumes that counter values in the ring-buffer are
@@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom,

		queue_ring_remove_head(queue);
	}
}

static void queue_add(struct dma_ops_domain *dom,
		      unsigned long address, unsigned long pages)
{
	struct flush_queue *queue;
	unsigned long flags;
	int idx;

	pages     = __roundup_pow_of_two(pages);
	address >>= PAGE_SHIFT;

	queue = get_cpu_ptr(dom->flush_queue);
	spin_lock_irqsave(&queue->lock, flags);

	queue_ring_free_flushed(dom, queue);

	if (queue_ring_full(queue)) {
		atomic64_inc(&dom->flush_start_cnt);
		domain_flush_tlb(&dom->domain);
		domain_flush_complete(&dom->domain);
		atomic64_inc(&dom->flush_finish_cnt);
		dma_ops_domain_flush_tlb(dom);
		queue_release(dom, queue);
	}

@@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom,
	queue->entries[idx].counter  = atomic64_read(&dom->flush_start_cnt);

	spin_unlock_irqrestore(&queue->lock, flags);

	if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
		mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));

	put_cpu_ptr(dom->flush_queue);
}

static void queue_flush_timeout(unsigned long data)
{
	struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
	int cpu;

	atomic_set(&dom->flush_timer_on, 0);

	dma_ops_domain_flush_tlb(dom);

	for_each_possible_cpu(cpu) {
		struct flush_queue *queue;
		unsigned long flags;

		queue = per_cpu_ptr(dom->flush_queue, cpu);
		spin_lock_irqsave(&queue->lock, flags);
		queue_ring_free_flushed(dom, queue);
		spin_unlock_irqrestore(&queue->lock, flags);
	}
}

/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
@@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)

	del_domain_from_list(&dom->domain);

	if (timer_pending(&dom->flush_timer))
		del_timer(&dom->flush_timer);

	dma_ops_domain_free_flush_queue(dom);

	put_iova_domain(&dom->iovad);
@@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
	if (dma_ops_domain_alloc_flush_queue(dma_dom))
		goto free_dma_dom;

	setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
		    (unsigned long)dma_dom);

	atomic_set(&dma_dom->flush_timer_on, 0);

	add_domain_to_list(&dma_dom->domain);

	return dma_dom;