Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e3b3ce6e authored by Mitchel Humpherys's avatar Mitchel Humpherys
Browse files

iommu/io-pgtable-fast: Prove correctness of TLB maintenance 



A common software error when it comes to page table code is missing TLB
maintenance.  Add some checks to the io-pgtable-fast code to detect when
an address that might be stale in the TLB is being re-used.  This can be
accomplished by writing a "stale TLB" flag value to the reserved bits of
the PTE during unmap and then removing the flag value when the TLBs are
invalidated (by sweeping the entire page table).  That way, whenever we
map we can know that there might be a stale TLB in the location being
mapped into if it contains the "stale TLB" flag value.

CRs-Fixed: 997751
Change-Id: Icf9c1e41977cb71e8b137190adb3b4a201c339da
Signed-off-by: default avatarMitchel Humpherys <mitchelh@codeaurora.org>
parent 41e1c625

drivers/iommu/Kconfig

+15 −0
Original line number Diff line number Diff line
@@ -58,6 +58,21 @@ config IOMMU_IO_PGTABLE_FAST_SELFTEST 


	  If unsure, say N here.



config IOMMU_IO_PGTABLE_FAST_PROVE_TLB

	bool "Prove correctness of TLB maintenance in the Fast DMA mapper"

	depends on IOMMU_IO_PGTABLE_FAST

	help

          Enables some debug features that help prove correctness of TLB

          maintenance routines in the Fast DMA mapper.  This option will

          slow things down considerably, so should only be used in a debug

          configuration.  This relies on the ability to set bits in an

          invalid page table entry, which is disallowed on some hardware

          due to errata.  If you're running on such a platform then this

          option can only be used with unit tests.  It will break real use

          cases.



	  If unsure, say N here.



endmenu



config OF_IOMMU

drivers/iommu/dma-mapping-fast.c

+42 −2
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ static bool __bit_covered_stale(unsigned long upcoming_stale, 
}



static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping,

					 struct dma_attrs *attrs,

					 size_t size)

{

	unsigned long bit, prev_search_start, nbits = size >> FAST_PAGE_SHIFT;
@@ -113,8 +114,11 @@ static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping, 
	    __bit_covered_stale(mapping->upcoming_stale_bit,

				prev_search_start,

				bit + nbits - 1)) {

		bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs);



		iommu_tlbiall(mapping->domain);

		mapping->have_stale_tlbs = false;

		av8l_fast_clear_stale_ptes(mapping->pgtbl_pmds, skip_sync);

	}



	return (bit << FAST_PAGE_SHIFT) + mapping->base;
@@ -286,7 +290,7 @@ static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page, 


	spin_lock_irqsave(&mapping->lock, flags);



	iova = __fast_smmu_alloc_iova(mapping, len);

	iova = __fast_smmu_alloc_iova(mapping, attrs, len);



	if (unlikely(iova == DMA_ERROR_CODE))

		goto fail;
@@ -426,7 +430,7 @@ static void *fast_smmu_alloc(struct device *dev, size_t size, 
	}



	spin_lock_irqsave(&mapping->lock, flags);

	dma_addr = __fast_smmu_alloc_iova(mapping, size);

	dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size);

	if (dma_addr == DMA_ERROR_CODE) {

		dev_err(dev, "no iova\n");

		spin_unlock_irqrestore(&mapping->lock, flags);
@@ -518,6 +522,39 @@ static int fast_smmu_mapping_error(struct device *dev, 
	return dma_addr == DMA_ERROR_CODE;

}



static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast,

					  void *data)

{

	av8l_fast_iopte *ptep = data;

	dma_addr_t iova;

	unsigned long bitmap_idx;



	bitmap_idx = (unsigned long)(ptep - fast->pgtbl_pmds);

	iova = bitmap_idx << FAST_PAGE_SHIFT;

	dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);

	dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);

	dev_err(fast->dev, "ptep: %p pmds: %p diff: %lu\n", ptep,

		fast->pgtbl_pmds, ptep - fast->pgtbl_pmds);

	print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,

		       32, 8, fast->bitmap, fast->bitmap_size, false);

}



static int fast_smmu_notify(struct notifier_block *self,

			    unsigned long action, void *data)

{

	struct dma_fast_smmu_mapping *fast = container_of(

		self, struct dma_fast_smmu_mapping, notifier);



	switch (action) {

	case MAPPED_OVER_STALE_TLB:

		__fast_smmu_mapped_over_stale(fast, data);

		return NOTIFY_OK;

	default:

		WARN(1, "Unhandled notifier action");

		return NOTIFY_DONE;

	}

}



static const struct dma_map_ops fast_smmu_dma_ops = {

	.alloc = fast_smmu_alloc,

	.free = fast_smmu_free,
@@ -619,6 +656,9 @@ int fast_smmu_attach_device(struct device *dev, 
	}

	mapping->fast->pgtbl_pmds = info.pmds;



	mapping->fast->notifier.notifier_call = fast_smmu_notify;

	av8l_register_notify(&mapping->fast->notifier);



	dev->archdata.mapping = mapping;

	set_dma_ops(dev, &fast_smmu_dma_ops);

drivers/iommu/io-pgtable-fast.c

+73 −5
Original line number Diff line number Diff line
@@ -139,6 +139,54 @@ struct av8l_fast_io_pgtable { 
#define AV8L_FAST_PAGE_SHIFT		12





#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB



#include <asm/cacheflush.h>

#include <linux/notifier.h>



static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list);



void av8l_register_notify(struct notifier_block *nb)

{

	atomic_notifier_chain_register(&av8l_notifier_list, nb);

}

EXPORT_SYMBOL(av8l_register_notify);



static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)

{

	if (unlikely(*ptep)) {

		atomic_notifier_call_chain(

			&av8l_notifier_list, MAPPED_OVER_STALE_TLB,

			(void *) ptep);

		pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n",

		       *ptep, ptep);

		pr_err("Nearby memory:\n");

		print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS,

			       32, 8, ptep - 16, 32 * sizeof(*ptep), false);

		BUG();

	}

}



void av8l_fast_clear_stale_ptes(av8l_fast_iopte *pmds, bool skip_sync)

{

	int i;

	av8l_fast_iopte *pmdp = pmds;



	for (i = 0; i < ((SZ_1G * 4UL) >> AV8L_FAST_PAGE_SHIFT); ++i) {

		if (!(*pmdp & AV8L_FAST_PTE_VALID)) {

			*pmdp = 0;

			if (!skip_sync)

				dmac_clean_range(pmdp, pmdp + 1);

		}

		pmdp++;

	}

}

#else

static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)

{

}

#endif



/* caller must take care of cache maintenance on *ptep */

int av8l_fast_map_public(av8l_fast_iopte *ptep, phys_addr_t paddr, size_t size,

			 int prot)
@@ -163,8 +211,10 @@ int av8l_fast_map_public(av8l_fast_iopte *ptep, phys_addr_t paddr, size_t size, 
		pte |= AV8L_FAST_PTE_AP_RW;



	paddr &= AV8L_FAST_PTE_ADDR_MASK;

	for (i = 0; i < nptes; i++, paddr += SZ_4K)

	for (i = 0; i < nptes; i++, paddr += SZ_4K) {

		__av8l_check_for_stale_tlb(ptep + i);

		*(ptep + i) = pte | paddr;

	}



	return 0;

}
@@ -184,12 +234,21 @@ static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova, 
	return 0;

}



/* caller must take care of cache maintenance on *ptep */

void av8l_fast_unmap_public(av8l_fast_iopte *ptep, size_t size)

static void __av8l_fast_unmap(av8l_fast_iopte *ptep, size_t size,

			      bool need_stale_tlb_tracking)

{

	unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;

	int val = need_stale_tlb_tracking

		? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI

		: 0;



	memset(ptep, 0, sizeof(*ptep) * nptes);

	memset(ptep, val, sizeof(*ptep) * nptes);

}



/* caller must take care of cache maintenance on *ptep */

void av8l_fast_unmap_public(av8l_fast_iopte *ptep, size_t size)

{

	__av8l_fast_unmap(ptep, size, true);

}



/* upper layer must take care of TLB invalidation */
@@ -200,7 +259,7 @@ static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, 
	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);

	unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;



	av8l_fast_unmap_public(ptep, size);

	__av8l_fast_unmap(ptep, size, false);



	data->iop.cfg.tlb->flush_pgtable(

		ptep, sizeof(*ptep) * nptes,
@@ -548,6 +607,9 @@ static int __init av8l_fast_positive_testing(void) 
			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 8K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
@@ -566,6 +628,9 @@ static int __init av8l_fast_positive_testing(void) 
			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 16K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
@@ -584,6 +649,9 @@ static int __init av8l_fast_positive_testing(void) 
			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 64K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_64K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {

include/linux/dma-mapping-fast.h

+1 −0
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ struct dma_fast_smmu_mapping { 
	av8l_fast_iopte	*pgtbl_pmds;



	spinlock_t	lock;

	struct notifier_block notifier;

};



#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST

include/linux/io-pgtable-fast.h

+33 −0
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@ 
#ifndef __LINUX_IO_PGTABLE_FAST_H

#define __LINUX_IO_PGTABLE_FAST_H



#include <linux/notifier.h>



typedef u64 av8l_fast_iopte;



#define iopte_pmd_offset(pmds, iova) (pmds + (iova >> 12))
@@ -21,4 +23,35 @@ int av8l_fast_map_public(av8l_fast_iopte *ptep, phys_addr_t paddr, size_t size, 
			 int prot);

void av8l_fast_unmap_public(av8l_fast_iopte *ptep, size_t size);



/* events for notifiers passed to av8l_register_notify */

#define MAPPED_OVER_STALE_TLB 1





#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB

/*

 * Doesn't matter what we use as long as bit 0 is unset.  The reason why we

 * need a different value at all is that there are certain hardware

 * platforms with erratum that require that a PTE actually be zero'd out

 * and not just have its valid bit unset.

 */

#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa



void av8l_fast_clear_stale_ptes(av8l_fast_iopte *puds, bool skip_sync);

void av8l_register_notify(struct notifier_block *nb);



#else  /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */



#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0



static inline void av8l_fast_clear_stale_ptes(av8l_fast_iopte *puds,

					      bool skip_sync)

{

}



static inline void av8l_register_notify(struct notifier_block *nb)

{

}



#endif	/* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */



#endif /* __LINUX_IO_PGTABLE_FAST_H */