Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6a32fd4d authored by David S. Miller's avatar David S. Miller
Browse files

[SPARC64]: Remove PGLIST_NENTS PCI IOMMU mapping limitation on SUN4V. 



Use a batching queue system for IOMMU mapping setup,
with a page sized batch.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 04d74758

arch/sparc64/kernel/pci_sun4v.c

+158 −75
Original line number Diff line number Diff line
@@ -26,11 +26,86 @@ 


#define PGLIST_NENTS	(PAGE_SIZE / sizeof(u64))



struct sun4v_pglist {

	u64	*pglist;

struct pci_iommu_batch {

	struct pci_dev	*pdev;		/* Device mapping is for.	*/

	unsigned long	prot;		/* IOMMU page protections	*/

	unsigned long	entry;		/* Index into IOTSB.		*/

	u64		*pglist;	/* List of physical pages	*/

	unsigned long	npages;		/* Number of pages in list.	*/

};



static DEFINE_PER_CPU(struct sun4v_pglist, iommu_pglists);

static DEFINE_PER_CPU(struct pci_iommu_batch, pci_iommu_batch);



/* Interrupts must be disabled.  */

static inline void pci_iommu_batch_start(struct pci_dev *pdev, unsigned long prot, unsigned long entry)

{

	struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);



	p->pdev		= pdev;

	p->prot		= prot;

	p->entry	= entry;

	p->npages	= 0;

}



/* Interrupts must be disabled.  */

static long pci_iommu_batch_flush(struct pci_iommu_batch *p)

{

	struct pcidev_cookie *pcp = p->pdev->sysdata;

	unsigned long devhandle = pcp->pbm->devhandle;

	unsigned long prot = p->prot;

	unsigned long entry = p->entry;

	u64 *pglist = p->pglist;

	unsigned long npages = p->npages;



	do {

		long num;



		num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),

					  npages, prot, __pa(pglist));

		if (unlikely(num < 0)) {

			if (printk_ratelimit())

				printk("pci_iommu_batch_flush: IOMMU map of "

				       "[%08lx:%08lx:%lx:%lx:%lx] failed with "

				       "status %ld\n",

				       devhandle, HV_PCI_TSBID(0, entry),

				       npages, prot, __pa(pglist), num);

			return -1;

		}



		entry += num;

		npages -= num;

		pglist += num;

	} while (npages != 0);



	p->entry = entry;

	p->npages = 0;



	return 0;

}



/* Interrupts must be disabled.  */

static inline long pci_iommu_batch_add(u64 phys_page)

{

	struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);



	BUG_ON(p->npages >= PGLIST_NENTS);



	p->pglist[p->npages++] = phys_page;

	if (p->npages == PGLIST_NENTS)

		return pci_iommu_batch_flush(p);



	return 0;

}



/* Interrupts must be disabled.  */

static inline long pci_iommu_batch_end(void)

{

	struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch);



	BUG_ON(p->npages >= PGLIST_NENTS);



	return pci_iommu_batch_flush(p);

}



static long pci_arena_alloc(struct pci_iommu_arena *arena, unsigned long npages)

{
@@ -86,65 +161,64 @@ static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr 
	unsigned long flags, order, first_page, npages, n;

	void *ret;

	long entry;

	u64 *pglist;

	u32 devhandle;

	int cpu;



	size = IO_PAGE_ALIGN(size);

	order = get_order(size);

	if (order >= MAX_ORDER)

	if (unlikely(order >= MAX_ORDER))

		return NULL;



	npages = size >> IO_PAGE_SHIFT;

	if (npages > PGLIST_NENTS)

		return NULL;



	first_page = __get_free_pages(GFP_ATOMIC, order);

	if (first_page == 0UL)

	if (unlikely(first_page == 0UL))

		return NULL;



	memset((char *)first_page, 0, PAGE_SIZE << order);



	pcp = pdev->sysdata;

	devhandle = pcp->pbm->devhandle;

	iommu = pcp->pbm->iommu;



	spin_lock_irqsave(&iommu->lock, flags);

	entry = pci_arena_alloc(&iommu->arena, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



	if (unlikely(entry < 0L)) {

		free_pages(first_page, order);

		return NULL;

	}

	if (unlikely(entry < 0L))

		goto arena_alloc_fail;



	*dma_addrp = (iommu->page_table_map_base +

		      (entry << IO_PAGE_SHIFT));

	ret = (void *) first_page;

	first_page = __pa(first_page);



	cpu = get_cpu();



	pglist = __get_cpu_var(iommu_pglists).pglist;

	for (n = 0; n < npages; n++)

		pglist[n] = first_page + (n * PAGE_SIZE);

	local_irq_save(flags);



	do {

		unsigned long num;



		num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),

					  npages,

	pci_iommu_batch_start(pdev,

			      (HV_PCI_MAP_ATTR_READ |

			       HV_PCI_MAP_ATTR_WRITE),

					  __pa(pglist));

		entry += num;

		npages -= num;

		pglist += num;

	} while (npages != 0);

			      entry);



	for (n = 0; n < npages; n++) {

		long err = pci_iommu_batch_add(first_page + (n * PAGE_SIZE));

		if (unlikely(err < 0L))

			goto iommu_map_fail;

	}



	if (unlikely(pci_iommu_batch_end() < 0L))

		goto iommu_map_fail;



	put_cpu();

	local_irq_restore(flags);



	return ret;



iommu_map_fail:

	/* Interrupts are disabled.  */

	spin_lock(&iommu->lock);

	pci_arena_free(&iommu->arena, entry, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



arena_alloc_fail:

	free_pages(first_page, order);

	return NULL;

}



static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma)
@@ -186,15 +260,12 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, 
	struct pci_iommu *iommu;

	unsigned long flags, npages, oaddr;

	unsigned long i, base_paddr;

	u32 devhandle, bus_addr, ret;

	u32 bus_addr, ret;

	unsigned long prot;

	long entry;

	u64 *pglist;

	int cpu;



	pcp = pdev->sysdata;

	iommu = pcp->pbm->iommu;

	devhandle = pcp->pbm->devhandle;



	if (unlikely(direction == PCI_DMA_NONE))

		goto bad;
@@ -202,8 +273,6 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, 
	oaddr = (unsigned long)ptr;

	npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);

	npages >>= IO_PAGE_SHIFT;

	if (unlikely(npages > PGLIST_NENTS))

		goto bad;



	spin_lock_irqsave(&iommu->lock, flags);

	entry = pci_arena_alloc(&iommu->arena, npages);
@@ -220,24 +289,19 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, 
	if (direction != PCI_DMA_TODEVICE)

		prot |= HV_PCI_MAP_ATTR_WRITE;



	cpu = get_cpu();



	pglist = __get_cpu_var(iommu_pglists).pglist;

	for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE)

		pglist[i] = base_paddr;

	local_irq_save(flags);



	do {

		unsigned long num;

	pci_iommu_batch_start(pdev, prot, entry);



		num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),

					  npages, prot,

					  __pa(pglist));

		entry += num;

		npages -= num;

		pglist += num;

	} while (npages != 0);

	for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) {

		long err = pci_iommu_batch_add(base_paddr);

		if (unlikely(err < 0L))

			goto iommu_map_fail;

	}

	if (unlikely(pci_iommu_batch_end() < 0L))

		goto iommu_map_fail;



	put_cpu();

	local_irq_restore(flags);



	return ret;
@@ -245,6 +309,14 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, 
	if (printk_ratelimit())

		WARN_ON(1);

	return PCI_DMA_ERROR_CODE;



iommu_map_fail:

	/* Interrupts are disabled.  */

	spin_lock(&iommu->lock);

	pci_arena_free(&iommu->arena, entry, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



	return PCI_DMA_ERROR_CODE;

}



static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
@@ -289,18 +361,19 @@ static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ 
#define SG_ENT_PHYS_ADDRESS(SG)	\

	(__pa(page_address((SG)->page)) + (SG)->offset)



static inline void fill_sg(long entry, u32 devhandle,

static inline long fill_sg(long entry, struct pci_dev *pdev,

			   struct scatterlist *sg,

			   int nused, int nelems, unsigned long prot)

{

	struct scatterlist *dma_sg = sg;

	struct scatterlist *sg_end = sg + nelems;

	int i, cpu, pglist_ent;

	u64 *pglist;

	unsigned long flags;

	int i;



	local_irq_save(flags);



	pci_iommu_batch_start(pdev, prot, entry);



	cpu = get_cpu();

	pglist = __get_cpu_var(iommu_pglists).pglist;

	pglist_ent = 0;

	for (i = 0; i < nused; i++) {

		unsigned long pteval = ~0UL;

		u32 dma_npages;
@@ -338,7 +411,12 @@ static inline void fill_sg(long entry, u32 devhandle, 


			pteval = (pteval & IOPTE_PAGE);

			while (len > 0) {

				pglist[pglist_ent++] = pteval;

				long err;



				err = pci_iommu_batch_add(pteval);

				if (unlikely(err < 0L))

					goto iommu_map_failed;



				pteval += IO_PAGE_SIZE;

				len -= (IO_PAGE_SIZE - offset);

				offset = 0;
@@ -366,18 +444,15 @@ static inline void fill_sg(long entry, u32 devhandle, 
		dma_sg++;

	}



	BUG_ON(pglist_ent == 0);



	do {

		unsigned long num;

	if (unlikely(pci_iommu_batch_end() < 0L))

		goto iommu_map_failed;



		num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),

					    pglist_ent);

		entry += num;

		pglist_ent -= num;

	} while (pglist_ent != 0);

	local_irq_restore(flags);

	return 0;



	put_cpu();

iommu_map_failed:

	local_irq_restore(flags);

	return -1L;

}



static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
@@ -385,9 +460,9 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n 
	struct pcidev_cookie *pcp;

	struct pci_iommu *iommu;

	unsigned long flags, npages, prot;

	u32 devhandle, dma_base;

	u32 dma_base;

	struct scatterlist *sgtmp;

	long entry;

	long entry, err;

	int used;



	/* Fast path single entry scatterlists. */
@@ -404,7 +479,6 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n 


	pcp = pdev->sysdata;

	iommu = pcp->pbm->iommu;

	devhandle = pcp->pbm->devhandle;

	

	if (unlikely(direction == PCI_DMA_NONE))

		goto bad;
@@ -441,7 +515,9 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n 
	if (direction != PCI_DMA_TODEVICE)

		prot |= HV_PCI_MAP_ATTR_WRITE;



	fill_sg(entry, devhandle, sglist, used, nelems, prot);

	err = fill_sg(entry, pdev, sglist, used, nelems, prot);

	if (unlikely(err < 0L))

		goto iommu_map_failed;



	return used;
@@ -449,6 +525,13 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n 
	if (printk_ratelimit())

		WARN_ON(1);

	return 0;



iommu_map_failed:

	spin_lock_irqsave(&iommu->lock, flags);

	pci_arena_free(&iommu->arena, entry, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



	return 0;

}



static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
@@ -1011,13 +1094,13 @@ void sun4v_pci_init(int node, char *model_name) 
		}

	}



	for (i = 0; i < NR_CPUS; i++) {

	for_each_cpu(i) {

		unsigned long page = get_zeroed_page(GFP_ATOMIC);



		if (!page)

			goto fatal_memory_error;



		per_cpu(iommu_pglists, i).pglist = (u64 *) page;

		per_cpu(pci_iommu_batch, i).pglist = (u64 *) page;

	}



	p = kmalloc(sizeof(struct pci_controller_info), GFP_ATOMIC);

arch/sparc64/kernel/pci_sun4v.h

+5 −5
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ 
#ifndef _PCI_SUN4V_H

#define _PCI_SUN4V_H



extern unsigned long pci_sun4v_iommu_map(unsigned long devhandle,

extern long pci_sun4v_iommu_map(unsigned long devhandle,

				unsigned long tsbid,

				unsigned long num_ttes,

				unsigned long io_attributes,

arch/sparc64/kernel/pci_sun4v_asm.S

+8 −3
Original line number Diff line number Diff line
@@ -11,14 +11,19 @@ 
	 * %o3:	io_attributes

	 * %o4:	io_page_list phys address

	 *

	 * returns %o0:	num ttes mapped

	 * returns %o0:	-status if status was non-zero, else

	 *         %o0:	num pages mapped

	 */

	.globl	pci_sun4v_iommu_map

pci_sun4v_iommu_map:

	mov	%o5, %g1

	mov	HV_FAST_PCI_IOMMU_MAP, %o5

	ta	HV_FAST_TRAP

	retl

	brnz,pn %o0, 1f

	 sub	%g0, %o0, %o0

	mov	%o1, %o0

1:	retl

	 nop



	/* %o0: devhandle

	 * %o1:	tsbid