[POWERPC] Use 4kB iommu pages even on 64kB-page systems (5d2efba6) · Commits · e / devices / android_kernel_oneplus_sm8150

arch/powerpc/kernel/iommu.c

+45 −32

Original line number	Original line	Diff line number	Diff line
	@@ -47,6 +47,17 @@ static int novmerge = 0;
	static int novmerge = 1;		static int novmerge = 1;
	#endif		#endif

			static inline unsigned long iommu_num_pages(unsigned long vaddr,
			unsigned long slen)
			{
			unsigned long npages;

			npages = IOMMU_PAGE_ALIGN(vaddr + slen) - (vaddr & IOMMU_PAGE_MASK);
			npages >>= IOMMU_PAGE_SHIFT;

			return npages;
			}

	static int __init setup_iommu(char *str)		static int __init setup_iommu(char *str)
	{		{
	if (!strcmp(str, "novmerge"))		if (!strcmp(str, "novmerge"))
	@@ -178,10 +189,10 @@ static dma_addr_t iommu_alloc(struct iommu_table tbl, void page,
	}		}

	entry += tbl->it_offset; /* Offset into real TCE table */		entry += tbl->it_offset; /* Offset into real TCE table */
	ret = entry << PAGE_SHIFT; /* Set the return dma address */		ret = entry << IOMMU_PAGE_SHIFT; /* Set the return dma address */

	/* Put the TCEs in the HW table */		/* Put the TCEs in the HW table */
	ppc_md.tce_build(tbl, entry, npages, (unsigned long)page & PAGE_MASK,		ppc_md.tce_build(tbl, entry, npages, (unsigned long)page & IOMMU_PAGE_MASK,
	direction);		direction);


	@@ -203,7 +214,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
	unsigned long entry, free_entry;		unsigned long entry, free_entry;
	unsigned long i;		unsigned long i;

	entry = dma_addr >> PAGE_SHIFT;		entry = dma_addr >> IOMMU_PAGE_SHIFT;
	free_entry = entry - tbl->it_offset;		free_entry = entry - tbl->it_offset;

	if (((free_entry + npages) > tbl->it_size) \|\|		if (((free_entry + npages) > tbl->it_size) \|\|
	@@ -270,7 +281,7 @@ int iommu_map_sg(struct device dev, struct iommu_table tbl,
	/* Init first segment length for backout at failure */		/* Init first segment length for backout at failure */
	outs->dma_length = 0;		outs->dma_length = 0;

	DBG("mapping %d elements:\n", nelems);		DBG("sg mapping %d elements:\n", nelems);

	spin_lock_irqsave(&(tbl->it_lock), flags);		spin_lock_irqsave(&(tbl->it_lock), flags);

	@@ -285,9 +296,8 @@ int iommu_map_sg(struct device dev, struct iommu_table tbl,
	}		}
	/* Allocate iommu entries for that segment */		/* Allocate iommu entries for that segment */
	vaddr = (unsigned long)page_address(s->page) + s->offset;		vaddr = (unsigned long)page_address(s->page) + s->offset;
	npages = PAGE_ALIGN(vaddr + slen) - (vaddr & PAGE_MASK);		npages = iommu_num_pages(vaddr, slen);
	npages >>= PAGE_SHIFT;		entry = iommu_range_alloc(tbl, npages, &handle, mask >> IOMMU_PAGE_SHIFT, 0);
	entry = iommu_range_alloc(tbl, npages, &handle, mask >> PAGE_SHIFT, 0);

	DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);		DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);

	@@ -301,14 +311,14 @@ int iommu_map_sg(struct device dev, struct iommu_table tbl,

	/* Convert entry to a dma_addr_t */		/* Convert entry to a dma_addr_t */
	entry += tbl->it_offset;		entry += tbl->it_offset;
	dma_addr = entry << PAGE_SHIFT;		dma_addr = entry << IOMMU_PAGE_SHIFT;
	dma_addr \|= s->offset;		dma_addr \|= (s->offset & ~IOMMU_PAGE_MASK);

	DBG(" - %lx pages, entry: %lx, dma_addr: %lx\n",		DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
	npages, entry, dma_addr);		npages, entry, dma_addr);

	/* Insert into HW table */		/* Insert into HW table */
	ppc_md.tce_build(tbl, entry, npages, vaddr & PAGE_MASK, direction);		ppc_md.tce_build(tbl, entry, npages, vaddr & IOMMU_PAGE_MASK, direction);

	/* If we are in an open segment, try merging */		/* If we are in an open segment, try merging */
	if (segstart != s) {		if (segstart != s) {
	@@ -323,7 +333,7 @@ int iommu_map_sg(struct device dev, struct iommu_table tbl,
	DBG(" can't merge, new segment.\n");		DBG(" can't merge, new segment.\n");
	} else {		} else {
	outs->dma_length += s->length;		outs->dma_length += s->length;
	DBG(" merged, new len: %lx\n", outs->dma_length);		DBG(" merged, new len: %ux\n", outs->dma_length);
	}		}
	}		}

	@@ -367,9 +377,8 @@ int iommu_map_sg(struct device dev, struct iommu_table tbl,
	if (s->dma_length != 0) {		if (s->dma_length != 0) {
	unsigned long vaddr, npages;		unsigned long vaddr, npages;

	vaddr = s->dma_address & PAGE_MASK;		vaddr = s->dma_address & IOMMU_PAGE_MASK;
	npages = (PAGE_ALIGN(s->dma_address + s->dma_length) - vaddr)		npages = iommu_num_pages(s->dma_address, s->dma_length);
	>> PAGE_SHIFT;
	__iommu_free(tbl, vaddr, npages);		__iommu_free(tbl, vaddr, npages);
	s->dma_address = DMA_ERROR_CODE;		s->dma_address = DMA_ERROR_CODE;
	s->dma_length = 0;		s->dma_length = 0;
	@@ -398,8 +407,7 @@ void iommu_unmap_sg(struct iommu_table tbl, struct scatterlist sglist,

	if (sglist->dma_length == 0)		if (sglist->dma_length == 0)
	break;		break;
	npages = (PAGE_ALIGN(dma_handle + sglist->dma_length)		npages = iommu_num_pages(dma_handle,sglist->dma_length);
	- (dma_handle & PAGE_MASK)) >> PAGE_SHIFT;
	__iommu_free(tbl, dma_handle, npages);		__iommu_free(tbl, dma_handle, npages);
	sglist++;		sglist++;
	}		}
	@@ -532,12 +540,11 @@ dma_addr_t iommu_map_single(struct iommu_table tbl, void vaddr,
	BUG_ON(direction == DMA_NONE);		BUG_ON(direction == DMA_NONE);

	uaddr = (unsigned long)vaddr;		uaddr = (unsigned long)vaddr;
	npages = PAGE_ALIGN(uaddr + size) - (uaddr & PAGE_MASK);		npages = iommu_num_pages(uaddr, size);
	npages >>= PAGE_SHIFT;

	if (tbl) {		if (tbl) {
	dma_handle = iommu_alloc(tbl, vaddr, npages, direction,		dma_handle = iommu_alloc(tbl, vaddr, npages, direction,
	mask >> PAGE_SHIFT, 0);		mask >> IOMMU_PAGE_SHIFT, 0);
	if (dma_handle == DMA_ERROR_CODE) {		if (dma_handle == DMA_ERROR_CODE) {
	if (printk_ratelimit()) {		if (printk_ratelimit()) {
	printk(KERN_INFO "iommu_alloc failed, "		printk(KERN_INFO "iommu_alloc failed, "
	@@ -545,7 +552,7 @@ dma_addr_t iommu_map_single(struct iommu_table tbl, void vaddr,
	tbl, vaddr, npages);		tbl, vaddr, npages);
	}		}
	} else		} else
	dma_handle \|= (uaddr & ~PAGE_MASK);		dma_handle \|= (uaddr & ~IOMMU_PAGE_MASK);
	}		}

	return dma_handle;		return dma_handle;
	@@ -554,11 +561,14 @@ dma_addr_t iommu_map_single(struct iommu_table tbl, void vaddr,
	void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle,		void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle,
	size_t size, enum dma_data_direction direction)		size_t size, enum dma_data_direction direction)
	{		{
			unsigned int npages;

	BUG_ON(direction == DMA_NONE);		BUG_ON(direction == DMA_NONE);

	if (tbl)		if (tbl) {
	iommu_free(tbl, dma_handle, (PAGE_ALIGN(dma_handle + size) -		npages = iommu_num_pages(dma_handle, size);
	(dma_handle & PAGE_MASK)) >> PAGE_SHIFT);		iommu_free(tbl, dma_handle, npages);
			}
	}		}

	/* Allocates a contiguous real buffer and creates mappings over it.		/* Allocates a contiguous real buffer and creates mappings over it.
	@@ -570,11 +580,11 @@ void iommu_alloc_coherent(struct iommu_table tbl, size_t size,
	{		{
	void *ret = NULL;		void *ret = NULL;
	dma_addr_t mapping;		dma_addr_t mapping;
	unsigned int npages, order;		unsigned int order;
			unsigned int nio_pages, io_order;
	struct page *page;		struct page *page;

	size = PAGE_ALIGN(size);		size = PAGE_ALIGN(size);
	npages = size >> PAGE_SHIFT;
	order = get_order(size);		order = get_order(size);

	/*		/*
	@@ -598,8 +608,10 @@ void iommu_alloc_coherent(struct iommu_table tbl, size_t size,
	memset(ret, 0, size);		memset(ret, 0, size);

	/* Set up tces to cover the allocated range */		/* Set up tces to cover the allocated range */
	mapping = iommu_alloc(tbl, ret, npages, DMA_BIDIRECTIONAL,		nio_pages = size >> IOMMU_PAGE_SHIFT;
	mask >> PAGE_SHIFT, order);		io_order = get_iommu_order(size);
			mapping = iommu_alloc(tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
			mask >> IOMMU_PAGE_SHIFT, io_order);
	if (mapping == DMA_ERROR_CODE) {		if (mapping == DMA_ERROR_CODE) {
	free_pages((unsigned long)ret, order);		free_pages((unsigned long)ret, order);
	return NULL;		return NULL;
	@@ -611,12 +623,13 @@ void iommu_alloc_coherent(struct iommu_table tbl, size_t size,
	void iommu_free_coherent(struct iommu_table *tbl, size_t size,		void iommu_free_coherent(struct iommu_table *tbl, size_t size,
	void *vaddr, dma_addr_t dma_handle)		void *vaddr, dma_addr_t dma_handle)
	{		{
	unsigned int npages;

	if (tbl) {		if (tbl) {
			unsigned int nio_pages;

			size = PAGE_ALIGN(size);
			nio_pages = size >> IOMMU_PAGE_SHIFT;
			iommu_free(tbl, dma_handle, nio_pages);
	size = PAGE_ALIGN(size);		size = PAGE_ALIGN(size);
	npages = size >> PAGE_SHIFT;
	iommu_free(tbl, dma_handle, npages);
	free_pages((unsigned long)vaddr, get_order(size));		free_pages((unsigned long)vaddr, get_order(size));
	}		}
	}		}

arch/powerpc/kernel/vio.c

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -92,9 +92,9 @@ static struct iommu_table vio_build_iommu_table(struct vio_dev dev)
	&tbl->it_index, &offset, &size);		&tbl->it_index, &offset, &size);

	/* TCE table size - measured in tce entries */		/* TCE table size - measured in tce entries */
	tbl->it_size = size >> PAGE_SHIFT;		tbl->it_size = size >> IOMMU_PAGE_SHIFT;
	/* offset for VIO should always be 0 */		/* offset for VIO should always be 0 */
	tbl->it_offset = offset >> PAGE_SHIFT;		tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
	tbl->it_busno = 0;		tbl->it_busno = 0;
	tbl->it_type = TCE_VB;		tbl->it_type = TCE_VB;

arch/powerpc/platforms/iseries/iommu.c

+2 −9

Original line number	Original line	Diff line number	Diff line
	@@ -43,9 +43,6 @@ static void tce_build_iSeries(struct iommu_table *tbl, long index, long npages,
	u64 rc;		u64 rc;
	u64 tce, rpn;		u64 tce, rpn;

	index <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	while (npages--) {		while (npages--) {
	rpn = virt_to_abs(uaddr) >> TCE_SHIFT;		rpn = virt_to_abs(uaddr) >> TCE_SHIFT;
	tce = (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;		tce = (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
	@@ -75,9 +72,6 @@ static void tce_free_iSeries(struct iommu_table *tbl, long index, long npages)
	{		{
	u64 rc;		u64 rc;

	npages <<= TCE_PAGE_FACTOR;
	index <<= TCE_PAGE_FACTOR;

	while (npages--) {		while (npages--) {
	rc = HvCallXm_setTce((u64)tbl->it_index, (u64)index, 0);		rc = HvCallXm_setTce((u64)tbl->it_index, (u64)index, 0);
	if (rc)		if (rc)
	@@ -136,10 +130,9 @@ void iommu_table_getparms_iSeries(unsigned long busno,
	panic("PCI_DMA: parms->size is zero, parms is 0x%p", parms);		panic("PCI_DMA: parms->size is zero, parms is 0x%p", parms);

	/* itc_size is in pages worth of table, it_size is in # of entries */		/* itc_size is in pages worth of table, it_size is in # of entries */
	tbl->it_size = ((parms->itc_size * TCE_PAGE_SIZE) /		tbl->it_size = (parms->itc_size * TCE_PAGE_SIZE) / TCE_ENTRY_SIZE;
	TCE_ENTRY_SIZE) >> TCE_PAGE_FACTOR;
	tbl->it_busno = parms->itc_busno;		tbl->it_busno = parms->itc_busno;
	tbl->it_offset = parms->itc_offset >> TCE_PAGE_FACTOR;		tbl->it_offset = parms->itc_offset;
	tbl->it_index = parms->itc_index;		tbl->it_index = parms->itc_index;
	tbl->it_blocksize = 1;		tbl->it_blocksize = 1;
	tbl->it_type = virtbus ? TCE_VB : TCE_PCI;		tbl->it_type = virtbus ? TCE_VB : TCE_PCI;

arch/powerpc/platforms/pseries/iommu.c

+8 −27

Original line number	Original line	Diff line number	Diff line
	@@ -57,9 +57,6 @@ static void tce_build_pSeries(struct iommu_table *tbl, long index,
	u64 *tcep;		u64 *tcep;
	u64 rpn;		u64 rpn;

	index <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	proto_tce = TCE_PCI_READ; // Read allowed		proto_tce = TCE_PCI_READ; // Read allowed

	if (direction != DMA_TO_DEVICE)		if (direction != DMA_TO_DEVICE)
	@@ -82,9 +79,6 @@ static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
	{		{
	u64 *tcep;		u64 *tcep;

	npages <<= TCE_PAGE_FACTOR;
	index <<= TCE_PAGE_FACTOR;

	tcep = ((u64 *)tbl->it_base) + index;		tcep = ((u64 *)tbl->it_base) + index;

	while (npages--)		while (npages--)
	@@ -95,7 +89,6 @@ static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
	{		{
	u64 *tcep;		u64 *tcep;

	index <<= TCE_PAGE_FACTOR;
	tcep = ((u64 *)tbl->it_base) + index;		tcep = ((u64 *)tbl->it_base) + index;

	return *tcep;		return *tcep;
	@@ -109,9 +102,6 @@ static void tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
	u64 proto_tce, tce;		u64 proto_tce, tce;
	u64 rpn;		u64 rpn;

	tcenum <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;		rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
	proto_tce = TCE_PCI_READ;		proto_tce = TCE_PCI_READ;
	if (direction != DMA_TO_DEVICE)		if (direction != DMA_TO_DEVICE)
	@@ -146,7 +136,7 @@ static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
	u64 rpn;		u64 rpn;
	long l, limit;		long l, limit;

	if (TCE_PAGE_FACTOR == 0 && npages == 1)		if (npages == 1)
	return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,		return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
	direction);		direction);

	@@ -164,9 +154,6 @@ static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
	__get_cpu_var(tce_page) = tcep;		__get_cpu_var(tce_page) = tcep;
	}		}

	tcenum <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;		rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
	proto_tce = TCE_PCI_READ;		proto_tce = TCE_PCI_READ;
	if (direction != DMA_TO_DEVICE)		if (direction != DMA_TO_DEVICE)
	@@ -207,9 +194,6 @@ static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages
	{		{
	u64 rc;		u64 rc;

	tcenum <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	while (npages--) {		while (npages--) {
	rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);		rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);

	@@ -229,9 +213,6 @@ static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long n
	{		{
	u64 rc;		u64 rc;

	tcenum <<= TCE_PAGE_FACTOR;
	npages <<= TCE_PAGE_FACTOR;

	rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);		rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);

	if (rc && printk_ratelimit()) {		if (rc && printk_ratelimit()) {
	@@ -248,7 +229,6 @@ static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
	u64 rc;		u64 rc;
	unsigned long tce_ret;		unsigned long tce_ret;

	tcenum <<= TCE_PAGE_FACTOR;
	rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);		rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);

	if (rc && printk_ratelimit()) {		if (rc && printk_ratelimit()) {
	@@ -289,7 +269,7 @@ static void iommu_table_setparms(struct pci_controller *phb,
	tbl->it_busno = phb->bus->number;		tbl->it_busno = phb->bus->number;

	/* Units of tce entries */		/* Units of tce entries */
	tbl->it_offset = phb->dma_window_base_cur >> PAGE_SHIFT;		tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;

	/* Test if we are going over 2GB of DMA space */		/* Test if we are going over 2GB of DMA space */
	if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {		if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
	@@ -300,7 +280,7 @@ static void iommu_table_setparms(struct pci_controller *phb,
	phb->dma_window_base_cur += phb->dma_window_size;		phb->dma_window_base_cur += phb->dma_window_size;

	/* Set the tce table size - measured in entries */		/* Set the tce table size - measured in entries */
	tbl->it_size = phb->dma_window_size >> PAGE_SHIFT;		tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;

	tbl->it_index = 0;		tbl->it_index = 0;
	tbl->it_blocksize = 16;		tbl->it_blocksize = 16;
	@@ -325,8 +305,8 @@ static void iommu_table_setparms_lpar(struct pci_controller *phb,
	tbl->it_base = 0;		tbl->it_base = 0;
	tbl->it_blocksize = 16;		tbl->it_blocksize = 16;
	tbl->it_type = TCE_PCI;		tbl->it_type = TCE_PCI;
	tbl->it_offset = offset >> PAGE_SHIFT;		tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
	tbl->it_size = size >> PAGE_SHIFT;		tbl->it_size = size >> IOMMU_PAGE_SHIFT;
	}		}

	static void iommu_bus_setup_pSeries(struct pci_bus *bus)		static void iommu_bus_setup_pSeries(struct pci_bus *bus)
	@@ -522,8 +502,6 @@ static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
	const void *dma_window = NULL;		const void *dma_window = NULL;
	struct pci_dn *pci;		struct pci_dn *pci;

	DBG("iommu_dev_setup_pSeriesLP, dev %p (%s)\n", dev, pci_name(dev));

	/* dev setup for LPAR is a little tricky, since the device tree might		/* dev setup for LPAR is a little tricky, since the device tree might
	* contain the dma-window properties per-device and not neccesarily		* contain the dma-window properties per-device and not neccesarily
	* for the bus. So we need to search upwards in the tree until we		* for the bus. So we need to search upwards in the tree until we
	@@ -532,6 +510,9 @@ static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
	*/		*/
	dn = pci_device_to_OF_node(dev);		dn = pci_device_to_OF_node(dev);

			DBG("iommu_dev_setup_pSeriesLP, dev %p (%s) %s\n",
			dev, pci_name(dev), dn->full_name);

	for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;		for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
	pdn = pdn->parent) {		pdn = pdn->parent) {
	dma_window = get_property(pdn, "ibm,dma-window", NULL);		dma_window = get_property(pdn, "ibm,dma-window", NULL);

arch/powerpc/sysdev/dart.h

+0 −1

Original line number	Original line	Diff line number	Diff line
	@@ -72,7 +72,6 @@

	#define DART_PAGE_SHIFT 12		#define DART_PAGE_SHIFT 12
	#define DART_PAGE_SIZE (1 << DART_PAGE_SHIFT)		#define DART_PAGE_SIZE (1 << DART_PAGE_SHIFT)
	#define DART_PAGE_FACTOR (PAGE_SHIFT - DART_PAGE_SHIFT)


	#endif /* _POWERPC_SYSDEV_DART_H */		#endif /* _POWERPC_SYSDEV_DART_H */