Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a788ade4 authored by Ben Skeggs's avatar Ben Skeggs
Browse files

drm/nouveau/dmem: use dma addresses during migration copies 



Removes the need for temporary VMM mappings.

Signed-off-by: default avatarBen Skeggs <bskeggs@redhat.com>
parent fd5e9856

drivers/gpu/drm/nouveau/nouveau_dmem.c

+53 −106
Original line number Original line Diff line number Diff line
@@ -80,17 +80,11 @@ struct nouveau_dmem { 
	struct mutex mutex;

	struct mutex mutex;

};

};





struct nouveau_migrate_hmem {

	struct scatterlist *sg;

	struct nouveau_mem mem;

	unsigned long npages;

	struct nvif_vma vma;

};



struct nouveau_dmem_fault {

struct nouveau_dmem_fault {

	struct nouveau_drm *drm;

	struct nouveau_drm *drm;

	struct nouveau_fence *fence;

	struct nouveau_fence *fence;

	struct nouveau_migrate_hmem hmem;

	dma_addr_t *dma;

	unsigned long npages;

};

};





struct nouveau_migrate {

struct nouveau_migrate {
@@ -98,87 +92,10 @@ struct nouveau_migrate { 
	struct nouveau_drm *drm;

	struct nouveau_drm *drm;

	struct nouveau_fence *fence;

	struct nouveau_fence *fence;

	unsigned long npages;

	unsigned long npages;

	struct nouveau_migrate_hmem hmem;

	dma_addr_t *dma;

	unsigned long dma_nr;

};

};





static void

nouveau_migrate_hmem_fini(struct nouveau_drm *drm,

			  struct nouveau_migrate_hmem *hmem)

{

	struct nvif_vmm *vmm = &drm->client.vmm.vmm;



	nouveau_mem_fini(&hmem->mem);

	nvif_vmm_put(vmm, &hmem->vma);



	if (hmem->sg) {

		dma_unmap_sg_attrs(drm->dev->dev, hmem->sg,

				   hmem->npages, DMA_BIDIRECTIONAL,

				   DMA_ATTR_SKIP_CPU_SYNC);

		kfree(hmem->sg);

		hmem->sg = NULL;

	}

}



static int

nouveau_migrate_hmem_init(struct nouveau_drm *drm,

			  struct nouveau_migrate_hmem *hmem,

			  unsigned long npages,

			  const unsigned long *pfns)

{

	struct nvif_vmm *vmm = &drm->client.vmm.vmm;

	unsigned long i;

	int ret;



	hmem->sg = kzalloc(npages * sizeof(*hmem->sg), GFP_KERNEL);

	if (hmem->sg == NULL)

		return -ENOMEM;



	for (i = 0, hmem->npages = 0; hmem->npages < npages; ++i) {

		struct page *page;



		if (!pfns[i] || pfns[i] == MIGRATE_PFN_ERROR)

			continue;



		page = migrate_pfn_to_page(pfns[i]);

		if (page == NULL) {

			ret = -EINVAL;

			goto error;

		}



		sg_set_page(&hmem->sg[hmem->npages], page, PAGE_SIZE, 0);

		hmem->npages++;

	}

	sg_mark_end(&hmem->sg[hmem->npages - 1]);



	i = dma_map_sg_attrs(drm->dev->dev, hmem->sg, hmem->npages,

			     DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);

	if (i != hmem->npages) {

		ret = -ENOMEM;

		goto error;

	}



	ret = nouveau_mem_sgl(&hmem->mem, &drm->client,

			      hmem->npages, hmem->sg);

	if (ret)

		goto error;



	ret = nvif_vmm_get(vmm, LAZY, false, hmem->mem.mem.page,

			   0, hmem->mem.mem.size, &hmem->vma);

	if (ret)

		goto error;



	ret = nouveau_mem_map(&hmem->mem, vmm, &hmem->vma);

	if (ret)

		goto error;



	return 0;



error:

	nouveau_migrate_hmem_fini(drm, hmem);

	return ret;

}





static void

static void

nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)

nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)

{

{
@@ -218,7 +135,8 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 
{

{

	struct nouveau_dmem_fault *fault = private;

	struct nouveau_dmem_fault *fault = private;

	struct nouveau_drm *drm = fault->drm;

	struct nouveau_drm *drm = fault->drm;

	unsigned long addr, i, c, npages = 0;

	struct device *dev = drm->dev->dev;

	unsigned long addr, i, npages = 0;

	nouveau_migrate_copy_t copy;

	nouveau_migrate_copy_t copy;

	int ret;

	int ret;
@@ -243,14 +161,14 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 
		npages++;

		npages++;

	}

	}





	/* Create scatter list FIXME: get rid of scatter list */

	/* Allocate storage for DMA addresses, so we can unmap later. */

	ret = nouveau_migrate_hmem_init(drm, &fault->hmem, npages, dst_pfns);

	fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);

	if (ret)

	if (!fault->dma)

		goto error;

		goto error;





	/* Copy things over */

	/* Copy things over */

	copy = drm->dmem->migrate.copy_func;

	copy = drm->dmem->migrate.copy_func;

	for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {

	for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {

		struct nouveau_dmem_chunk *chunk;

		struct nouveau_dmem_chunk *chunk;

		struct page *spage, *dpage;

		struct page *spage, *dpage;

		u64 src_addr, dst_addr;

		u64 src_addr, dst_addr;
@@ -259,9 +177,6 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 
		if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)

		if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)

			continue;

			continue;





		dst_addr = fault->hmem.vma.addr + (c << PAGE_SHIFT);

		c++;



		spage = migrate_pfn_to_page(src_pfns[i]);

		spage = migrate_pfn_to_page(src_pfns[i]);

		if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {

		if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {

			dst_pfns[i] = MIGRATE_PFN_ERROR;

			dst_pfns[i] = MIGRATE_PFN_ERROR;
@@ -269,11 +184,23 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 
			continue;

			continue;

		}

		}





		fault->dma[fault->npages] =

			dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,

					   PCI_DMA_BIDIRECTIONAL,

					   DMA_ATTR_SKIP_CPU_SYNC);

		if (dma_mapping_error(dev, fault->dma[fault->npages])) {

			dst_pfns[i] = MIGRATE_PFN_ERROR;

			__free_page(dpage);

			continue;

		}



		dst_addr = fault->dma[fault->npages++];



		chunk = (void *)hmm_devmem_page_get_drvdata(spage);

		chunk = (void *)hmm_devmem_page_get_drvdata(spage);

		src_addr = page_to_pfn(spage) - chunk->pfn_first;

		src_addr = page_to_pfn(spage) - chunk->pfn_first;

		src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;

		src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;





		ret = copy(drm, 1, NOUVEAU_APER_VIRT, dst_addr,

		ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,

				   NOUVEAU_APER_VRAM, src_addr);

				   NOUVEAU_APER_VRAM, src_addr);

		if (ret) {

		if (ret) {

			dst_pfns[i] = MIGRATE_PFN_ERROR;

			dst_pfns[i] = MIGRATE_PFN_ERROR;
@@ -321,7 +248,12 @@ void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma, 
		 * the hmem object below (nouveau_migrate_hmem_fini()).

		 * the hmem object below (nouveau_migrate_hmem_fini()).

		 */

		 */

	}

	}

	nouveau_migrate_hmem_fini(drm, &fault->hmem);



	while (fault->npages--) {

		dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],

			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);

	}

	kfree(fault->dma);

}

}





static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {

static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
@@ -732,7 +664,8 @@ nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, 
{

{

	struct nouveau_migrate *migrate = private;

	struct nouveau_migrate *migrate = private;

	struct nouveau_drm *drm = migrate->drm;

	struct nouveau_drm *drm = migrate->drm;

	unsigned long addr, i, c, npages = 0;

	struct device *dev = drm->dev->dev;

	unsigned long addr, i, npages = 0;

	nouveau_migrate_copy_t copy;

	nouveau_migrate_copy_t copy;

	int ret;

	int ret;
@@ -758,14 +691,14 @@ nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, 
	if (!npages)

	if (!npages)

		return;

		return;





	/* Create scatter list FIXME: get rid of scatter list */

	/* Allocate storage for DMA addresses, so we can unmap later. */

	ret = nouveau_migrate_hmem_init(drm, &migrate->hmem, npages, src_pfns);

	migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);

	if (ret)

	if (!migrate->dma)

		goto error;

		goto error;





	/* Copy things over */

	/* Copy things over */

	copy = drm->dmem->migrate.copy_func;

	copy = drm->dmem->migrate.copy_func;

	for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {

	for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {

		struct nouveau_dmem_chunk *chunk;

		struct nouveau_dmem_chunk *chunk;

		struct page *spage, *dpage;

		struct page *spage, *dpage;

		u64 src_addr, dst_addr;

		u64 src_addr, dst_addr;
@@ -785,11 +718,20 @@ nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, 
			continue;

			continue;

		}

		}





		src_addr = migrate->hmem.vma.addr + (c << PAGE_SHIFT);

		migrate->dma[migrate->dma_nr] =

		c++;

			dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,

					   PCI_DMA_BIDIRECTIONAL,

					   DMA_ATTR_SKIP_CPU_SYNC);

		if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {

			nouveau_dmem_page_free_locked(drm, dpage);

			dst_pfns[i] = 0;

			continue;

		}



		src_addr = migrate->dma[migrate->dma_nr++];





		ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,

		ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,

				   NOUVEAU_APER_VIRT, src_addr);

				   NOUVEAU_APER_HOST, src_addr);

		if (ret) {

		if (ret) {

			nouveau_dmem_page_free_locked(drm, dpage);

			nouveau_dmem_page_free_locked(drm, dpage);

			dst_pfns[i] = 0;

			dst_pfns[i] = 0;
@@ -836,7 +778,12 @@ void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma, 
		 * the hmem object below (nouveau_migrate_hmem_fini()) ?

		 * the hmem object below (nouveau_migrate_hmem_fini()) ?

		 */

		 */

	}

	}

	nouveau_migrate_hmem_fini(drm, &migrate->hmem);



	while (migrate->dma_nr--) {

		dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],

			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);

	}

	kfree(migrate->dma);





	/*

	/*

	 * FIXME optimization: update GPU page table to point to newly

	 * FIXME optimization: update GPU page table to point to newly