Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 26c0c9e3 authored by Ben Skeggs's avatar Ben Skeggs
Browse files

drm/nv50-nvc0: delay GART binding until move_notify time 



The immediate benefit of doing this is that on NV50 and up, the GPU
virtual address of any buffer is now constant, regardless of what
memtype they're placed in.

Signed-off-by: default avatarBen Skeggs <bskeggs@redhat.com>
parent d5f42394

drivers/gpu/drm/nouveau/nouveau_bo.c

+68 −46
Original line number Original line Diff line number Diff line
@@ -138,11 +138,8 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, 
	}

	}

	nvbo->channel = NULL;

	nvbo->channel = NULL;





	if (nvbo->vma.node) {

	if (nvbo->vma.node)

		if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)

		nvbo->bo.offset = nvbo->vma.offset;

		nvbo->bo.offset = nvbo->vma.offset;

	}



	*pnvbo = nvbo;

	*pnvbo = nvbo;

	return 0;

	return 0;

}

}
@@ -312,11 +309,8 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 
	if (ret)

	if (ret)

		return ret;

		return ret;





	if (nvbo->vma.node) {

	if (nvbo->vma.node)

		if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)

		nvbo->bo.offset = nvbo->vma.offset;

		nvbo->bo.offset = nvbo->vma.offset;

	}



	return 0;

	return 0;

}

}
@@ -426,6 +420,9 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 
		man->default_caching = TTM_PL_FLAG_WC;

		man->default_caching = TTM_PL_FLAG_WC;

		break;

		break;

	case TTM_PL_TT:

	case TTM_PL_TT:

		if (dev_priv->card_type >= NV_50)

			man->func = &nouveau_gart_manager;

		else

			man->func = &ttm_bo_manager_func;

			man->func = &ttm_bo_manager_func;

		switch (dev_priv->gart_info.type) {

		switch (dev_priv->gart_info.type) {

		case NOUVEAU_GART_AGP:

		case NOUVEAU_GART_AGP:
@@ -501,25 +498,18 @@ static int 
nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,

nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,

		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)

		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)

{

{

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	struct nouveau_mem *old_node = old_mem->mm_node;

	struct nouveau_mem *new_node = new_mem->mm_node;

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	u32 page_count = new_mem->num_pages;

	u32 page_count = new_mem->num_pages;

	u64 src_offset, dst_offset;

	u64 src_offset, dst_offset;

	int ret;

	int ret;





	src_offset = old_mem->start << PAGE_SHIFT;

	src_offset = old_node->tmp_vma.offset;

	if (old_mem->mem_type == TTM_PL_VRAM) {

	if (new_node->tmp_vma.node)

		struct nouveau_mem *node = old_mem->mm_node;

		dst_offset = new_node->tmp_vma.offset;

		src_offset  = node->tmp_vma.offset;

	} else {

		src_offset += dev_priv->gart_info.aper_base;

	}



	dst_offset = new_mem->start << PAGE_SHIFT;

	if (new_mem->mem_type == TTM_PL_VRAM)

		dst_offset  = nvbo->vma.offset;

	else

	else

		dst_offset += dev_priv->gart_info.aper_base;

		dst_offset = nvbo->vma.offset;





	page_count = new_mem->num_pages;

	page_count = new_mem->num_pages;

	while (page_count) {

	while (page_count) {
@@ -554,25 +544,18 @@ static int 
nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,

nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,

		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)

		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)

{

{

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	struct nouveau_mem *old_node = old_mem->mm_node;

	struct nouveau_mem *new_node = new_mem->mm_node;

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	u64 length = (new_mem->num_pages << PAGE_SHIFT);

	u64 length = (new_mem->num_pages << PAGE_SHIFT);

	u64 src_offset, dst_offset;

	u64 src_offset, dst_offset;

	int ret;

	int ret;





	src_offset = old_mem->start << PAGE_SHIFT;

	src_offset = old_node->tmp_vma.offset;

	if (old_mem->mem_type == TTM_PL_VRAM) {

	if (new_node->tmp_vma.node)

		struct nouveau_mem *node = old_mem->mm_node;

		dst_offset = new_node->tmp_vma.offset;

		src_offset  = node->tmp_vma.offset;

	} else {

		src_offset += dev_priv->gart_info.aper_base;

	}



	dst_offset = new_mem->start << PAGE_SHIFT;

	if (new_mem->mem_type == TTM_PL_VRAM)

		dst_offset  = nvbo->vma.offset;

	else

	else

		dst_offset += dev_priv->gart_info.aper_base;

		dst_offset = nvbo->vma.offset;





	while (length) {

	while (length) {

		u32 amount, stride, height;

		u32 amount, stride, height;
@@ -728,16 +711,28 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, 
	/* create temporary vma for old memory, this will get cleaned

	/* create temporary vma for old memory, this will get cleaned

	 * up after ttm destroys the ttm_mem_reg

	 * up after ttm destroys the ttm_mem_reg

	 */

	 */

	if (dev_priv->card_type >= NV_50 && old_mem->mem_type == TTM_PL_VRAM) {

	if (dev_priv->card_type >= NV_50) {

		struct nouveau_mem *node = old_mem->mm_node;

		struct nouveau_mem *node = old_mem->mm_node;



		if (!node->tmp_vma.node) {

		ret = nouveau_vm_get(chan->vm, old_mem->num_pages << PAGE_SHIFT,

			u32 page_shift = nvbo->vma.node->type;

				     nvbo->vma.node->type, NV_MEM_ACCESS_RO,

			if (old_mem->mem_type == TTM_PL_TT)

				page_shift = nvbo->vma.vm->spg_shift;



			ret = nouveau_vm_get(chan->vm,

					     old_mem->num_pages << PAGE_SHIFT,

					     page_shift, NV_MEM_ACCESS_RO,

					     &node->tmp_vma);

					     &node->tmp_vma);

			if (ret)

			if (ret)

				goto out;

				goto out;

		}





		if (old_mem->mem_type == TTM_PL_VRAM)

			nouveau_vm_map(&node->tmp_vma, node);

			nouveau_vm_map(&node->tmp_vma, node);

		else {

			nouveau_vm_map_sg(&node->tmp_vma, 0,

					  old_mem->num_pages << PAGE_SHIFT,

					  node, node->pages);

		}

	}

	}





	if (dev_priv->card_type < NV_50)

	if (dev_priv->card_type < NV_50)
@@ -764,6 +759,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 
		      bool no_wait_reserve, bool no_wait_gpu,

		      bool no_wait_reserve, bool no_wait_gpu,

		      struct ttm_mem_reg *new_mem)

		      struct ttm_mem_reg *new_mem)

{

{

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;

	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;

	struct ttm_placement placement;

	struct ttm_placement placement;

	struct ttm_mem_reg tmp_mem;

	struct ttm_mem_reg tmp_mem;
@@ -783,7 +779,23 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 
	if (ret)

	if (ret)

		goto out;

		goto out;





	if (dev_priv->card_type >= NV_50) {

		struct nouveau_bo *nvbo = nouveau_bo(bo);

		struct nouveau_mem *node = tmp_mem.mm_node;

		struct nouveau_vma *vma = &nvbo->vma;

		if (vma->node->type != vma->vm->spg_shift)

			vma = &node->tmp_vma;

		nouveau_vm_map_sg(vma, 0, tmp_mem.num_pages << PAGE_SHIFT,

				  node, node->pages);

	}



	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);

	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);



	if (dev_priv->card_type >= NV_50) {

		struct nouveau_bo *nvbo = nouveau_bo(bo);

		nouveau_vm_unmap(&nvbo->vma);

	}



	if (ret)

	if (ret)

		goto out;

		goto out;
@@ -830,16 +842,26 @@ static void 
nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)

nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)

{

{

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	struct nouveau_mem *node = new_mem->mm_node;

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	struct nouveau_vma *vma = &nvbo->vma;

	struct nouveau_vm *vm = vma->vm;





	if (dev_priv->card_type < NV_50)

	if (dev_priv->card_type < NV_50)

		return;

		return;





	switch (new_mem->mem_type) {

	switch (new_mem->mem_type) {

	case TTM_PL_VRAM:

	case TTM_PL_VRAM:

		nouveau_vm_map(&nvbo->vma, new_mem->mm_node);

		nouveau_vm_map(vma, node);

		break;

		break;

	case TTM_PL_TT:

	case TTM_PL_TT:

		if (vma->node->type != vm->spg_shift) {

			nouveau_vm_unmap(vma);

			vma = &node->tmp_vma;

		}

		nouveau_vm_map_sg(vma, 0, new_mem->num_pages << PAGE_SHIFT,

				  node, node->pages);

		break;

	default:

	default:

		nouveau_vm_unmap(&nvbo->vma);

		nouveau_vm_unmap(&nvbo->vma);

		break;

		break;

drivers/gpu/drm/nouveau/nouveau_drv.h

+2 −1
Original line number Original line Diff line number Diff line
@@ -73,6 +73,7 @@ struct nouveau_mem { 
	u8  page_shift;

	u8  page_shift;





	struct list_head regions;

	struct list_head regions;

	dma_addr_t *pages;

	u32 memtype;

	u32 memtype;

	u64 offset;

	u64 offset;

	u64 size;

	u64 size;
@@ -706,7 +707,6 @@ struct drm_nouveau_private { 
		} dummy;

		} dummy;





		struct nouveau_gpuobj *sg_ctxdma;

		struct nouveau_gpuobj *sg_ctxdma;

		struct nouveau_vma vma;

	} gart_info;

	} gart_info;





	/* nv10-nv40 tiling regions */

	/* nv10-nv40 tiling regions */
@@ -846,6 +846,7 @@ extern void nv10_mem_put_tile_region(struct drm_device *dev, 
				     struct nouveau_tile_reg *tile,

				     struct nouveau_tile_reg *tile,

				     struct nouveau_fence *fence);

				     struct nouveau_fence *fence);

extern const struct ttm_mem_type_manager_func nouveau_vram_manager;

extern const struct ttm_mem_type_manager_func nouveau_vram_manager;

extern const struct ttm_mem_type_manager_func nouveau_gart_manager;





/* nouveau_notifier.c */

/* nouveau_notifier.c */

extern int  nouveau_notifier_init_channel(struct nouveau_channel *);

extern int  nouveau_notifier_init_channel(struct nouveau_channel *);

drivers/gpu/drm/nouveau/nouveau_mem.c

+81 −0
Original line number Original line Diff line number Diff line
@@ -785,3 +785,84 @@ const struct ttm_mem_type_manager_func nouveau_vram_manager = { 
	nouveau_vram_manager_del,

	nouveau_vram_manager_del,

	nouveau_vram_manager_debug

	nouveau_vram_manager_debug

};

};



static int

nouveau_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)

{

	return 0;

}



static int

nouveau_gart_manager_fini(struct ttm_mem_type_manager *man)

{

	return 0;

}



static void

nouveau_gart_manager_del(struct ttm_mem_type_manager *man,

			 struct ttm_mem_reg *mem)

{

	struct nouveau_mem *node = mem->mm_node;



	if (node->tmp_vma.node) {

		nouveau_vm_unmap(&node->tmp_vma);

		nouveau_vm_put(&node->tmp_vma);

	}

	mem->mm_node = NULL;

}



static int

nouveau_gart_manager_new(struct ttm_mem_type_manager *man,

			 struct ttm_buffer_object *bo,

			 struct ttm_placement *placement,

			 struct ttm_mem_reg *mem)

{

	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);

	struct nouveau_bo *nvbo = nouveau_bo(bo);

	struct nouveau_vma *vma = &nvbo->vma;

	struct nouveau_vm *vm = vma->vm;

	struct nouveau_mem *node;

	int ret;



	if (unlikely((mem->num_pages << PAGE_SHIFT) >=

		     dev_priv->gart_info.aper_size))

		return -ENOMEM;



	node = kzalloc(sizeof(*node), GFP_KERNEL);

	if (!node)

		return -ENOMEM;



	/* This node must be for evicting large-paged VRAM

	 * to system memory.  Due to a nv50 limitation of

	 * not being able to mix large/small pages within

	 * the same PDE, we need to create a temporary

	 * small-paged VMA for the eviction.

	 */

	if (vma->node->type != vm->spg_shift) {

		ret = nouveau_vm_get(vm, (u64)vma->node->length << 12,

				     vm->spg_shift, NV_MEM_ACCESS_RW,

				     &node->tmp_vma);

		if (ret) {

			kfree(node);

			return ret;

		}

	}



	node->page_shift = nvbo->vma.node->type;

	mem->mm_node = node;

	mem->start   = 0;

	return 0;

}



void

nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)

{

}



const struct ttm_mem_type_manager_func nouveau_gart_manager = {

	nouveau_gart_manager_init,

	nouveau_gart_manager_fini,

	nouveau_gart_manager_new,

	nouveau_gart_manager_del,

	nouveau_gart_manager_debug

};

drivers/gpu/drm/nouveau/nouveau_notifier.c

+11 −5
Original line number Original line Diff line number Diff line
@@ -98,6 +98,7 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle, 
		       int size, uint32_t *b_offset)

		       int size, uint32_t *b_offset)

{

{

	struct drm_device *dev = chan->dev;

	struct drm_device *dev = chan->dev;

	struct drm_nouveau_private *dev_priv = dev->dev_private;

	struct nouveau_gpuobj *nobj = NULL;

	struct nouveau_gpuobj *nobj = NULL;

	struct drm_mm_node *mem;

	struct drm_mm_node *mem;

	uint32_t offset;

	uint32_t offset;
@@ -111,11 +112,16 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle, 
		return -ENOMEM;

		return -ENOMEM;

	}

	}





	if (dev_priv->card_type < NV_50) {

		if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM)

		if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM)

			target = NV_MEM_TARGET_VRAM;

			target = NV_MEM_TARGET_VRAM;

		else

		else

			target = NV_MEM_TARGET_GART;

			target = NV_MEM_TARGET_GART;

		offset  = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;

		offset  = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;

	} else {

		target = NV_MEM_TARGET_VM;

		offset = chan->notifier_bo->vma.offset;

	}

	offset += mem->start;

	offset += mem->start;





	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset,

	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset,

drivers/gpu/drm/nouveau/nouveau_sgdma.c

+9 −21
Original line number Original line Diff line number Diff line
@@ -375,12 +375,10 @@ static int 
nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)

nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)

{

{

	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;

	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;

	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;

	struct nouveau_mem *node = mem->mm_node;



	/* noop: bound in move_notify() */

	nvbe->offset = mem->start << PAGE_SHIFT;

	node->pages = nvbe->pages;



	nvbe->pages = (dma_addr_t *)node;

	nouveau_vm_map_sg(&dev_priv->gart_info.vma, nvbe->offset,

			  nvbe->nr_pages << PAGE_SHIFT, nvbe->pages);

	nvbe->bound = true;

	nvbe->bound = true;

	return 0;

	return 0;

}

}
@@ -389,13 +387,10 @@ static int 
nv50_sgdma_unbind(struct ttm_backend *be)

nv50_sgdma_unbind(struct ttm_backend *be)

{

{

	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;

	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;

	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;

	struct nouveau_mem *node = (struct nouveau_mem *)nvbe->pages;



	/* noop: unbound in move_notify() */

	if (!nvbe->bound)

	nvbe->pages = node->pages;

		return 0;

	node->pages = NULL;



	nouveau_vm_unmap_at(&dev_priv->gart_info.vma, nvbe->offset,

			    nvbe->nr_pages << PAGE_SHIFT);

	nvbe->bound = false;

	nvbe->bound = false;

	return 0;

	return 0;

}

}
@@ -457,13 +452,7 @@ nouveau_sgdma_init(struct drm_device *dev) 
	}

	}





	if (dev_priv->card_type >= NV_50) {

	if (dev_priv->card_type >= NV_50) {

		ret = nouveau_vm_get(dev_priv->chan_vm, aper_size,

		dev_priv->gart_info.aper_base = 0;

				     12, NV_MEM_ACCESS_RW,

				     &dev_priv->gart_info.vma);

		if (ret)

			return ret;



		dev_priv->gart_info.aper_base = dev_priv->gart_info.vma.offset;

		dev_priv->gart_info.aper_size = aper_size;

		dev_priv->gart_info.aper_size = aper_size;

		dev_priv->gart_info.type = NOUVEAU_GART_HW;

		dev_priv->gart_info.type = NOUVEAU_GART_HW;

		dev_priv->gart_info.func = &nv50_sgdma_backend;

		dev_priv->gart_info.func = &nv50_sgdma_backend;
@@ -522,7 +511,6 @@ nouveau_sgdma_takedown(struct drm_device *dev) 
	struct drm_nouveau_private *dev_priv = dev->dev_private;

	struct drm_nouveau_private *dev_priv = dev->dev_private;





	nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);

	nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);

	nouveau_vm_put(&dev_priv->gart_info.vma);





	if (dev_priv->gart_info.dummy.page) {

	if (dev_priv->gart_info.dummy.page) {

		pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,

		pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,