Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 32faa34d authored by Dave Airlie's avatar Dave Airlie
Browse files

drm/Merge branch 'drm-ttm-glisse' of ../drm-radeon-next into drm-core-next

Merge topic branch containing Jerome's TTM changes, contains one change from
Konrad to swiotlb export.

* 'drm-ttm-glisse' of ../drm-radeon-next:
  drm/ttm: callback move_notify any time bo placement change v4
  drm/ttm: simplify memory accounting for ttm user v2
  drm/ttm: isolate dma data from ttm_tt V4
  drm/nouveau: enable the ttm dma pool when swiotlb is active V3
  drm/radeon/kms: enable the ttm dma pool if swiotlb is on V4
  drm/ttm: provide dma aware ttm page pool code V9
  drm/ttm: introduce callback for ttm_tt populate & unpopulate V4
  drm/ttm: merge ttm_backend and ttm_tt V5
  drm/ttm: page allocation use page array instead of list
  drm/ttm: test for dma_address array allocation failure
  drm/ttm: use ttm put pages function to properly restore cache attribute
  drm/ttm: remove unused backend flags field
  drm/ttm: remove split btw highmen and lowmem page
  drm/ttm: remove userspace backed ttm object support
  swiotlb: Expose swiotlb_nr_tlb function to modules
parents bcdd6b2f dc97b340
Loading
Loading
Loading
Loading
+90 −8
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@
 */

#include "drmP.h"
#include "ttm/ttm_page_alloc.h"

#include "nouveau_drm.h"
#include "nouveau_drv.h"
@@ -92,6 +93,7 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_bo *nvbo;
	size_t acc_size;
	int ret;

	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
@@ -114,9 +116,12 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
	nouveau_bo_placement_set(nvbo, flags, 0);

	acc_size = ttm_bo_dma_acc_size(&dev_priv->ttm.bdev, size,
				       sizeof(struct nouveau_bo));

	ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
			  ttm_bo_type_device, &nvbo->placement,
			  align >> PAGE_SHIFT, 0, false, NULL, size,
			  align >> PAGE_SHIFT, 0, false, NULL, acc_size,
			  nouveau_bo_del_ttm);
	if (ret) {
		/* ttm will call nouveau_bo_del_ttm if it fails.. */
@@ -343,8 +348,10 @@ nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
		*mem = val;
}

static struct ttm_backend *
nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
static struct ttm_tt *
nouveau_ttm_tt_create(struct ttm_bo_device *bdev,
		      unsigned long size, uint32_t page_flags,
		      struct page *dummy_read_page)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct drm_device *dev = dev_priv->dev;
@@ -352,11 +359,13 @@ nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
	switch (dev_priv->gart_info.type) {
#if __OS_HAS_AGP
	case NOUVEAU_GART_AGP:
		return ttm_agp_backend_init(bdev, dev->agp->bridge);
		return ttm_agp_tt_create(bdev, dev->agp->bridge,
					 size, page_flags, dummy_read_page);
#endif
	case NOUVEAU_GART_PDMA:
	case NOUVEAU_GART_HW:
		return nouveau_sgdma_init_ttm(dev);
		return nouveau_sgdma_create_ttm(bdev, size, page_flags,
						dummy_read_page);
	default:
		NV_ERROR(dev, "Unknown GART type %d\n",
			 dev_priv->gart_info.type);
@@ -806,10 +815,10 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
	struct nouveau_vma *vma;

	list_for_each_entry(vma, &nvbo->vma_list, head) {
		if (new_mem->mem_type == TTM_PL_VRAM) {
		if (new_mem && new_mem->mem_type == TTM_PL_VRAM) {
			nouveau_vm_map(vma, new_mem->mm_node);
		} else
		if (new_mem->mem_type == TTM_PL_TT &&
		if (new_mem && new_mem->mem_type == TTM_PL_TT &&
		    nvbo->page_shift == vma->vm->spg_shift) {
			nouveau_vm_map_sg(vma, 0, new_mem->
					  num_pages << PAGE_SHIFT,
@@ -1044,8 +1053,81 @@ nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
	nouveau_fence_unref(&old_fence);
}

static int
nouveau_ttm_tt_populate(struct ttm_tt *ttm)
{
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
	struct drm_nouveau_private *dev_priv;
	struct drm_device *dev;
	unsigned i;
	int r;

	if (ttm->state != tt_unpopulated)
		return 0;

	dev_priv = nouveau_bdev(ttm->bdev);
	dev = dev_priv->dev;

#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
		return ttm_dma_populate((void *)ttm, dev->dev);
	}
#endif

	r = ttm_pool_populate(ttm);
	if (r) {
		return r;
	}

	for (i = 0; i < ttm->num_pages; i++) {
		ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i],
						   0, PAGE_SIZE,
						   PCI_DMA_BIDIRECTIONAL);
		if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) {
			while (--i) {
				pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
					       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
				ttm_dma->dma_address[i] = 0;
			}
			ttm_pool_unpopulate(ttm);
			return -EFAULT;
		}
	}
	return 0;
}

static void
nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
	struct drm_nouveau_private *dev_priv;
	struct drm_device *dev;
	unsigned i;

	dev_priv = nouveau_bdev(ttm->bdev);
	dev = dev_priv->dev;

#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
		ttm_dma_unpopulate((void *)ttm, dev->dev);
		return;
	}
#endif

	for (i = 0; i < ttm->num_pages; i++) {
		if (ttm_dma->dma_address[i]) {
			pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
				       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
		}
	}

	ttm_pool_unpopulate(ttm);
}

struct ttm_bo_driver nouveau_bo_driver = {
	.create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry,
	.ttm_tt_create = &nouveau_ttm_tt_create,
	.ttm_tt_populate = &nouveau_ttm_tt_populate,
	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
	.invalidate_caches = nouveau_bo_invalidate_caches,
	.init_mem_type = nouveau_bo_init_mem_type,
	.evict_flags = nouveau_bo_evict_flags,
+1 −0
Original line number Diff line number Diff line
@@ -178,6 +178,7 @@ static struct drm_info_list nouveau_debugfs_list[] = {
	{ "memory", nouveau_debugfs_memory_info, 0, NULL },
	{ "vbios.rom", nouveau_debugfs_vbios_image, 0, NULL },
	{ "ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL },
	{ "ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL },
};
#define NOUVEAU_DEBUGFS_ENTRIES ARRAY_SIZE(nouveau_debugfs_list)

+4 −1
Original line number Diff line number Diff line
@@ -1000,7 +1000,10 @@ extern int nouveau_sgdma_init(struct drm_device *);
extern void nouveau_sgdma_takedown(struct drm_device *);
extern uint32_t nouveau_sgdma_get_physical(struct drm_device *,
					   uint32_t offset);
extern struct ttm_backend *nouveau_sgdma_init_ttm(struct drm_device *);
extern struct ttm_tt *nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
					       unsigned long size,
					       uint32_t page_flags,
					       struct page *dummy_read_page);

/* nouveau_debugfs.c */
#if defined(CONFIG_DRM_NOUVEAU_DEBUG)
+6 −0
Original line number Diff line number Diff line
@@ -407,6 +407,12 @@ nouveau_mem_vram_init(struct drm_device *dev)
	ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(dma_bits));
	if (ret)
		return ret;
	ret = pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(dma_bits));
	if (ret) {
		/* Reset to default value. */
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(32));
	}


	ret = nouveau_ttm_global_init(dev_priv);
	if (ret)
+53 −123
Original line number Diff line number Diff line
@@ -8,88 +8,30 @@
#define NV_CTXDMA_PAGE_MASK  (NV_CTXDMA_PAGE_SIZE - 1)

struct nouveau_sgdma_be {
	struct ttm_backend backend;
	/* this has to be the first field so populate/unpopulated in
	 * nouve_bo.c works properly, otherwise have to move them here
	 */
	struct ttm_dma_tt ttm;
	struct drm_device *dev;

	dma_addr_t *pages;
	unsigned nr_pages;
	bool unmap_pages;

	u64 offset;
	bool bound;
};

static int
nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
		       struct page **pages, struct page *dummy_read_page,
		       dma_addr_t *dma_addrs)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct drm_device *dev = nvbe->dev;
	int i;

	NV_DEBUG(nvbe->dev, "num_pages = %ld\n", num_pages);

	nvbe->pages = dma_addrs;
	nvbe->nr_pages = num_pages;
	nvbe->unmap_pages = true;

	/* this code path isn't called and is incorrect anyways */
	if (0) { /* dma_addrs[0] != DMA_ERROR_CODE) { */
		nvbe->unmap_pages = false;
		return 0;
	}

	for (i = 0; i < num_pages; i++) {
		nvbe->pages[i] = pci_map_page(dev->pdev, pages[i], 0,
					      PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
		if (pci_dma_mapping_error(dev->pdev, nvbe->pages[i])) {
			nvbe->nr_pages = --i;
			be->func->clear(be);
			return -EFAULT;
		}
	}

	return 0;
}

static void
nouveau_sgdma_clear(struct ttm_backend *be)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct drm_device *dev = nvbe->dev;

	if (nvbe->bound)
		be->func->unbind(be);

	if (nvbe->unmap_pages) {
		while (nvbe->nr_pages--) {
			pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
				       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
		}
	}
}

static void
nouveau_sgdma_destroy(struct ttm_backend *be)
nouveau_sgdma_destroy(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;

	if (be) {
	if (ttm) {
		NV_DEBUG(nvbe->dev, "\n");

		if (nvbe) {
			if (nvbe->pages)
				be->func->clear(be);
		ttm_dma_tt_fini(&nvbe->ttm);
		kfree(nvbe);
	}
}
}

static int
nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
nv04_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_device *dev = nvbe->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
@@ -99,8 +41,8 @@ nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)

	nvbe->offset = mem->start << PAGE_SHIFT;
	pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
	for (i = 0; i < nvbe->nr_pages; i++) {
		dma_addr_t dma_offset = nvbe->pages[i];
	for (i = 0; i < ttm->num_pages; i++) {
		dma_addr_t dma_offset = nvbe->ttm.dma_address[i];
		uint32_t offset_l = lower_32_bits(dma_offset);

		for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
@@ -109,14 +51,13 @@ nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
		}
	}

	nvbe->bound = true;
	return 0;
}

static int
nv04_sgdma_unbind(struct ttm_backend *be)
nv04_sgdma_unbind(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_device *dev = nvbe->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
@@ -124,22 +65,19 @@ nv04_sgdma_unbind(struct ttm_backend *be)

	NV_DEBUG(dev, "\n");

	if (!nvbe->bound)
	if (ttm->state != tt_bound)
		return 0;

	pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
	for (i = 0; i < nvbe->nr_pages; i++) {
	for (i = 0; i < ttm->num_pages; i++) {
		for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
			nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
	}

	nvbe->bound = false;
	return 0;
}

static struct ttm_backend_func nv04_sgdma_backend = {
	.populate		= nouveau_sgdma_populate,
	.clear			= nouveau_sgdma_clear,
	.bind			= nv04_sgdma_bind,
	.unbind			= nv04_sgdma_unbind,
	.destroy		= nouveau_sgdma_destroy
@@ -158,14 +96,14 @@ nv41_sgdma_flush(struct nouveau_sgdma_be *nvbe)
}

static int
nv41_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
nv41_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
	dma_addr_t *list = nvbe->pages;
	dma_addr_t *list = nvbe->ttm.dma_address;
	u32 pte = mem->start << 2;
	u32 cnt = nvbe->nr_pages;
	u32 cnt = ttm->num_pages;

	nvbe->offset = mem->start << PAGE_SHIFT;

@@ -175,18 +113,17 @@ nv41_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
	}

	nv41_sgdma_flush(nvbe);
	nvbe->bound = true;
	return 0;
}

static int
nv41_sgdma_unbind(struct ttm_backend *be)
nv41_sgdma_unbind(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
	u32 pte = (nvbe->offset >> 12) << 2;
	u32 cnt = nvbe->nr_pages;
	u32 cnt = ttm->num_pages;

	while (cnt--) {
		nv_wo32(pgt, pte, 0x00000000);
@@ -194,24 +131,22 @@ nv41_sgdma_unbind(struct ttm_backend *be)
	}

	nv41_sgdma_flush(nvbe);
	nvbe->bound = false;
	return 0;
}

static struct ttm_backend_func nv41_sgdma_backend = {
	.populate		= nouveau_sgdma_populate,
	.clear			= nouveau_sgdma_clear,
	.bind			= nv41_sgdma_bind,
	.unbind			= nv41_sgdma_unbind,
	.destroy		= nouveau_sgdma_destroy
};

static void
nv44_sgdma_flush(struct nouveau_sgdma_be *nvbe)
nv44_sgdma_flush(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_device *dev = nvbe->dev;

	nv_wr32(dev, 0x100814, (nvbe->nr_pages - 1) << 12);
	nv_wr32(dev, 0x100814, (ttm->num_pages - 1) << 12);
	nv_wr32(dev, 0x100808, nvbe->offset | 0x20);
	if (!nv_wait(dev, 0x100808, 0x00000001, 0x00000001))
		NV_ERROR(dev, "gart flush timeout: 0x%08x\n",
@@ -270,14 +205,14 @@ nv44_sgdma_fill(struct nouveau_gpuobj *pgt, dma_addr_t *list, u32 base, u32 cnt)
}

static int
nv44_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
nv44_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
	dma_addr_t *list = nvbe->pages;
	dma_addr_t *list = nvbe->ttm.dma_address;
	u32 pte = mem->start << 2, tmp[4];
	u32 cnt = nvbe->nr_pages;
	u32 cnt = ttm->num_pages;
	int i;

	nvbe->offset = mem->start << PAGE_SHIFT;
@@ -305,19 +240,18 @@ nv44_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
	if (cnt)
		nv44_sgdma_fill(pgt, list, pte, cnt);

	nv44_sgdma_flush(nvbe);
	nvbe->bound = true;
	nv44_sgdma_flush(ttm);
	return 0;
}

static int
nv44_sgdma_unbind(struct ttm_backend *be)
nv44_sgdma_unbind(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
	u32 pte = (nvbe->offset >> 12) << 2;
	u32 cnt = nvbe->nr_pages;
	u32 cnt = ttm->num_pages;

	if (pte & 0x0000000c) {
		u32  max = 4 - ((pte >> 2) & 0x3);
@@ -339,55 +273,47 @@ nv44_sgdma_unbind(struct ttm_backend *be)
	if (cnt)
		nv44_sgdma_fill(pgt, NULL, pte, cnt);

	nv44_sgdma_flush(nvbe);
	nvbe->bound = false;
	nv44_sgdma_flush(ttm);
	return 0;
}

static struct ttm_backend_func nv44_sgdma_backend = {
	.populate		= nouveau_sgdma_populate,
	.clear			= nouveau_sgdma_clear,
	.bind			= nv44_sgdma_bind,
	.unbind			= nv44_sgdma_unbind,
	.destroy		= nouveau_sgdma_destroy
};

static int
nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
nv50_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
	struct nouveau_mem *node = mem->mm_node;

	/* noop: bound in move_notify() */
	node->pages = nvbe->pages;
	nvbe->pages = (dma_addr_t *)node;
	nvbe->bound = true;
	node->pages = nvbe->ttm.dma_address;
	return 0;
}

static int
nv50_sgdma_unbind(struct ttm_backend *be)
nv50_sgdma_unbind(struct ttm_tt *ttm)
{
	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
	struct nouveau_mem *node = (struct nouveau_mem *)nvbe->pages;
	/* noop: unbound in move_notify() */
	nvbe->pages = node->pages;
	node->pages = NULL;
	nvbe->bound = false;
	return 0;
}

static struct ttm_backend_func nv50_sgdma_backend = {
	.populate		= nouveau_sgdma_populate,
	.clear			= nouveau_sgdma_clear,
	.bind			= nv50_sgdma_bind,
	.unbind			= nv50_sgdma_unbind,
	.destroy		= nouveau_sgdma_destroy
};

struct ttm_backend *
nouveau_sgdma_init_ttm(struct drm_device *dev)
struct ttm_tt *
nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
			 unsigned long size, uint32_t page_flags,
			 struct page *dummy_read_page)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct drm_device *dev = dev_priv->dev;
	struct nouveau_sgdma_be *nvbe;

	nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
@@ -395,9 +321,13 @@ nouveau_sgdma_init_ttm(struct drm_device *dev)
		return NULL;

	nvbe->dev = dev;
	nvbe->ttm.ttm.func = dev_priv->gart_info.func;

	nvbe->backend.func = dev_priv->gart_info.func;
	return &nvbe->backend;
	if (ttm_dma_tt_init(&nvbe->ttm, bdev, size, page_flags, dummy_read_page)) {
		kfree(nvbe);
		return NULL;
	}
	return &nvbe->ttm.ttm;
}

int
Loading