Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b5b7bef9 authored by Mika Kuoppala's avatar Mika Kuoppala
Browse files

drm/i915/gtt: Use a common type for page directories 



All page directories are identical in function, only the position in the
hierarchy differ. Use same base type for directory functionality.

v2: cleanup, size always 512, init to null

Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Matthew Auld <matthew.william.auld@gmail.com>
Cc: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
Signed-off-by: default avatarMika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190614164350.30415-2-mika.kuoppala@linux.intel.com
parent 7d82cc35

drivers/gpu/drm/i915/gem/i915_gem_context.c

+1 −1
Original line number Diff line number Diff line
@@ -1027,7 +1027,7 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data) 


	if (i915_vm_is_4lvl(vm)) {

		struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

		const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4);

		const dma_addr_t pd_daddr = px_dma(ppgtt->pd);



		cs = intel_ring_begin(rq, 6);

		if (IS_ERR(cs))

drivers/gpu/drm/i915/gt/intel_lrc_reg.h

+1 −1
Original line number Diff line number Diff line
@@ -55,7 +55,7 @@ 


#define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \

	u32 *reg_state__ = (reg_state); \

	const u64 addr__ = px_dma(&ppgtt->pml4); \

	const u64 addr__ = px_dma(ppgtt->pd); \

	(reg_state__)[CTX_PDP0_UDW + 1] = upper_32_bits(addr__); \

	(reg_state__)[CTX_PDP0_LDW + 1] = lower_32_bits(addr__); \

} while (0)

drivers/gpu/drm/i915/gt/intel_ringbuffer.c

+1 −1
Original line number Diff line number Diff line
@@ -1484,7 +1484,7 @@ static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt) 


	*cs++ = MI_LOAD_REGISTER_IMM(1);

	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));

	*cs++ = ppgtt->pd.base.ggtt_offset << 10;

	*cs++ = ppgtt->pd->base.ggtt_offset << 10;



	intel_ring_advance(rq, cs);

drivers/gpu/drm/i915/gvt/scheduler.c

+19 −11
Original line number Diff line number Diff line
@@ -375,11 +375,13 @@ static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, 
		return -EINVAL;



	if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {

		px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0];

		px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0];

	} else {

		for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {

			px_dma(ppgtt->pdp.page_directory[i]) =

				mm->ppgtt_mm.shadow_pdps[i];

			struct i915_page_directory * const pd =

				i915_pd_entry(ppgtt->pd, i);



			px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i];

		}

	}
@@ -1107,11 +1109,14 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s, 
	int i;



	if (i915_vm_is_4lvl(&ppgtt->vm)) {

		px_dma(&ppgtt->pml4) = s->i915_context_pml4;

		px_dma(ppgtt->pd) = s->i915_context_pml4;

	} else {

		for (i = 0; i < GEN8_3LVL_PDPES; i++)

			px_dma(ppgtt->pdp.page_directory[i]) =

				s->i915_context_pdps[i];

		for (i = 0; i < GEN8_3LVL_PDPES; i++) {

			struct i915_page_directory * const pd =

				i915_pd_entry(ppgtt->pd, i);



			px_dma(pd) = s->i915_context_pdps[i];

		}

	}

}
@@ -1165,11 +1170,14 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s, 
	int i;



	if (i915_vm_is_4lvl(&ppgtt->vm)) {

		s->i915_context_pml4 = px_dma(&ppgtt->pml4);

		s->i915_context_pml4 = px_dma(ppgtt->pd);

	} else {

		for (i = 0; i < GEN8_3LVL_PDPES; i++)

			s->i915_context_pdps[i] =

				px_dma(ppgtt->pdp.page_directory[i]);

		for (i = 0; i < GEN8_3LVL_PDPES; i++) {

			struct i915_page_directory * const pd =

				i915_pd_entry(ppgtt->pd, i);



			s->i915_context_pdps[i] = px_dma(pd);

		}

	}

}

drivers/gpu/drm/i915/i915_gem_gtt.c

+179 −172
Original line number Diff line number Diff line
@@ -662,7 +662,8 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm) 
		return ERR_PTR(-ENOMEM);

	}



	atomic_set(&pt->used_ptes, 0);

	atomic_set(&pt->used, 0);



	return pt;

}
@@ -684,11 +685,30 @@ static void gen6_initialize_pt(struct i915_address_space *vm, 
	fill32_px(vm, pt, vm->scratch_pte);

}



static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)

static struct i915_page_directory *__alloc_pd(void)

{

	struct i915_page_directory *pd;



	pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);



	if (unlikely(!pd))

		return NULL;



	memset(&pd->base, 0, sizeof(pd->base));

	atomic_set(&pd->used, 0);

	spin_lock_init(&pd->lock);



	/* for safety */

	pd->entry[0] = NULL;



	return pd;

}



static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)

{

	struct i915_page_directory *pd;



	pd = __alloc_pd();

	if (unlikely(!pd))

		return ERR_PTR(-ENOMEM);
@@ -697,8 +717,6 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) 
		return ERR_PTR(-ENOMEM);

	}



	atomic_set(&pd->used_pdes, 0);

	spin_lock_init(&pd->lock);

	return pd;

}
@@ -714,88 +732,56 @@ static void gen8_initialize_pd(struct i915_address_space *vm, 
{

	fill_px(vm, pd,

		gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));

	memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);

}



static int __pdp_init(struct i915_address_space *vm,

		      struct i915_page_directory_pointer *pdp)

{

	const unsigned int pdpes = i915_pdpes_per_pdp(vm);



	pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),

					    I915_GFP_ALLOW_FAIL);

	if (unlikely(!pdp->page_directory))

		return -ENOMEM;



	memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);



	atomic_set(&pdp->used_pdpes, 0);

	spin_lock_init(&pdp->lock);

	return 0;

	memset_p(pd->entry, vm->scratch_pt, I915_PDES);

}



static void __pdp_fini(struct i915_page_directory_pointer *pdp)

static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm)

{

	kfree(pdp->page_directory);

	pdp->page_directory = NULL;

}



static struct i915_page_directory_pointer *

alloc_pdp(struct i915_address_space *vm)

{

	struct i915_page_directory_pointer *pdp;

	int ret = -ENOMEM;



	GEM_BUG_ON(!i915_vm_is_4lvl(vm));

	struct i915_page_directory *pdp;



	pdp = kmalloc(sizeof(*pdp), I915_GFP_ALLOW_FAIL);

	pdp = __alloc_pd();

	if (!pdp)

		return ERR_PTR(-ENOMEM);



	ret = __pdp_init(vm, pdp);

	if (ret)

		goto fail_bitmap;



	ret = setup_px(vm, pdp);

	if (ret)

		goto fail_page_m;



	return pdp;



fail_page_m:

	__pdp_fini(pdp);

fail_bitmap:

	if (i915_vm_is_4lvl(vm)) {

		if (unlikely(setup_px(vm, pdp))) {

			kfree(pdp);

			return ERR_PTR(-ENOMEM);

		}

	}



	return ERR_PTR(ret);

	return pdp;

}



static void free_pdp(struct i915_address_space *vm,

		     struct i915_page_directory_pointer *pdp)

		     struct i915_page_directory *pdp)

{

	__pdp_fini(pdp);



	if (!i915_vm_is_4lvl(vm))

		return;



	if (i915_vm_is_4lvl(vm))

		cleanup_px(vm, pdp);



	kfree(pdp);

}



static void gen8_initialize_pdp(struct i915_address_space *vm,

				struct i915_page_directory_pointer *pdp)

static void gen8_initialize_4lvl_pdp(struct i915_address_space *vm,

				     struct i915_page_directory *pdp)

{

	fill_px(vm, pdp,

		gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC));

	memset_p(pdp->entry, vm->scratch_pd, 512);

}



static void gen8_initialize_3lvl_pdp(struct i915_address_space *vm,

				     struct i915_page_directory *pdp)

{

	memset_p(pdp->entry, vm->scratch_pd, GEN8_3LVL_PDPES);

}



static void gen8_initialize_pml4(struct i915_address_space *vm,

				 struct i915_pml4 *pml4)

				 struct i915_page_directory *pml4)

{

	fill_px(vm, pml4,

		gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));

	memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);

	spin_lock_init(&pml4->lock);

	memset_p(pml4->entry, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);

}



/*
@@ -823,8 +809,8 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm, 
	memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);

	kunmap_atomic(vaddr);



	GEM_BUG_ON(num_entries > atomic_read(&pt->used_ptes));

	return !atomic_sub_return(num_entries, &pt->used_ptes);

	GEM_BUG_ON(num_entries > atomic_read(&pt->used));

	return !atomic_sub_return(num_entries, &pt->used);

}



static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
@@ -855,12 +841,12 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, 
			continue;



		spin_lock(&pd->lock);

		if (!atomic_read(&pt->used_ptes)) {

		if (!atomic_read(&pt->used)) {

			gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);

			pd->page_table[pde] = vm->scratch_pt;

			pd->entry[pde] = vm->scratch_pt;



			GEM_BUG_ON(!atomic_read(&pd->used_pdes));

			atomic_dec(&pd->used_pdes);

			GEM_BUG_ON(!atomic_read(&pd->used));

			atomic_dec(&pd->used);

			free = true;

		}

		spin_unlock(&pd->lock);
@@ -868,11 +854,11 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, 
			free_pt(vm, pt);

	}



	return !atomic_read(&pd->used_pdes);

	return !atomic_read(&pd->used);

}



static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,

				struct i915_page_directory_pointer *pdp,

				struct i915_page_directory *pdp,

				struct i915_page_directory *pd,

				unsigned int pdpe)

{
@@ -890,7 +876,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, 
 * Caller can use the return value to update higher-level entries

 */

static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,

				 struct i915_page_directory_pointer *pdp,

				 struct i915_page_directory * const pdp,

				 u64 start, u64 length)

{

	struct i915_page_directory *pd;
@@ -905,12 +891,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, 
			continue;



		spin_lock(&pdp->lock);

		if (!atomic_read(&pd->used_pdes)) {

		if (!atomic_read(&pd->used)) {

			gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);

			pdp->page_directory[pdpe] = vm->scratch_pd;

			pdp->entry[pdpe] = vm->scratch_pd;



			GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));

			atomic_dec(&pdp->used_pdpes);

			GEM_BUG_ON(!atomic_read(&pdp->used));

			atomic_dec(&pdp->used);

			free = true;

		}

		spin_unlock(&pdp->lock);
@@ -918,17 +904,17 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, 
			free_pd(vm, pd);

	}



	return !atomic_read(&pdp->used_pdpes);

	return !atomic_read(&pdp->used);

}



static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,

				  u64 start, u64 length)

{

	gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);

	gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);

}



static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,

				 struct i915_page_directory_pointer *pdp,

static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4,

				 struct i915_page_directory *pdp,

				 unsigned int pml4e)

{

	gen8_ppgtt_pml4e_t *vaddr;
@@ -946,8 +932,8 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, 
				  u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_pml4 *pml4 = &ppgtt->pml4;

	struct i915_page_directory_pointer *pdp;

	struct i915_page_directory * const pml4 = ppgtt->pd;

	struct i915_page_directory *pdp;

	unsigned int pml4e;



	GEM_BUG_ON(!i915_vm_is_4lvl(vm));
@@ -960,9 +946,9 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, 
			continue;



		spin_lock(&pml4->lock);

		if (!atomic_read(&pdp->used_pdpes)) {

		if (!atomic_read(&pdp->used)) {

			gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);

			pml4->pdps[pml4e] = vm->scratch_pdp;

			pml4->entry[pml4e] = vm->scratch_pdp;

			free = true;

		}

		spin_unlock(&pml4->lock);
@@ -999,7 +985,7 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start) 


static __always_inline bool

gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,

			      struct i915_page_directory_pointer *pdp,

			      struct i915_page_directory *pdp,

			      struct sgt_dma *iter,

			      struct gen8_insert_pte *idx,

			      enum i915_cache_level cache_level,
@@ -1011,8 +997,8 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, 
	bool ret;



	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

	pd = pdp->page_directory[idx->pdpe];

	vaddr = kmap_atomic_px(pd->page_table[idx->pde]);

	pd = i915_pd_entry(pdp, idx->pdpe);

	vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

	do {

		vaddr[idx->pte] = pte_encode | iter->dma;
@@ -1042,11 +1028,11 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, 
				}



				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

				pd = pdp->page_directory[idx->pdpe];

				pd = pdp->entry[idx->pdpe];

			}



			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(pd->page_table[idx->pde]);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

		}

	} while (1);

	kunmap_atomic(vaddr);
@@ -1063,14 +1049,14 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm, 
	struct sgt_dma iter = sgt_dma(vma);

	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);



	gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,

	gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,

				      cache_level, flags);



	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;

}



static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,

					   struct i915_page_directory_pointer **pdps,

					   struct i915_page_directory *pml4,

					   struct sgt_dma *iter,

					   enum i915_cache_level cache_level,

					   u32 flags)
@@ -1081,8 +1067,9 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, 


	do {

		struct gen8_insert_pte idx = gen8_insert_pte(start);

		struct i915_page_directory_pointer *pdp = pdps[idx.pml4e];

		struct i915_page_directory *pd = pdp->page_directory[idx.pdpe];

		struct i915_page_directory *pdp =

			i915_pdp_entry(pml4, idx.pml4e);

		struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);

		unsigned int page_size;

		bool maybe_64K = false;

		gen8_pte_t encode = pte_encode;
@@ -1100,7 +1087,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, 


			vaddr = kmap_atomic_px(pd);

		} else {

			struct i915_page_table *pt = pd->page_table[idx.pde];

			struct i915_page_table *pt = i915_pt_entry(pd, idx.pde);



			index = idx.pte;

			max = GEN8_PTES;
@@ -1175,7 +1162,8 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, 
				u16 i;



				encode = vma->vm->scratch_pte;

				vaddr = kmap_atomic_px(pd->page_table[idx.pde]);

				vaddr = kmap_atomic_px(i915_pt_entry(pd,

								     idx.pde));



				for (i = 1; i < index; i += 16)

					memset64(vaddr + i, encode, 15);
@@ -1195,15 +1183,16 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, 
{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct sgt_dma iter = sgt_dma(vma);

	struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;

	struct i915_page_directory * const pml4 = ppgtt->pd;



	if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {

		gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level,

		gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level,

					       flags);

	} else {

		struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);



		while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],

		while (gen8_ppgtt_insert_pte_entries(ppgtt,

						     i915_pdp_entry(pml4, idx.pml4e++),

						     &iter, &idx, cache_level,

						     flags))

			GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
@@ -1218,8 +1207,8 @@ static void gen8_free_page_tables(struct i915_address_space *vm, 
	int i;



	for (i = 0; i < I915_PDES; i++) {

		if (pd->page_table[i] != vm->scratch_pt)

			free_pt(vm, pd->page_table[i]);

		if (pd->entry[i] != vm->scratch_pt)

			free_pt(vm, pd->entry[i]);

	}

}
@@ -1278,7 +1267,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) 
	gen8_initialize_pt(vm, vm->scratch_pt);

	gen8_initialize_pd(vm, vm->scratch_pd);

	if (i915_vm_is_4lvl(vm))

		gen8_initialize_pdp(vm, vm->scratch_pdp);

		gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp);



	return 0;
@@ -1300,7 +1289,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create) 
	int i;



	if (i915_vm_is_4lvl(vm)) {

		const u64 daddr = px_dma(&ppgtt->pml4);

		const u64 daddr = px_dma(ppgtt->pd);



		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));

		I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
@@ -1337,17 +1326,17 @@ static void gen8_free_scratch(struct i915_address_space *vm) 
}



static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,

				    struct i915_page_directory_pointer *pdp)

				    struct i915_page_directory *pdp)

{

	const unsigned int pdpes = i915_pdpes_per_pdp(vm);

	int i;



	for (i = 0; i < pdpes; i++) {

		if (pdp->page_directory[i] == vm->scratch_pd)

		if (pdp->entry[i] == vm->scratch_pd)

			continue;



		gen8_free_page_tables(vm, pdp->page_directory[i]);

		free_pd(vm, pdp->page_directory[i]);

		gen8_free_page_tables(vm, pdp->entry[i]);

		free_pd(vm, pdp->entry[i]);

	}



	free_pdp(vm, pdp);
@@ -1355,16 +1344,19 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, 


static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)

{

	struct i915_page_directory * const pml4 = ppgtt->pd;

	int i;



	for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {

		if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)

		struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);



		if (pdp == ppgtt->vm.scratch_pdp)

			continue;



		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);

		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);

	}



	cleanup_px(&ppgtt->vm, &ppgtt->pml4);

	free_pd(&ppgtt->vm, pml4);

}



static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
@@ -1378,7 +1370,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 
	if (i915_vm_is_4lvl(vm))

		gen8_ppgtt_cleanup_4lvl(ppgtt);

	else

		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);

		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);



	gen8_free_scratch(vm);

}
@@ -1407,10 +1399,10 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, 
			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))

				gen8_initialize_pt(vm, pt);



			old = cmpxchg(&pd->page_table[pde], vm->scratch_pt, pt);

			old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);

			if (old == vm->scratch_pt) {

				gen8_ppgtt_set_pde(vm, pd, pt, pde);

				atomic_inc(&pd->used_pdes);

				atomic_inc(&pd->used);

			} else {

				free_pt(vm, pt);

				pt = old;
@@ -1419,7 +1411,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, 
			spin_lock(&pd->lock);

		}



		atomic_add(count, &pt->used_ptes);

		atomic_add(count, &pt->used);

	}

	spin_unlock(&pd->lock);
@@ -1431,7 +1423,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, 
}



static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,

				struct i915_page_directory_pointer *pdp,

				struct i915_page_directory *pdp,

				u64 start, u64 length)

{

	struct i915_page_directory *pd;
@@ -1452,11 +1444,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, 


			gen8_initialize_pd(vm, pd);



			old = cmpxchg(&pdp->page_directory[pdpe],

				      vm->scratch_pd, pd);

			old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd);

			if (old == vm->scratch_pd) {

				gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);

				atomic_inc(&pdp->used_pdpes);

				atomic_inc(&pdp->used);

			} else {

				free_pd(vm, pd);

				pd = old;
@@ -1464,7 +1455,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, 


			spin_lock(&pdp->lock);

		}

		atomic_inc(&pd->used_pdes);

		atomic_inc(&pd->used);

		spin_unlock(&pdp->lock);



		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
@@ -1472,7 +1463,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, 
			goto unwind_pd;



		spin_lock(&pdp->lock);

		atomic_dec(&pd->used_pdes);

		atomic_dec(&pd->used);

	}

	spin_unlock(&pdp->lock);
@@ -1480,10 +1471,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, 


unwind_pd:

	spin_lock(&pdp->lock);

	if (atomic_dec_and_test(&pd->used_pdes)) {

	if (atomic_dec_and_test(&pd->used)) {

		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);

		GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));

		atomic_dec(&pdp->used_pdpes);

		GEM_BUG_ON(!atomic_read(&pdp->used));

		atomic_dec(&pdp->used);

		free_pd(vm, pd);

	}

	spin_unlock(&pdp->lock);
@@ -1496,23 +1487,24 @@ static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, 
				 u64 start, u64 length)

{

	return gen8_ppgtt_alloc_pdp(vm,

				    &i915_vm_to_ppgtt(vm)->pdp, start, length);

				    i915_vm_to_ppgtt(vm)->pd, start, length);

}



static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_pml4 *pml4 = &ppgtt->pml4;

	struct i915_page_directory_pointer *pdp;

	struct i915_page_directory * const pml4 = ppgtt->pd;

	struct i915_page_directory *pdp;

	u64 from = start;

	u32 pml4e;

	int ret;



	spin_lock(&pml4->lock);

	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {



		if (pdp == vm->scratch_pdp) {

			struct i915_page_directory_pointer *old;

			struct i915_page_directory *old;



			spin_unlock(&pml4->lock);
@@ -1520,9 +1512,9 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 
			if (IS_ERR(pdp))

				goto unwind;



			gen8_initialize_pdp(vm, pdp);

			gen8_initialize_4lvl_pdp(vm, pdp);



			old = cmpxchg(&pml4->pdps[pml4e], vm->scratch_pdp, pdp);

			old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp);

			if (old == vm->scratch_pdp) {

				gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);

			} else {
@@ -1532,7 +1524,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 


			spin_lock(&pml4->lock);

		}

		atomic_inc(&pdp->used_pdpes);

		atomic_inc(&pdp->used);

		spin_unlock(&pml4->lock);



		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
@@ -1540,7 +1532,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 
			goto unwind_pdp;



		spin_lock(&pml4->lock);

		atomic_dec(&pdp->used_pdpes);

		atomic_dec(&pdp->used);

	}

	spin_unlock(&pml4->lock);
@@ -1548,7 +1540,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 


unwind_pdp:

	spin_lock(&pml4->lock);

	if (atomic_dec_and_test(&pdp->used_pdpes)) {

	if (atomic_dec_and_test(&pdp->used)) {

		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);

		free_pdp(vm, pdp);

	}
@@ -1561,7 +1553,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)

{

	struct i915_address_space *vm = &ppgtt->vm;

	struct i915_page_directory_pointer *pdp = &ppgtt->pdp;

	struct i915_page_directory *pdp = ppgtt->pd;

	struct i915_page_directory *pd;

	u64 start = 0, length = ppgtt->vm.total;

	u64 from = start;
@@ -1574,10 +1566,12 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) 


		gen8_initialize_pd(vm, pd);

		gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);

		atomic_inc(&pdp->used_pdpes);



		atomic_inc(&pdp->used);

	}



	atomic_inc(&pdp->used_pdpes); /* never remove */

	atomic_inc(&pdp->used); /* never remove */



	return 0;



unwind:
@@ -1586,7 +1580,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) 
		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);

		free_pd(vm, pd);

	}

	atomic_set(&pdp->used_pdpes, 0);

	atomic_set(&pdp->used, 0);

	return -ENOMEM;

}
@@ -1641,27 +1635,25 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) 
	if (err)

		goto err_free;



	if (i915_vm_is_4lvl(&ppgtt->vm)) {

		err = setup_px(&ppgtt->vm, &ppgtt->pml4);

		if (err)

	ppgtt->pd = alloc_pdp(&ppgtt->vm);

	if (IS_ERR(ppgtt->pd)) {

		err = PTR_ERR(ppgtt->pd);

		goto err_scratch;

	}



		gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4);

	if (i915_vm_is_4lvl(&ppgtt->vm)) {

		gen8_initialize_pml4(&ppgtt->vm, ppgtt->pd);



		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;

		ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;

		ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;

	} else {

		err = __pdp_init(&ppgtt->vm, &ppgtt->pdp);

		if (err)

			goto err_scratch;

		gen8_initialize_3lvl_pdp(&ppgtt->vm, ppgtt->pd);



		if (intel_vgpu_active(i915)) {

			err = gen8_preallocate_top_level_pdp(ppgtt);

			if (err) {

				__pdp_fini(&ppgtt->pdp);

				goto err_scratch;

			}

			if (err)

				goto err_pdp;

		}



		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
@@ -1676,6 +1668,8 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) 


	return ppgtt;



err_pdp:

	free_pdp(&ppgtt->vm, ppgtt->pd);

err_scratch:

	gen8_free_scratch(&ppgtt->vm);

err_free:
@@ -1741,15 +1735,16 @@ static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) 
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,

				   u64 start, u64 length)

{

	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));

	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;

	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));

	const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;

	const gen6_pte_t scratch_pte = vm->scratch_pte;

	unsigned int pde = first_entry / GEN6_PTES;

	unsigned int pte = first_entry % GEN6_PTES;

	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;

	const gen6_pte_t scratch_pte = vm->scratch_pte;



	while (num_entries) {

		struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];

		struct i915_page_table * const pt =

			i915_pt_entry(ppgtt->base.pd, pde++);

		const unsigned int count = min(num_entries, GEN6_PTES - pte);

		gen6_pte_t *vaddr;
@@ -1757,8 +1752,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 


		num_entries -= count;



		GEM_BUG_ON(count > atomic_read(&pt->used_ptes));

		if (!atomic_sub_return(count, &pt->used_ptes))

		GEM_BUG_ON(count > atomic_read(&pt->used));

		if (!atomic_sub_return(count, &pt->used))

			ppgtt->scan_for_unused_pt = true;



		/*
@@ -1782,6 +1777,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 
				      u32 flags)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_page_directory * const pd = ppgtt->pd;

	unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE;

	unsigned act_pt = first_entry / GEN6_PTES;

	unsigned act_pte = first_entry % GEN6_PTES;
@@ -1789,9 +1785,9 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 
	struct sgt_dma iter = sgt_dma(vma);

	gen6_pte_t *vaddr;



	GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);

	GEM_BUG_ON(i915_pt_entry(pd, act_pt) == vm->scratch_pt);



	vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);

	vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));

	do {

		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@@ -1807,7 +1803,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 


		if (++act_pte == GEN6_PTES) {

			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));

			act_pte = 0;

		}

	} while (1);
@@ -1820,6 +1816,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, 
			       u64 start, u64 length)

{

	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));

	struct i915_page_directory * const pd = ppgtt->base.pd;

	struct i915_page_table *pt;

	intel_wakeref_t wakeref;

	u64 from = start;
@@ -1828,14 +1825,14 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, 


	wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);



	spin_lock(&ppgtt->base.pd.lock);

	gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {

	spin_lock(&pd->lock);

	gen6_for_each_pde(pt, pd, start, length, pde) {

		const unsigned int count = gen6_pte_count(start, length);



		if (pt == vm->scratch_pt) {

			struct i915_page_table *old;



			spin_unlock(&ppgtt->base.pd.lock);

			spin_unlock(&pd->lock);



			pt = alloc_pt(vm);

			if (IS_ERR(pt))
@@ -1843,10 +1840,8 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, 


			gen6_initialize_pt(vm, pt);



			old = cmpxchg(&ppgtt->base.pd.page_table[pde],

				      vm->scratch_pt, pt);

			old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);

			if (old == vm->scratch_pt) {

				ppgtt->base.pd.page_table[pde] = pt;

				if (i915_vma_is_bound(ppgtt->vma,

						      I915_VMA_GLOBAL_BIND)) {

					gen6_write_pde(ppgtt, pde, pt);
@@ -1857,12 +1852,12 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, 
				pt = old;

			}



			spin_lock(&ppgtt->base.pd.lock);

			spin_lock(&pd->lock);

		}



		atomic_add(count, &pt->used_ptes);

		atomic_add(count, &pt->used);

	}

	spin_unlock(&ppgtt->base.pd.lock);

	spin_unlock(&pd->lock);



	if (flush) {

		mark_tlbs_dirty(&ppgtt->base);
@@ -1882,6 +1877,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, 
static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)

{

	struct i915_address_space * const vm = &ppgtt->base.vm;

	struct i915_page_directory * const pd = ppgtt->base.pd;

	struct i915_page_table *unused;

	u32 pde;

	int ret;
@@ -1901,9 +1897,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) 
	}



	gen6_initialize_pt(vm, vm->scratch_pt);

	gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)

		ppgtt->base.pd.page_table[pde] = vm->scratch_pt;

	spin_lock_init(&ppgtt->base.pd.lock);



	gen6_for_all_pdes(unused, pd, pde)

		pd->entry[pde] = vm->scratch_pt;



	return 0;

}
@@ -1916,10 +1912,11 @@ static void gen6_ppgtt_free_scratch(struct i915_address_space *vm) 


static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)

{

	struct i915_page_directory * const pd = ppgtt->base.pd;

	struct i915_page_table *pt;

	u32 pde;



	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)

	gen6_for_all_pdes(pt, pd, pde)

		if (pt != ppgtt->base.vm.scratch_pt)

			free_pt(&ppgtt->base.vm, pt);

}
@@ -1983,6 +1980,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 


	gen6_ppgtt_free_pd(ppgtt);

	gen6_ppgtt_free_scratch(vm);

	kfree(ppgtt->base.pd);

}



static int pd_vma_set_pages(struct i915_vma *vma)
@@ -2008,10 +2006,10 @@ static int pd_vma_bind(struct i915_vma *vma, 
	struct i915_page_table *pt;

	unsigned int pde;



	ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);

	ppgtt->base.pd->base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);

	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;



	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)

	gen6_for_all_pdes(pt, ppgtt->base.pd, pde)

		gen6_write_pde(ppgtt, pde, pt);



	mark_tlbs_dirty(&ppgtt->base);
@@ -2023,6 +2021,7 @@ static int pd_vma_bind(struct i915_vma *vma, 
static void pd_vma_unbind(struct i915_vma *vma)

{

	struct gen6_ppgtt *ppgtt = vma->private;

	struct i915_page_directory * const pd = ppgtt->base.pd;

	struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;

	struct i915_page_table *pt;

	unsigned int pde;
@@ -2031,12 +2030,12 @@ static void pd_vma_unbind(struct i915_vma *vma) 
		return;



	/* Free all no longer used page tables */

	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {

		if (atomic_read(&pt->used_ptes) || pt == scratch_pt)

	gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {

		if (atomic_read(&pt->used) || pt == scratch_pt)

			continue;



		free_pt(&ppgtt->base.vm, pt);

		ppgtt->base.pd.page_table[pde] = scratch_pt;

		pd->entry[pde] = scratch_pt;

	}



	ppgtt->scan_for_unused_pt = false;
@@ -2165,9 +2164,15 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) 
		goto err_free;

	}



	ppgtt->base.pd = __alloc_pd();

	if (!ppgtt->base.pd) {

		err = -ENOMEM;

		goto err_work;

	}



	err = gen6_ppgtt_init_scratch(ppgtt);

	if (err)

		goto err_work;

		goto err_pd;



	ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);

	if (IS_ERR(ppgtt->vma)) {
@@ -2179,6 +2184,8 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) 


err_scratch:

	gen6_ppgtt_free_scratch(&ppgtt->base.vm);

err_pd:

	kfree(ppgtt->base.pd);

err_work:

	kfree(ppgtt->work);

err_free: