Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8a98e839 authored by Chris Wilson's avatar Chris Wilson
Browse files

drm/i915/gtt: Recursive ppgtt alloc for gen8 



Refactor the separate allocation routines into a single recursive
function.

Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarAbdiel Janulgue <abdiel.janulgue@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190712112725.2892-4-chris@chris-wilson.co.uk
parent 09975b86

drivers/gpu/drm/i915/i915_gem_gtt.c

+97 −175
Original line number Diff line number Diff line
@@ -1007,199 +1007,119 @@ static void gen8_ppgtt_clear(struct i915_address_space *vm, 
			   start, start + length, vm->top);

}



static void gen8_ppgtt_clear_pd(struct i915_address_space *vm,

				struct i915_page_directory *pd,

				u64 start, u64 length)

{

	GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));

	GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));



	start >>= GEN8_PTE_SHIFT;

	length >>= GEN8_PTE_SHIFT;



	__gen8_ppgtt_clear(vm, pd, start, start + length, 1);

}



static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,

				 struct i915_page_directory * const pdp,

				 u64 start, u64 length)

{

	GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));

	GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));



	start >>= GEN8_PTE_SHIFT;

	length >>= GEN8_PTE_SHIFT;



	__gen8_ppgtt_clear(vm, pdp, start, start + length, 2);

}



static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,

			       struct i915_page_directory *pd,

			       u64 start, u64 length)

static int __gen8_ppgtt_alloc(struct i915_address_space * const vm,

			      struct i915_page_directory * const pd,

			      u64 * const start, u64 end, int lvl)

{

	struct i915_page_table *pt, *alloc = NULL;

	u64 from = start;

	unsigned int pde;

	const struct i915_page_scratch * const scratch = &vm->scratch[lvl];

	struct i915_page_table *alloc = NULL;

	unsigned int idx, len;

	int ret = 0;



	len = gen8_pd_range(*start, end, lvl--, &idx);

	DBG("%s(%p):{lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d}\n",

	    __func__, vm, lvl + 1, *start, end,

	    idx, len, atomic_read(px_used(pd)));

	GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1));



	spin_lock(&pd->lock);

	gen8_for_each_pde(pt, pd, start, length, pde) {

		const int count = gen8_pte_count(start, length);

	GEM_BUG_ON(!atomic_read(px_used(pd))); /* Must be pinned! */

	do {

		struct i915_page_table *pt = pd->entry[idx];



		if (!pt) {

			spin_unlock(&pd->lock);



			DBG("%s(%p):{ lvl:%d, idx:%d } allocating new tree\n",

			    __func__, vm, lvl + 1, idx);



			pt = fetch_and_zero(&alloc);

			if (!pt)

				pt = alloc_pt(vm);

			if (lvl) {

				if (!pt) {

					pt = &alloc_pd(vm)->pt;

					if (IS_ERR(pt)) {

						ret = PTR_ERR(pt);

				goto unwind;

						goto out;

					}

				}



			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))

				fill_px(pt, vm->scratch[0].encode);



			spin_lock(&pd->lock);

			if (!pd->entry[pde]) {

				set_pd_entry(pd, pde, pt);

				fill_px(pt, vm->scratch[lvl].encode);

			} else {

				alloc = pt;

				pt = pd->entry[pde];

				if (!pt) {

					pt = alloc_pt(vm);

					if (IS_ERR(pt)) {

						ret = PTR_ERR(pt);

						goto out;

					}

				}



		atomic_add(count, &pt->used);

				if (intel_vgpu_active(vm->i915) ||

				    gen8_pt_count(*start, end) < I915_PDES)

					fill_px(pt, vm->scratch[lvl].encode);

			}

	spin_unlock(&pd->lock);

	goto out;



unwind:

	gen8_ppgtt_clear_pd(vm, pd, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

			spin_lock(&pd->lock);

			if (likely(!pd->entry[idx]))

				set_pd_entry(pd, idx, pt);

			else

				alloc = pt, pt = pd->entry[idx];

		}



static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,

				struct i915_page_directory *pdp,

				u64 start, u64 length)

{

	struct i915_page_directory *pd, *alloc = NULL;

	u64 from = start;

	unsigned int pdpe;

	int ret = 0;



	spin_lock(&pdp->lock);

	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {

		if (!pd) {

			spin_unlock(&pdp->lock);

		if (lvl) {

			atomic_inc(&pt->used);

			spin_unlock(&pd->lock);



			pd = fetch_and_zero(&alloc);

			if (!pd)

				pd = alloc_pd(vm);

			if (IS_ERR(pd)) {

				ret = PTR_ERR(pd);

				goto unwind;

			ret = __gen8_ppgtt_alloc(vm, as_pd(pt),

						 start, end, lvl);

			if (unlikely(ret)) {

				if (release_pd_entry(pd, idx, pt, scratch))

					free_px(vm, pt);

				goto out;

			}



			fill_px(pd, vm->scratch[1].encode);



			spin_lock(&pdp->lock);

			if (!pdp->entry[pdpe]) {

				set_pd_entry(pdp, pdpe, pd);

			spin_lock(&pd->lock);

			atomic_dec(&pt->used);

			GEM_BUG_ON(!atomic_read(&pt->used));

		} else {

				alloc = pd;

				pd = pdp->entry[pdpe];

			}

		}

		atomic_inc(px_used(pd));

		spin_unlock(&pdp->lock);

			unsigned int count = gen8_pt_count(*start, end);



		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);

		if (unlikely(ret))

			goto unwind_pd;

			DBG("%s(%p):{lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d} inserting pte\n",

			    __func__, vm, lvl, *start, end,

			    gen8_pd_index(*start, 0), count,

			    atomic_read(&pt->used));



		spin_lock(&pdp->lock);

		atomic_dec(px_used(pd));

			atomic_add(count, &pt->used);

			GEM_BUG_ON(atomic_read(&pt->used) > I915_PDES);

			*start += count;

		}

	spin_unlock(&pdp->lock);

	goto out;



unwind_pd:

	if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2]))

		free_px(vm, pd);

unwind:

	gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);

	} while (idx++, --len);

	spin_unlock(&pd->lock);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,

static int gen8_ppgtt_alloc(struct i915_address_space *vm,

			    u64 start, u64 length)

{

	return gen8_ppgtt_alloc_pdp(vm,

				    i915_vm_to_ppgtt(vm)->pd, start, length);

}



static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_page_directory * const pml4 = ppgtt->pd;

	struct i915_page_directory *pdp, *alloc = NULL;

	u64 from = start;

	int ret = 0;

	u32 pml4e;



	spin_lock(&pml4->lock);

	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {

		if (!pdp) {

			spin_unlock(&pml4->lock);



			pdp = fetch_and_zero(&alloc);

			if (!pdp)

				pdp = alloc_pd(vm);

			if (IS_ERR(pdp)) {

				ret = PTR_ERR(pdp);

				goto unwind;

			}



			fill_px(pdp, vm->scratch[2].encode);

	int err;



			spin_lock(&pml4->lock);

			if (!pml4->entry[pml4e]) {

				set_pd_entry(pml4, pml4e, pdp);

			} else {

				alloc = pdp;

				pdp = pml4->entry[pml4e];

			}

		}

		atomic_inc(px_used(pdp));

		spin_unlock(&pml4->lock);

	GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));

	GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));



		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);

		if (unlikely(ret))

			goto unwind_pdp;

	start >>= GEN8_PTE_SHIFT;

	length >>= GEN8_PTE_SHIFT;

	GEM_BUG_ON(length == 0);



		spin_lock(&pml4->lock);

		atomic_dec(px_used(pdp));

	}

	spin_unlock(&pml4->lock);

	goto out;

	err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd,

				 &start, start + length, vm->top);

	if (unlikely(err))

		__gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,

				   from, start, vm->top);



unwind_pdp:

	if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))

		free_px(vm, pdp);

unwind:

	gen8_ppgtt_clear(vm, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

	return err;

}



static inline struct sgt_dma {
@@ -1496,19 +1416,22 @@ static int gen8_init_scratch(struct i915_address_space *vm) 
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)

{

	struct i915_address_space *vm = &ppgtt->vm;

	struct i915_page_directory *pdp = ppgtt->pd;

	struct i915_page_directory *pd;

	u64 start = 0, length = ppgtt->vm.total;

	unsigned int pdpe;

	struct i915_page_directory *pd = ppgtt->pd;

	unsigned int idx;



	GEM_BUG_ON(vm->top != 2);

	GEM_BUG_ON((vm->total >> __gen8_pte_shift(2)) != GEN8_3LVL_PDPES);



	for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) {

		struct i915_page_directory *pde;



	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {

		pd = alloc_pd(vm);

		if (IS_ERR(pd))

			return PTR_ERR(pd);

		pde = alloc_pd(vm);

		if (IS_ERR(pde))

			return PTR_ERR(pde);



		fill_px(pd, vm->scratch[1].encode);

		set_pd_entry(pdp, pdpe, pd);

		atomic_inc(px_used(pd)); /* keep pinned */

		fill_px(pde, vm->scratch[1].encode);

		set_pd_entry(pd, idx, pde);

		atomic_inc(px_used(pde)); /* keep pinned */

	}



	return 0;
@@ -1597,7 +1520,6 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) 
	}



	if (i915_vm_is_4lvl(&ppgtt->vm)) {

		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;

		ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;

	} else {

		if (intel_vgpu_active(i915)) {
@@ -1606,10 +1528,10 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) 
				goto err_free_pd;

		}



		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;

		ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;

	}



	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;

	ppgtt->vm.clear_range = gen8_ppgtt_clear;



	if (intel_vgpu_active(i915))