Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 049fc527 authored by Chunming Zhou's avatar Chunming Zhou Committed by Alex Deucher
Browse files

drm/amdgpu: dispatch jobs in cs 



BO validation is moved to scheduler except usrptr which must be validated
in user process

Signed-off-by: default avatarChunming Zhou <david1.zhou@amd.com>
Acked-by: default avatarChristian K?nig <christian.koenig@amd.com>
Reviewed-by: default avatarJammy Zhou <Jammy.Zhou@amd.com>
parent 372bc1e1

drivers/gpu/drm/amd/amdgpu/amdgpu.h

+1 −0
Original line number Diff line number Diff line
@@ -1243,6 +1243,7 @@ struct amdgpu_cs_parser { 
	struct work_struct job_work;

	int (*prepare_job)(struct amdgpu_cs_parser *sched_job);

	int (*run_job)(struct amdgpu_cs_parser *sched_job);

	int (*free_job)(struct amdgpu_cs_parser *sched_job);

};



static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx)

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c

+199 −57
Original line number Diff line number Diff line
@@ -41,6 +41,11 @@ struct amdgpu_cs_buckets { 
	struct list_head bucket[AMDGPU_CS_NUM_BUCKETS];

};



static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser,

				  int error, bool backoff);

static void amdgpu_cs_parser_fini_early(struct amdgpu_cs_parser *parser, int error, bool backoff);

static void amdgpu_cs_parser_fini_late(struct amdgpu_cs_parser *parser);



static void amdgpu_cs_buckets_init(struct amdgpu_cs_buckets *b)

{

	unsigned i;
@@ -126,12 +131,52 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, 
	return 0;

}



static void amdgpu_job_work_func(struct work_struct *work)

{

	struct amdgpu_cs_parser *sched_job =

		container_of(work, struct amdgpu_cs_parser,

			     job_work);

	mutex_lock(&sched_job->job_lock);

	sched_job->free_job(sched_job);

	mutex_unlock(&sched_job->job_lock);

	/* after processing job, free memory */

	kfree(sched_job);

}

struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev,

                                               struct drm_file *filp,

                                               struct amdgpu_ctx *ctx,

                                               struct amdgpu_ib *ibs,

                                               uint32_t num_ibs)

{

	struct amdgpu_cs_parser *parser;

	int i;



	parser = kzalloc(sizeof(struct amdgpu_cs_parser), GFP_KERNEL);

	if (!parser)

		return NULL;



	parser->adev = adev;

	parser->filp = filp;

	parser->ctx = ctx;

	parser->ibs = ibs;

	parser->num_ibs = num_ibs;

	if (amdgpu_enable_scheduler) {

		mutex_init(&parser->job_lock);

		INIT_WORK(&parser->job_work, amdgpu_job_work_func);

	}

	for (i = 0; i < num_ibs; i++)

		ibs[i].ctx = ctx;



	return parser;

}



int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)

{

	union drm_amdgpu_cs *cs = data;

	uint64_t *chunk_array_user;

	uint64_t *chunk_array = NULL;

	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;

	struct amdgpu_bo_list *bo_list = NULL;

	unsigned size, i;

	int r = 0;
@@ -143,7 +188,17 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) 
		r = -EINVAL;

		goto out;

	}

	p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);

	bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);

	if (bo_list && !bo_list->has_userptr) {

		p->bo_list = kzalloc(sizeof(struct amdgpu_bo_list), GFP_KERNEL);

		if (!p->bo_list)

			return -ENOMEM;

		amdgpu_bo_list_copy(p->adev, p->bo_list, bo_list);

		amdgpu_bo_list_put(bo_list);

	} else if (bo_list && bo_list->has_userptr)

		p->bo_list = bo_list;

	else

		p->bo_list = NULL;



	/* get chunks */

	INIT_LIST_HEAD(&p->validated);
@@ -424,8 +479,26 @@ static int cmp_size_smaller_first(void *priv, struct list_head *a, 
 **/

static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)

{

	unsigned i;

       amdgpu_cs_parser_fini_early(parser, error, backoff);

       amdgpu_cs_parser_fini_late(parser);

}



static int amdgpu_cs_parser_run_job(

	struct amdgpu_cs_parser *sched_job)

{

       amdgpu_cs_parser_fini_early(sched_job, 0, true);

       return 0;

}



static int amdgpu_cs_parser_free_job(

	struct amdgpu_cs_parser *sched_job)

{

       amdgpu_cs_parser_fini_late(sched_job);

       return 0;

}



static void amdgpu_cs_parser_fini_early(struct amdgpu_cs_parser *parser, int error, bool backoff)

{

	if (!error) {

		/* Sort the buffer list from the smallest to largest buffer,

		 * which affects the order of buffers in the LRU list.
@@ -446,11 +519,19 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo 
		ttm_eu_backoff_reservation(&parser->ticket,

					   &parser->validated);

	}

}



static void amdgpu_cs_parser_fini_late(struct amdgpu_cs_parser *parser)

{

	unsigned i;

	if (parser->ctx)

		amdgpu_ctx_put(parser->ctx);

	if (parser->bo_list)

	if (parser->bo_list) {

		if (!parser->bo_list->has_userptr)

			amdgpu_bo_list_free(parser->bo_list);

		else

			amdgpu_bo_list_put(parser->bo_list);

	}

	drm_free_large(parser->vm_bos);

	for (i = 0; i < parser->nchunks; i++)

		drm_free_large(parser->chunks[i].kdata);
@@ -461,6 +542,9 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo 
	kfree(parser->ibs);

	if (parser->uf.bo)

		drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base);



	if (!amdgpu_enable_scheduler)

		kfree(parser);

}



static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
@@ -533,7 +617,7 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, 
		goto out;

	}

	amdgpu_cs_sync_rings(parser);



	if (!amdgpu_enable_scheduler)

		r = amdgpu_ib_schedule(adev, parser->num_ibs, parser->ibs,

				       parser->filp);
@@ -731,33 +815,14 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 
	return 0;

}



int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)

static int amdgpu_cs_parser_prepare_job(struct amdgpu_cs_parser *sched_job)

{

	struct amdgpu_device *adev = dev->dev_private;

	union drm_amdgpu_cs *cs = data;

	struct amdgpu_cs_parser parser;

        int r, i;

	struct amdgpu_cs_parser *parser = sched_job;

	struct amdgpu_device *adev = sched_job->adev;

        bool reserved_buffers = false;



	down_read(&adev->exclusive_lock);

	if (!adev->accel_working) {

		up_read(&adev->exclusive_lock);

		return -EBUSY;

	}

	/* initialize parser */

	memset(&parser, 0, sizeof(struct amdgpu_cs_parser));

	parser.filp = filp;

	parser.adev = adev;

	r = amdgpu_cs_parser_init(&parser, data);

	if (r) {

		DRM_ERROR("Failed to initialize parser !\n");

		amdgpu_cs_parser_fini(&parser, r, false);

		up_read(&adev->exclusive_lock);

		r = amdgpu_cs_handle_lockup(adev, r);

		return r;

	}



	r = amdgpu_cs_parser_relocs(&parser);

        r = amdgpu_cs_parser_relocs(parser);

        if (r) {

                if (r != -ERESTARTSYS) {

			if (r == -ENOMEM)
@@ -769,33 +834,104 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 


	if (!r) {

		reserved_buffers = true;

		r = amdgpu_cs_ib_fill(adev, &parser);

		r = amdgpu_cs_ib_fill(adev, parser);

	}



	if (!r) {

		r = amdgpu_cs_dependencies(adev, &parser);

		r = amdgpu_cs_dependencies(adev, parser);

		if (r)

			DRM_ERROR("Failed in the dependencies handling %d!\n", r);

	}

        if (r) {

                amdgpu_cs_parser_fini(parser, r, reserved_buffers);

               return r;

        }



        for (i = 0; i < parser->num_ibs; i++)

                trace_amdgpu_cs(parser, i);



       r = amdgpu_cs_ib_vm_chunk(adev, parser);

       return r;

}



static struct amdgpu_ring *amdgpu_cs_parser_get_ring(

	struct amdgpu_device *adev,

	struct amdgpu_cs_parser *parser)

{

	int i, r;



	struct amdgpu_cs_chunk *chunk;

	struct drm_amdgpu_cs_chunk_ib *chunk_ib;

	struct amdgpu_ring *ring;

	for (i = 0; i < parser->nchunks; i++) {

		chunk = &parser->chunks[i];

		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;



		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)

			continue;



		r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,

				       chunk_ib->ip_instance, chunk_ib->ring,

				       &ring);

		if (r)

			return NULL;

		break;

	}

	return ring;

}



int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)

{

	struct amdgpu_device *adev = dev->dev_private;

	union drm_amdgpu_cs *cs = data;

	struct amdgpu_cs_parser *parser;

	int r;



	down_read(&adev->exclusive_lock);

	if (!adev->accel_working) {

		up_read(&adev->exclusive_lock);

		return -EBUSY;

	}



	parser = amdgpu_cs_parser_create(adev, filp, NULL, NULL, 0);

	if (!parser)

		return -ENOMEM;

	r = amdgpu_cs_parser_init(parser, data);

	if (r) {

		amdgpu_cs_parser_fini(&parser, r, reserved_buffers);

		DRM_ERROR("Failed to initialize parser !\n");

		amdgpu_cs_parser_fini(parser, r, false);

		up_read(&adev->exclusive_lock);

		r = amdgpu_cs_handle_lockup(adev, r);

		return r;

	}



	for (i = 0; i < parser.num_ibs; i++)

		trace_amdgpu_cs(&parser, i);



	r = amdgpu_cs_ib_vm_chunk(adev, &parser);

	if (r) {

	if (amdgpu_enable_scheduler && parser->num_ibs) {

		struct amdgpu_ring * ring =

			amdgpu_cs_parser_get_ring(adev, parser);

		parser->uf.sequence = atomic64_inc_return(

			&parser->ctx->rings[ring->idx].c_entity.last_queued_v_seq);

		if ((parser->bo_list && parser->bo_list->has_userptr)) {

			r = amdgpu_cs_parser_prepare_job(parser);

			if (r)

				goto out;

		} else

			parser->prepare_job = amdgpu_cs_parser_prepare_job;



		parser->run_job = amdgpu_cs_parser_run_job;

		parser->free_job = amdgpu_cs_parser_free_job;

		amd_sched_push_job(ring->scheduler,

				   &parser->ctx->rings[ring->idx].c_entity,

				   parser);

		cs->out.handle = parser->uf.sequence;

		up_read(&adev->exclusive_lock);

		return 0;

	}

	r = amdgpu_cs_parser_prepare_job(parser);

	if (r)

		goto out;



	cs->out.handle = parser.ibs[parser.num_ibs - 1].sequence;

	cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence;

out:

	amdgpu_cs_parser_fini(&parser, r, true);

	amdgpu_cs_parser_fini(parser, r, true);

	up_read(&adev->exclusive_lock);

	r = amdgpu_cs_handle_lockup(adev, r);

	return r;
@@ -829,7 +965,13 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 
	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);

	if (ctx == NULL)

		return -EINVAL;



	if (amdgpu_enable_scheduler) {

		r = amd_sched_wait_ts(&ctx->rings[ring->idx].c_entity,

				      wait->in.handle, true, timeout);

		if (r)

			return r;

		r = 1;

	} else {

		fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);

		if (IS_ERR(fence))

			r = PTR_ERR(fence);
@@ -840,7 +982,7 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 


		} else

			r = 1;



	}

	amdgpu_ctx_put(ctx);

	if (r < 0)

		return r;