Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 85061e29 authored by Tarun Karra's avatar Tarun Karra
Browse files

msm: kgsl: Enable A5XX preemption 



Enable A5XX preemption and re-architect code for
preemption states clear, trigger and complete states
to be device specific.

Change-Id: Ie80e1d893a67e57b52e9172525f6c8121637f1bd
Signed-off-by: default avatarTarun Karra <tkarra@codeaurora.org>
parent a5eb2c93

drivers/gpu/msm/adreno-gpulist.h

+2 −1
Original line number Diff line number Diff line
@@ -140,7 +140,8 @@ static const struct adreno_gpu_core adreno_gpulist[] = { 
		.major = 3,

		.minor = 0,

		.patchid = ANY_ID,

		.features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM,

		.features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM |

							ADRENO_PREEMPTION,

		.pm4fw_name = "a530_pm4.fw",

		.pfpfw_name = "a530_pfp.fw",

		.zap_name = "a530_zap",

drivers/gpu/msm/adreno.c

+6 −0
Original line number Diff line number Diff line
@@ -1470,6 +1470,12 @@ static int _adreno_start(struct adreno_device *adreno_dev) 
	if (status)

		goto error_mmu_off;



	if (gpudev->hw_init) {

		status = gpudev->hw_init(adreno_dev);

		if (status)

			goto error_mmu_off;

	}



	/* Set up LM before initializing the GPMU */

	if (gpudev->lm_init)

		gpudev->lm_init(adreno_dev);

drivers/gpu/msm/adreno.h

+3 −4
Original line number Diff line number Diff line
@@ -332,6 +332,7 @@ struct adreno_device { 
	const struct firmware *lm_fw;

	uint32_t *lm_sequence;

	uint32_t lm_size;

	struct kgsl_memdesc preemption_counters;

};



/**
@@ -637,6 +638,7 @@ struct adreno_gpudev { 
	void (*snapshot)(struct adreno_device *, struct kgsl_snapshot *);

	void (*gpudev_init)(struct adreno_device *);

	int (*rb_init)(struct adreno_device *, struct adreno_ringbuffer *);

	int (*hw_init)(struct adreno_device *);

	int (*microcode_read)(struct adreno_device *);

	int (*microcode_load)(struct adreno_device *, unsigned int start_type);

	void (*perfcounter_init)(struct adreno_device *);
@@ -663,11 +665,8 @@ struct adreno_gpudev { 
	int (*preemption_token)(struct adreno_device *,

				struct adreno_ringbuffer *, unsigned int *,

				uint64_t gpuaddr);

	void (*preemption_start)(struct adreno_device *,

				struct adreno_ringbuffer *);

	void (*preemption_save)(struct adreno_device *,

				struct adreno_ringbuffer *);

	void (*preemption_init)(struct adreno_device *);

	void (*preemption_schedule)(struct adreno_device *);

};



struct log_field {

drivers/gpu/msm/adreno_a4xx.c

+344 −2
Original line number Diff line number Diff line
@@ -1771,6 +1771,349 @@ static struct adreno_snapshot_data a4xx_snapshot_data = { 
	.sect_sizes = &a4xx_snap_sizes,

};



/**

 * a4xx_preempt_trig_state() - Schedule preemption in TRIGGERRED

 * state

 * @adreno_dev: Device which is in TRIGGERRED state

 */

static void a4xx_preempt_trig_state(

			struct adreno_device *adreno_dev)

{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct kgsl_device *device = &(adreno_dev->dev);

	unsigned int rbbase, val;



	/*

	 * Hardware not yet idle means that preemption interrupt

	 * may still occur, nothing to do here until interrupt signals

	 * completion of preemption, just return here

	 */

	if (!adreno_hw_isidle(adreno_dev))

		return;



	/*

	 * We just changed states, reschedule dispatcher to change

	 * preemption states

	 */

	if (ADRENO_DISPATCHER_PREEMPT_TRIGGERED !=

		atomic_read(&dispatcher->preemption_state)) {

		adreno_dispatcher_schedule(device);

		return;

	}



	/*

	 * H/W is idle and we did not get a preemption interrupt, may

	 * be device went idle w/o encountering any preempt token or

	 * we already preempted w/o interrupt

	 */

	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase);

	 /* Did preemption occur, if so then change states and return */

	if (rbbase != adreno_dev->cur_rb->buffer_desc.gpuaddr) {

		adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val);

		if (val && rbbase == adreno_dev->next_rb->buffer_desc.gpuaddr) {

			KGSL_DRV_INFO(device,

			"Preemption completed without interrupt\n");

			trace_adreno_hw_preempt_trig_to_comp(adreno_dev->cur_rb,

					adreno_dev->next_rb);

			atomic_set(&dispatcher->preemption_state,

				ADRENO_DISPATCHER_PREEMPT_COMPLETE);

			adreno_dispatcher_schedule(device);

			return;

		}

		adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);

		/* reschedule dispatcher to take care of the fault */

		adreno_dispatcher_schedule(device);

		return;

	}

	/*

	 * Check if preempt token was submitted after preemption trigger, if so

	 * then preemption should have occurred, since device is already idle it

	 * means something went wrong - trigger FT

	 */

	if (dispatcher->preempt_token_submit) {

		adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);

		/* reschedule dispatcher to take care of the fault */

		adreno_dispatcher_schedule(device);

		return;

	}

	/*

	 * Preempt token was not submitted after preemption trigger so device

	 * may have gone idle before preemption could occur, if there are

	 * commands that got submitted to current RB after triggering preemption

	 * then submit them as those commands may have a preempt token in them

	 */

	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR,

			&adreno_dev->cur_rb->rptr);

	if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) {

		/*

		 * Memory barrier before informing the

		 * hardware of new commands

		 */

		mb();

		kgsl_pwrscale_busy(device);

		adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR,

			adreno_dev->cur_rb->wptr);

		return;

	}



	/* Submit preempt token to make preemption happen */

	if (adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, NULL, 0))

		BUG();

	if (adreno_ringbuffer_submit_preempt_token(adreno_dev->cur_rb,

						adreno_dev->next_rb))

		BUG();

	dispatcher->preempt_token_submit = 1;

	adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr;

	trace_adreno_hw_preempt_token_submit(adreno_dev->cur_rb,

						adreno_dev->next_rb);

}



/**

 * a4xx_preempt_clear_state() - Schedule preemption in

 * CLEAR state. Preemption can be issued in this state.

 * @adreno_dev: Device which is in CLEAR state

 */

static void a4xx_preempt_clear_state(

			struct adreno_device *adreno_dev)



{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct kgsl_device *device = &(adreno_dev->dev);

	struct adreno_dispatcher_cmdqueue *dispatch_tempq;

	struct kgsl_cmdbatch *cmdbatch;

	struct adreno_ringbuffer *highest_busy_rb;

	int switch_low_to_high;

	int ret;



	/* Device not awake means there is nothing to do */

	if (!kgsl_state_is_awake(device))

		return;



	/* keep updating the current rptr when preemption is clear */

	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR,

			&(adreno_dev->cur_rb->rptr));



	highest_busy_rb = adreno_dispatcher_get_highest_busy_rb(adreno_dev);

	if (!highest_busy_rb)

		return;



	switch_low_to_high = adreno_compare_prio_level(

					highest_busy_rb->id,

					adreno_dev->cur_rb->id);



	/* already current then return */

	if (!switch_low_to_high)

		return;



	if (switch_low_to_high < 0) {

		/*

		 * if switching to lower priority make sure that the rptr and

		 * wptr are equal, when the lower rb is not starved

		 */

		if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr)

			return;

		/*

		 * switch to default context because when we switch back

		 * to higher context then its not known which pt will

		 * be current, so by making it default here the next

		 * commands submitted will set the right pt

		 */

		ret = adreno_drawctxt_switch(adreno_dev,

				adreno_dev->cur_rb,

				NULL, 0);

		/*

		 * lower priority RB has to wait until space opens up in

		 * higher RB

		 */

		if (ret)

			return;



		adreno_writereg(adreno_dev,

			ADRENO_REG_CP_PREEMPT_DISABLE, 1);

	}



	/*

	 * setup registers to do the switch to highest priority RB

	 * which is not empty or may be starving away(poor thing)

	 */

	a4xx_preemption_start(adreno_dev, highest_busy_rb);



	/* turn on IOMMU as the preemption may trigger pt switch */

	kgsl_mmu_enable_clk(&device->mmu);



	atomic_set(&dispatcher->preemption_state,

			ADRENO_DISPATCHER_PREEMPT_TRIGGERED);



	adreno_dev->next_rb = highest_busy_rb;

	mod_timer(&dispatcher->preempt_timer, jiffies +

		msecs_to_jiffies(ADRENO_DISPATCH_PREEMPT_TIMEOUT));



	trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb,

						adreno_dev->next_rb);

	/* issue PREEMPT trigger */

	adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1);

	/*

	 * IOMMU clock can be safely switched off after the timestamp

	 * of the first command in the new rb

	 */

	dispatch_tempq = &adreno_dev->next_rb->dispatch_q;

	if (dispatch_tempq->head != dispatch_tempq->tail)

		cmdbatch = dispatch_tempq->cmd_q[dispatch_tempq->head];

	else

		cmdbatch = 0;

	if (cmdbatch)

		adreno_ringbuffer_mmu_disable_clk_on_ts(device,

			adreno_dev->next_rb,

			cmdbatch->global_ts);

	else

		adreno_ringbuffer_mmu_disable_clk_on_ts(device,

			adreno_dev->next_rb, adreno_dev->next_rb->timestamp);

	/* submit preempt token packet to ensure preemption */

	if (switch_low_to_high < 0) {

		ret = adreno_ringbuffer_submit_preempt_token(

			adreno_dev->cur_rb, adreno_dev->next_rb);

		/*

		 * unexpected since we are submitting this when rptr = wptr,

		 * this was checked above already

		 */

		BUG_ON(ret);

		dispatcher->preempt_token_submit = 1;

		adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr;

	} else {

		dispatcher->preempt_token_submit = 0;

		adreno_dispatcher_schedule(device);

		adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF;

	}

}



/**

 * a4xx_preempt_complete_state() - Schedule preemption in

 * COMPLETE state

 * @adreno_dev: Device which is in COMPLETE state

 */

static void a4xx_preempt_complete_state(

			struct adreno_device *adreno_dev)



{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct kgsl_device *device = &(adreno_dev->dev);

	struct adreno_dispatcher_cmdqueue *dispatch_q;

	unsigned int wptr, rbbase;

	unsigned int val, val1;



	del_timer_sync(&dispatcher->preempt_timer);



	adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &val);

	adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val1);



	if (val || !val1) {

		KGSL_DRV_ERR(device,

		"Invalid state after preemption CP_PREEMPT: %08x, CP_PREEMPT_DEBUG: %08x\n",

		val, val1);

		adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);

		adreno_dispatcher_schedule(device);

		return;

	}

	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase);

	if (rbbase != adreno_dev->next_rb->buffer_desc.gpuaddr) {

		KGSL_DRV_ERR(device,

		"RBBASE incorrect after preemption, expected %x got %016llx\b",

		rbbase,

		adreno_dev->next_rb->buffer_desc.gpuaddr);

		adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);

		adreno_dispatcher_schedule(device);

		return;

	}



	a4xx_preemption_save(adreno_dev, adreno_dev->cur_rb);



	dispatch_q = &(adreno_dev->cur_rb->dispatch_q);

	/* new RB is the current RB */

	trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb,

						adreno_dev->cur_rb);

	adreno_dev->prev_rb = adreno_dev->cur_rb;

	adreno_dev->cur_rb = adreno_dev->next_rb;

	adreno_dev->cur_rb->preempted_midway = 0;

	adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF;

	adreno_dev->next_rb = 0;

	if (adreno_disp_preempt_fair_sched) {

		/* starved rb is now scheduled so unhalt dispatcher */

		if (ADRENO_DISPATCHER_RB_STARVE_TIMER_ELAPSED ==

			adreno_dev->cur_rb->starve_timer_state)

			adreno_put_gpu_halt(adreno_dev);

		adreno_dev->cur_rb->starve_timer_state =

				ADRENO_DISPATCHER_RB_STARVE_TIMER_SCHEDULED;

		adreno_dev->cur_rb->sched_timer = jiffies;

		/*

		 * If the outgoing RB is has commands then set the

		 * busy time for it

		 */

		if (adreno_dev->prev_rb->rptr != adreno_dev->prev_rb->wptr) {

			adreno_dev->prev_rb->starve_timer_state =

				ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT;

			adreno_dev->prev_rb->sched_timer = jiffies;

		} else {

			adreno_dev->prev_rb->starve_timer_state =

				ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT;

		}

	}

	atomic_set(&dispatcher->preemption_state,

		ADRENO_DISPATCHER_PREEMPT_CLEAR);

	if (adreno_compare_prio_level(adreno_dev->prev_rb->id,

				adreno_dev->cur_rb->id) < 0) {

		if (adreno_dev->prev_rb->wptr_preempt_end !=

			adreno_dev->prev_rb->rptr)

			adreno_dev->prev_rb->preempted_midway = 1;

	} else if (adreno_dev->prev_rb->wptr_preempt_end !=

		adreno_dev->prev_rb->rptr) {

		BUG();

	}

	/* submit wptr if required for new rb */

	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr);

	if (adreno_dev->cur_rb->wptr != wptr) {

		kgsl_pwrscale_busy(device);

		adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR,

					adreno_dev->cur_rb->wptr);

	}

	/* clear preemption register */

	adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, 0);

	adreno_preempt_process_dispatch_queue(adreno_dev, dispatch_q);

}



static void a4xx_preemption_schedule(

				struct adreno_device *adreno_dev)

{

	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;

	struct kgsl_device *device = &(adreno_dev->dev);



	if (!adreno_is_preemption_enabled(adreno_dev))

		return;



	mutex_lock(&device->mutex);



	switch (atomic_read(&dispatcher->preemption_state)) {

	case ADRENO_DISPATCHER_PREEMPT_CLEAR:

		a4xx_preempt_clear_state(adreno_dev);

		break;

	case ADRENO_DISPATCHER_PREEMPT_TRIGGERED:

		a4xx_preempt_trig_state(adreno_dev);

		/*

		 * if we transitioned to next state then fall-through

		 * processing to next state

		 */

		if (!adreno_preempt_state(adreno_dev,

			ADRENO_DISPATCHER_PREEMPT_COMPLETE))

			break;

	case ADRENO_DISPATCHER_PREEMPT_COMPLETE:

		a4xx_preempt_complete_state(adreno_dev);

		break;

	default:

		BUG();

	}



	mutex_unlock(&device->mutex);

}



struct adreno_gpudev adreno_a4xx_gpudev = {

	.reg_offsets = &a4xx_reg_offsets,

	.ft_perf_counters = a4xx_ft_perf_counters,
@@ -1798,6 +2141,5 @@ struct adreno_gpudev adreno_a4xx_gpudev = { 
	.regulator_disable = a4xx_regulator_disable,

	.preemption_pre_ibsubmit = a4xx_preemption_pre_ibsubmit,

	.preemption_token = a4xx_preemption_token,

	.preemption_start = a4xx_preemption_start,

	.preemption_save = a4xx_preemption_save,

	.preemption_schedule = a4xx_preemption_schedule,

};