Loading drivers/gpu/msm/adreno-gpulist.h +2 −1 Original line number Diff line number Diff line Loading @@ -140,7 +140,8 @@ static const struct adreno_gpu_core adreno_gpulist[] = { .major = 3, .minor = 0, .patchid = ANY_ID, .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM, .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM | ADRENO_PREEMPTION, .pm4fw_name = "a530_pm4.fw", .pfpfw_name = "a530_pfp.fw", .zap_name = "a530_zap", Loading drivers/gpu/msm/adreno.c +6 −0 Original line number Diff line number Diff line Loading @@ -1470,6 +1470,12 @@ static int _adreno_start(struct adreno_device *adreno_dev) if (status) goto error_mmu_off; if (gpudev->hw_init) { status = gpudev->hw_init(adreno_dev); if (status) goto error_mmu_off; } /* Set up LM before initializing the GPMU */ if (gpudev->lm_init) gpudev->lm_init(adreno_dev); Loading drivers/gpu/msm/adreno.h +3 −4 Original line number Diff line number Diff line Loading @@ -332,6 +332,7 @@ struct adreno_device { const struct firmware *lm_fw; uint32_t *lm_sequence; uint32_t lm_size; struct kgsl_memdesc preemption_counters; }; /** Loading Loading @@ -637,6 +638,7 @@ struct adreno_gpudev { void (*snapshot)(struct adreno_device *, struct kgsl_snapshot *); void (*gpudev_init)(struct adreno_device *); int (*rb_init)(struct adreno_device *, struct adreno_ringbuffer *); int (*hw_init)(struct adreno_device *); int (*microcode_read)(struct adreno_device *); int (*microcode_load)(struct adreno_device *, unsigned int start_type); void (*perfcounter_init)(struct adreno_device *); Loading @@ -663,11 +665,8 @@ struct adreno_gpudev { int (*preemption_token)(struct adreno_device *, struct adreno_ringbuffer *, unsigned int *, uint64_t gpuaddr); void (*preemption_start)(struct adreno_device *, struct adreno_ringbuffer *); void (*preemption_save)(struct adreno_device *, struct adreno_ringbuffer *); void (*preemption_init)(struct adreno_device *); void (*preemption_schedule)(struct adreno_device *); }; struct log_field { Loading drivers/gpu/msm/adreno_a4xx.c +344 −2 Original line number Diff line number Diff line Loading @@ -1771,6 +1771,349 @@ static struct adreno_snapshot_data a4xx_snapshot_data = { .sect_sizes = &a4xx_snap_sizes, }; /** * a4xx_preempt_trig_state() - Schedule preemption in TRIGGERRED * state * @adreno_dev: Device which is in TRIGGERRED state */ static void a4xx_preempt_trig_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); unsigned int rbbase, val; /* * Hardware not yet idle means that preemption interrupt * may still occur, nothing to do here until interrupt signals * completion of preemption, just return here */ if (!adreno_hw_isidle(adreno_dev)) return; /* * We just changed states, reschedule dispatcher to change * preemption states */ if (ADRENO_DISPATCHER_PREEMPT_TRIGGERED != atomic_read(&dispatcher->preemption_state)) { adreno_dispatcher_schedule(device); return; } /* * H/W is idle and we did not get a preemption interrupt, may * be device went idle w/o encountering any preempt token or * we already preempted w/o interrupt */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); /* Did preemption occur, if so then change states and return */ if (rbbase != adreno_dev->cur_rb->buffer_desc.gpuaddr) { adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val); if (val && rbbase == adreno_dev->next_rb->buffer_desc.gpuaddr) { KGSL_DRV_INFO(device, "Preemption completed without interrupt\n"); trace_adreno_hw_preempt_trig_to_comp(adreno_dev->cur_rb, adreno_dev->next_rb); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_COMPLETE); adreno_dispatcher_schedule(device); return; } adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); /* reschedule dispatcher to take care of the fault */ adreno_dispatcher_schedule(device); return; } /* * Check if preempt token was submitted after preemption trigger, if so * then preemption should have occurred, since device is already idle it * means something went wrong - trigger FT */ if (dispatcher->preempt_token_submit) { adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); /* reschedule dispatcher to take care of the fault */ adreno_dispatcher_schedule(device); return; } /* * Preempt token was not submitted after preemption trigger so device * may have gone idle before preemption could occur, if there are * commands that got submitted to current RB after triggering preemption * then submit them as those commands may have a preempt token in them */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &adreno_dev->cur_rb->rptr); if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) { /* * Memory barrier before informing the * hardware of new commands */ mb(); kgsl_pwrscale_busy(device); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, adreno_dev->cur_rb->wptr); return; } /* Submit preempt token to make preemption happen */ if (adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, NULL, 0)) BUG(); if (adreno_ringbuffer_submit_preempt_token(adreno_dev->cur_rb, adreno_dev->next_rb)) BUG(); dispatcher->preempt_token_submit = 1; adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; trace_adreno_hw_preempt_token_submit(adreno_dev->cur_rb, adreno_dev->next_rb); } /** * a4xx_preempt_clear_state() - Schedule preemption in * CLEAR state. Preemption can be issued in this state. * @adreno_dev: Device which is in CLEAR state */ static void a4xx_preempt_clear_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); struct adreno_dispatcher_cmdqueue *dispatch_tempq; struct kgsl_cmdbatch *cmdbatch; struct adreno_ringbuffer *highest_busy_rb; int switch_low_to_high; int ret; /* Device not awake means there is nothing to do */ if (!kgsl_state_is_awake(device)) return; /* keep updating the current rptr when preemption is clear */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &(adreno_dev->cur_rb->rptr)); highest_busy_rb = adreno_dispatcher_get_highest_busy_rb(adreno_dev); if (!highest_busy_rb) return; switch_low_to_high = adreno_compare_prio_level( highest_busy_rb->id, adreno_dev->cur_rb->id); /* already current then return */ if (!switch_low_to_high) return; if (switch_low_to_high < 0) { /* * if switching to lower priority make sure that the rptr and * wptr are equal, when the lower rb is not starved */ if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) return; /* * switch to default context because when we switch back * to higher context then its not known which pt will * be current, so by making it default here the next * commands submitted will set the right pt */ ret = adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, NULL, 0); /* * lower priority RB has to wait until space opens up in * higher RB */ if (ret) return; adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DISABLE, 1); } /* * setup registers to do the switch to highest priority RB * which is not empty or may be starving away(poor thing) */ a4xx_preemption_start(adreno_dev, highest_busy_rb); /* turn on IOMMU as the preemption may trigger pt switch */ kgsl_mmu_enable_clk(&device->mmu); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_TRIGGERED); adreno_dev->next_rb = highest_busy_rb; mod_timer(&dispatcher->preempt_timer, jiffies + msecs_to_jiffies(ADRENO_DISPATCH_PREEMPT_TIMEOUT)); trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb, adreno_dev->next_rb); /* issue PREEMPT trigger */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); /* * IOMMU clock can be safely switched off after the timestamp * of the first command in the new rb */ dispatch_tempq = &adreno_dev->next_rb->dispatch_q; if (dispatch_tempq->head != dispatch_tempq->tail) cmdbatch = dispatch_tempq->cmd_q[dispatch_tempq->head]; else cmdbatch = 0; if (cmdbatch) adreno_ringbuffer_mmu_disable_clk_on_ts(device, adreno_dev->next_rb, cmdbatch->global_ts); else adreno_ringbuffer_mmu_disable_clk_on_ts(device, adreno_dev->next_rb, adreno_dev->next_rb->timestamp); /* submit preempt token packet to ensure preemption */ if (switch_low_to_high < 0) { ret = adreno_ringbuffer_submit_preempt_token( adreno_dev->cur_rb, adreno_dev->next_rb); /* * unexpected since we are submitting this when rptr = wptr, * this was checked above already */ BUG_ON(ret); dispatcher->preempt_token_submit = 1; adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; } else { dispatcher->preempt_token_submit = 0; adreno_dispatcher_schedule(device); adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; } } /** * a4xx_preempt_complete_state() - Schedule preemption in * COMPLETE state * @adreno_dev: Device which is in COMPLETE state */ static void a4xx_preempt_complete_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); struct adreno_dispatcher_cmdqueue *dispatch_q; unsigned int wptr, rbbase; unsigned int val, val1; del_timer_sync(&dispatcher->preempt_timer); adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &val); adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val1); if (val || !val1) { KGSL_DRV_ERR(device, "Invalid state after preemption CP_PREEMPT: %08x, CP_PREEMPT_DEBUG: %08x\n", val, val1); adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); adreno_dispatcher_schedule(device); return; } adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); if (rbbase != adreno_dev->next_rb->buffer_desc.gpuaddr) { KGSL_DRV_ERR(device, "RBBASE incorrect after preemption, expected %x got %016llx\b", rbbase, adreno_dev->next_rb->buffer_desc.gpuaddr); adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); adreno_dispatcher_schedule(device); return; } a4xx_preemption_save(adreno_dev, adreno_dev->cur_rb); dispatch_q = &(adreno_dev->cur_rb->dispatch_q); /* new RB is the current RB */ trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb, adreno_dev->cur_rb); adreno_dev->prev_rb = adreno_dev->cur_rb; adreno_dev->cur_rb = adreno_dev->next_rb; adreno_dev->cur_rb->preempted_midway = 0; adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; adreno_dev->next_rb = 0; if (adreno_disp_preempt_fair_sched) { /* starved rb is now scheduled so unhalt dispatcher */ if (ADRENO_DISPATCHER_RB_STARVE_TIMER_ELAPSED == adreno_dev->cur_rb->starve_timer_state) adreno_put_gpu_halt(adreno_dev); adreno_dev->cur_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_SCHEDULED; adreno_dev->cur_rb->sched_timer = jiffies; /* * If the outgoing RB is has commands then set the * busy time for it */ if (adreno_dev->prev_rb->rptr != adreno_dev->prev_rb->wptr) { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT; adreno_dev->prev_rb->sched_timer = jiffies; } else { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT; } } atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_CLEAR); if (adreno_compare_prio_level(adreno_dev->prev_rb->id, adreno_dev->cur_rb->id) < 0) { if (adreno_dev->prev_rb->wptr_preempt_end != adreno_dev->prev_rb->rptr) adreno_dev->prev_rb->preempted_midway = 1; } else if (adreno_dev->prev_rb->wptr_preempt_end != adreno_dev->prev_rb->rptr) { BUG(); } /* submit wptr if required for new rb */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); if (adreno_dev->cur_rb->wptr != wptr) { kgsl_pwrscale_busy(device); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, adreno_dev->cur_rb->wptr); } /* clear preemption register */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, 0); adreno_preempt_process_dispatch_queue(adreno_dev, dispatch_q); } static void a4xx_preemption_schedule( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); if (!adreno_is_preemption_enabled(adreno_dev)) return; mutex_lock(&device->mutex); switch (atomic_read(&dispatcher->preemption_state)) { case ADRENO_DISPATCHER_PREEMPT_CLEAR: a4xx_preempt_clear_state(adreno_dev); break; case ADRENO_DISPATCHER_PREEMPT_TRIGGERED: a4xx_preempt_trig_state(adreno_dev); /* * if we transitioned to next state then fall-through * processing to next state */ if (!adreno_preempt_state(adreno_dev, ADRENO_DISPATCHER_PREEMPT_COMPLETE)) break; case ADRENO_DISPATCHER_PREEMPT_COMPLETE: a4xx_preempt_complete_state(adreno_dev); break; default: BUG(); } mutex_unlock(&device->mutex); } struct adreno_gpudev adreno_a4xx_gpudev = { .reg_offsets = &a4xx_reg_offsets, .ft_perf_counters = a4xx_ft_perf_counters, Loading Loading @@ -1798,6 +2141,5 @@ struct adreno_gpudev adreno_a4xx_gpudev = { .regulator_disable = a4xx_regulator_disable, .preemption_pre_ibsubmit = a4xx_preemption_pre_ibsubmit, .preemption_token = a4xx_preemption_token, .preemption_start = a4xx_preemption_start, .preemption_save = a4xx_preemption_save, .preemption_schedule = a4xx_preemption_schedule, }; Loading
drivers/gpu/msm/adreno-gpulist.h +2 −1 Original line number Diff line number Diff line Loading @@ -140,7 +140,8 @@ static const struct adreno_gpu_core adreno_gpulist[] = { .major = 3, .minor = 0, .patchid = ANY_ID, .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM, .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM | ADRENO_PREEMPTION, .pm4fw_name = "a530_pm4.fw", .pfpfw_name = "a530_pfp.fw", .zap_name = "a530_zap", Loading
drivers/gpu/msm/adreno.c +6 −0 Original line number Diff line number Diff line Loading @@ -1470,6 +1470,12 @@ static int _adreno_start(struct adreno_device *adreno_dev) if (status) goto error_mmu_off; if (gpudev->hw_init) { status = gpudev->hw_init(adreno_dev); if (status) goto error_mmu_off; } /* Set up LM before initializing the GPMU */ if (gpudev->lm_init) gpudev->lm_init(adreno_dev); Loading
drivers/gpu/msm/adreno.h +3 −4 Original line number Diff line number Diff line Loading @@ -332,6 +332,7 @@ struct adreno_device { const struct firmware *lm_fw; uint32_t *lm_sequence; uint32_t lm_size; struct kgsl_memdesc preemption_counters; }; /** Loading Loading @@ -637,6 +638,7 @@ struct adreno_gpudev { void (*snapshot)(struct adreno_device *, struct kgsl_snapshot *); void (*gpudev_init)(struct adreno_device *); int (*rb_init)(struct adreno_device *, struct adreno_ringbuffer *); int (*hw_init)(struct adreno_device *); int (*microcode_read)(struct adreno_device *); int (*microcode_load)(struct adreno_device *, unsigned int start_type); void (*perfcounter_init)(struct adreno_device *); Loading @@ -663,11 +665,8 @@ struct adreno_gpudev { int (*preemption_token)(struct adreno_device *, struct adreno_ringbuffer *, unsigned int *, uint64_t gpuaddr); void (*preemption_start)(struct adreno_device *, struct adreno_ringbuffer *); void (*preemption_save)(struct adreno_device *, struct adreno_ringbuffer *); void (*preemption_init)(struct adreno_device *); void (*preemption_schedule)(struct adreno_device *); }; struct log_field { Loading
drivers/gpu/msm/adreno_a4xx.c +344 −2 Original line number Diff line number Diff line Loading @@ -1771,6 +1771,349 @@ static struct adreno_snapshot_data a4xx_snapshot_data = { .sect_sizes = &a4xx_snap_sizes, }; /** * a4xx_preempt_trig_state() - Schedule preemption in TRIGGERRED * state * @adreno_dev: Device which is in TRIGGERRED state */ static void a4xx_preempt_trig_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); unsigned int rbbase, val; /* * Hardware not yet idle means that preemption interrupt * may still occur, nothing to do here until interrupt signals * completion of preemption, just return here */ if (!adreno_hw_isidle(adreno_dev)) return; /* * We just changed states, reschedule dispatcher to change * preemption states */ if (ADRENO_DISPATCHER_PREEMPT_TRIGGERED != atomic_read(&dispatcher->preemption_state)) { adreno_dispatcher_schedule(device); return; } /* * H/W is idle and we did not get a preemption interrupt, may * be device went idle w/o encountering any preempt token or * we already preempted w/o interrupt */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); /* Did preemption occur, if so then change states and return */ if (rbbase != adreno_dev->cur_rb->buffer_desc.gpuaddr) { adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val); if (val && rbbase == adreno_dev->next_rb->buffer_desc.gpuaddr) { KGSL_DRV_INFO(device, "Preemption completed without interrupt\n"); trace_adreno_hw_preempt_trig_to_comp(adreno_dev->cur_rb, adreno_dev->next_rb); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_COMPLETE); adreno_dispatcher_schedule(device); return; } adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); /* reschedule dispatcher to take care of the fault */ adreno_dispatcher_schedule(device); return; } /* * Check if preempt token was submitted after preemption trigger, if so * then preemption should have occurred, since device is already idle it * means something went wrong - trigger FT */ if (dispatcher->preempt_token_submit) { adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); /* reschedule dispatcher to take care of the fault */ adreno_dispatcher_schedule(device); return; } /* * Preempt token was not submitted after preemption trigger so device * may have gone idle before preemption could occur, if there are * commands that got submitted to current RB after triggering preemption * then submit them as those commands may have a preempt token in them */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &adreno_dev->cur_rb->rptr); if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) { /* * Memory barrier before informing the * hardware of new commands */ mb(); kgsl_pwrscale_busy(device); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, adreno_dev->cur_rb->wptr); return; } /* Submit preempt token to make preemption happen */ if (adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, NULL, 0)) BUG(); if (adreno_ringbuffer_submit_preempt_token(adreno_dev->cur_rb, adreno_dev->next_rb)) BUG(); dispatcher->preempt_token_submit = 1; adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; trace_adreno_hw_preempt_token_submit(adreno_dev->cur_rb, adreno_dev->next_rb); } /** * a4xx_preempt_clear_state() - Schedule preemption in * CLEAR state. Preemption can be issued in this state. * @adreno_dev: Device which is in CLEAR state */ static void a4xx_preempt_clear_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); struct adreno_dispatcher_cmdqueue *dispatch_tempq; struct kgsl_cmdbatch *cmdbatch; struct adreno_ringbuffer *highest_busy_rb; int switch_low_to_high; int ret; /* Device not awake means there is nothing to do */ if (!kgsl_state_is_awake(device)) return; /* keep updating the current rptr when preemption is clear */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &(adreno_dev->cur_rb->rptr)); highest_busy_rb = adreno_dispatcher_get_highest_busy_rb(adreno_dev); if (!highest_busy_rb) return; switch_low_to_high = adreno_compare_prio_level( highest_busy_rb->id, adreno_dev->cur_rb->id); /* already current then return */ if (!switch_low_to_high) return; if (switch_low_to_high < 0) { /* * if switching to lower priority make sure that the rptr and * wptr are equal, when the lower rb is not starved */ if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) return; /* * switch to default context because when we switch back * to higher context then its not known which pt will * be current, so by making it default here the next * commands submitted will set the right pt */ ret = adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, NULL, 0); /* * lower priority RB has to wait until space opens up in * higher RB */ if (ret) return; adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DISABLE, 1); } /* * setup registers to do the switch to highest priority RB * which is not empty or may be starving away(poor thing) */ a4xx_preemption_start(adreno_dev, highest_busy_rb); /* turn on IOMMU as the preemption may trigger pt switch */ kgsl_mmu_enable_clk(&device->mmu); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_TRIGGERED); adreno_dev->next_rb = highest_busy_rb; mod_timer(&dispatcher->preempt_timer, jiffies + msecs_to_jiffies(ADRENO_DISPATCH_PREEMPT_TIMEOUT)); trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb, adreno_dev->next_rb); /* issue PREEMPT trigger */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); /* * IOMMU clock can be safely switched off after the timestamp * of the first command in the new rb */ dispatch_tempq = &adreno_dev->next_rb->dispatch_q; if (dispatch_tempq->head != dispatch_tempq->tail) cmdbatch = dispatch_tempq->cmd_q[dispatch_tempq->head]; else cmdbatch = 0; if (cmdbatch) adreno_ringbuffer_mmu_disable_clk_on_ts(device, adreno_dev->next_rb, cmdbatch->global_ts); else adreno_ringbuffer_mmu_disable_clk_on_ts(device, adreno_dev->next_rb, adreno_dev->next_rb->timestamp); /* submit preempt token packet to ensure preemption */ if (switch_low_to_high < 0) { ret = adreno_ringbuffer_submit_preempt_token( adreno_dev->cur_rb, adreno_dev->next_rb); /* * unexpected since we are submitting this when rptr = wptr, * this was checked above already */ BUG_ON(ret); dispatcher->preempt_token_submit = 1; adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; } else { dispatcher->preempt_token_submit = 0; adreno_dispatcher_schedule(device); adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; } } /** * a4xx_preempt_complete_state() - Schedule preemption in * COMPLETE state * @adreno_dev: Device which is in COMPLETE state */ static void a4xx_preempt_complete_state( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); struct adreno_dispatcher_cmdqueue *dispatch_q; unsigned int wptr, rbbase; unsigned int val, val1; del_timer_sync(&dispatcher->preempt_timer); adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &val); adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val1); if (val || !val1) { KGSL_DRV_ERR(device, "Invalid state after preemption CP_PREEMPT: %08x, CP_PREEMPT_DEBUG: %08x\n", val, val1); adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); adreno_dispatcher_schedule(device); return; } adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); if (rbbase != adreno_dev->next_rb->buffer_desc.gpuaddr) { KGSL_DRV_ERR(device, "RBBASE incorrect after preemption, expected %x got %016llx\b", rbbase, adreno_dev->next_rb->buffer_desc.gpuaddr); adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); adreno_dispatcher_schedule(device); return; } a4xx_preemption_save(adreno_dev, adreno_dev->cur_rb); dispatch_q = &(adreno_dev->cur_rb->dispatch_q); /* new RB is the current RB */ trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb, adreno_dev->cur_rb); adreno_dev->prev_rb = adreno_dev->cur_rb; adreno_dev->cur_rb = adreno_dev->next_rb; adreno_dev->cur_rb->preempted_midway = 0; adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; adreno_dev->next_rb = 0; if (adreno_disp_preempt_fair_sched) { /* starved rb is now scheduled so unhalt dispatcher */ if (ADRENO_DISPATCHER_RB_STARVE_TIMER_ELAPSED == adreno_dev->cur_rb->starve_timer_state) adreno_put_gpu_halt(adreno_dev); adreno_dev->cur_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_SCHEDULED; adreno_dev->cur_rb->sched_timer = jiffies; /* * If the outgoing RB is has commands then set the * busy time for it */ if (adreno_dev->prev_rb->rptr != adreno_dev->prev_rb->wptr) { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT; adreno_dev->prev_rb->sched_timer = jiffies; } else { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT; } } atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_CLEAR); if (adreno_compare_prio_level(adreno_dev->prev_rb->id, adreno_dev->cur_rb->id) < 0) { if (adreno_dev->prev_rb->wptr_preempt_end != adreno_dev->prev_rb->rptr) adreno_dev->prev_rb->preempted_midway = 1; } else if (adreno_dev->prev_rb->wptr_preempt_end != adreno_dev->prev_rb->rptr) { BUG(); } /* submit wptr if required for new rb */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); if (adreno_dev->cur_rb->wptr != wptr) { kgsl_pwrscale_busy(device); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, adreno_dev->cur_rb->wptr); } /* clear preemption register */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, 0); adreno_preempt_process_dispatch_queue(adreno_dev, dispatch_q); } static void a4xx_preemption_schedule( struct adreno_device *adreno_dev) { struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = &(adreno_dev->dev); if (!adreno_is_preemption_enabled(adreno_dev)) return; mutex_lock(&device->mutex); switch (atomic_read(&dispatcher->preemption_state)) { case ADRENO_DISPATCHER_PREEMPT_CLEAR: a4xx_preempt_clear_state(adreno_dev); break; case ADRENO_DISPATCHER_PREEMPT_TRIGGERED: a4xx_preempt_trig_state(adreno_dev); /* * if we transitioned to next state then fall-through * processing to next state */ if (!adreno_preempt_state(adreno_dev, ADRENO_DISPATCHER_PREEMPT_COMPLETE)) break; case ADRENO_DISPATCHER_PREEMPT_COMPLETE: a4xx_preempt_complete_state(adreno_dev); break; default: BUG(); } mutex_unlock(&device->mutex); } struct adreno_gpudev adreno_a4xx_gpudev = { .reg_offsets = &a4xx_reg_offsets, .ft_perf_counters = a4xx_ft_perf_counters, Loading Loading @@ -1798,6 +2141,5 @@ struct adreno_gpudev adreno_a4xx_gpudev = { .regulator_disable = a4xx_regulator_disable, .preemption_pre_ibsubmit = a4xx_preemption_pre_ibsubmit, .preemption_token = a4xx_preemption_token, .preemption_start = a4xx_preemption_start, .preemption_save = a4xx_preemption_save, .preemption_schedule = a4xx_preemption_schedule, };
