Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fa7f517c authored by Christian König's avatar Christian König Committed by Dave Airlie
Browse files

drm/radeon: rework page flip handling v4 



Instead of trying to flip inside the vblank period when
the buffer is idle, offload blocking for idle to a kernel
thread and program the flip directly into the hardware.

v2: add error handling, fix EBUSY handling
v3: add proper exclusive_lock handling
v4: update crtc->primary->fb when the flip actually happens

Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent b15eb4ea

drivers/gpu/drm/radeon/radeon.h

+9 −7
Original line number Original line Diff line number Diff line
@@ -676,14 +676,16 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell); 
 * IRQS.

 * IRQS.

 */

 */





struct radeon_unpin_work {

struct radeon_flip_work {

	struct work_struct work;

	struct work_struct		flip_work;

	struct work_struct		unpin_work;

	struct radeon_device		*rdev;

	struct radeon_device		*rdev;

	int				crtc_id;

	int				crtc_id;

	struct radeon_fence *fence;

	struct drm_framebuffer		*fb;

	struct drm_pending_vblank_event *event;

	struct drm_pending_vblank_event *event;

	struct radeon_bo		*old_rbo;

	struct radeon_bo		*old_rbo;

	u64 new_crtc_base;

	struct radeon_bo		*new_rbo;

	struct radeon_fence		*fence;

};

};





struct r500_irq_stat_regs {

struct r500_irq_stat_regs {

drivers/gpu/drm/radeon/radeon_display.c

+136 −103
Original line number Original line Diff line number Diff line
@@ -249,16 +249,21 @@ static void radeon_crtc_destroy(struct drm_crtc *crtc) 
	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);

	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);





	drm_crtc_cleanup(crtc);

	drm_crtc_cleanup(crtc);

	destroy_workqueue(radeon_crtc->flip_queue);

	kfree(radeon_crtc);

	kfree(radeon_crtc);

}

}





/*

/**

 * Handle unpin events outside the interrupt handler proper.

 * radeon_unpin_work_func - unpin old buffer object

 *

 * @__work - kernel work item

 *

 * Unpin the old frame buffer object outside of the interrupt handler

 */

 */

static void radeon_unpin_work_func(struct work_struct *__work)

static void radeon_unpin_work_func(struct work_struct *__work)

{

{

	struct radeon_unpin_work *work =

	struct radeon_flip_work *work =

		container_of(__work, struct radeon_unpin_work, work);

		container_of(__work, struct radeon_flip_work, unpin_work);

	int r;

	int r;





	/* unpin of the old buffer */

	/* unpin of the old buffer */
@@ -279,30 +284,19 @@ static void radeon_unpin_work_func(struct work_struct *__work) 
void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id)

void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id)

{

{

	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];

	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];

	struct radeon_unpin_work *work;

	struct radeon_flip_work *work;

	unsigned long flags;

	unsigned long flags;

	u32 update_pending;

	u32 update_pending;

	int vpos, hpos;

	int vpos, hpos;





	spin_lock_irqsave(&rdev->ddev->event_lock, flags);

	spin_lock_irqsave(&rdev->ddev->event_lock, flags);

	work = radeon_crtc->unpin_work;

	work = radeon_crtc->flip_work;

	if (work == NULL ||

	if (work == NULL) {

	    (work->fence && !radeon_fence_signaled(work->fence))) {

		spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

		spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

		return;

		return;

	}

	}

	/* New pageflip, or just completion of a previous one? */



	if (!radeon_crtc->deferred_flip_completion) {

		/* do the flip (mmio) */

		radeon_page_flip(rdev, crtc_id, work->new_crtc_base);

	update_pending = radeon_page_flip_pending(rdev, crtc_id);

	update_pending = radeon_page_flip_pending(rdev, crtc_id);

	} else {

		/* This is just a completion of a flip queued in crtc

		 * at last invocation. Make sure we go directly to

		 * completion routine.

		 */

		update_pending = 0;

	}





	/* Has the pageflip already completed in crtc, or is it certain

	/* Has the pageflip already completed in crtc, or is it certain

	 * to complete in this vblank?

	 * to complete in this vblank?
@@ -320,20 +314,10 @@ void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id) 
		 */

		 */

		update_pending = 0;

		update_pending = 0;

	}

	}

	if (update_pending) {

		/* crtc didn't flip in this target vblank interval,

		 * but flip is pending in crtc. It will complete it

		 * in next vblank interval, so complete the flip at

		 * next vblank irq.

		 */

		radeon_crtc->deferred_flip_completion = 1;

		spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

		return;

	} else {

	spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

	spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

	if (!update_pending)

		radeon_crtc_handle_flip(rdev, crtc_id);

		radeon_crtc_handle_flip(rdev, crtc_id);

}

}

}





/**

/**

 * radeon_crtc_handle_flip - page flip completed

 * radeon_crtc_handle_flip - page flip completed
@@ -346,7 +330,7 @@ void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id) 
void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)

void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)

{

{

	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];

	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];

	struct radeon_unpin_work *work;

	struct radeon_flip_work *work;

	unsigned long flags;

	unsigned long flags;





	/* this can happen at init */

	/* this can happen at init */
@@ -354,15 +338,14 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) 
		return;

		return;





	spin_lock_irqsave(&rdev->ddev->event_lock, flags);

	spin_lock_irqsave(&rdev->ddev->event_lock, flags);

	work = radeon_crtc->unpin_work;

	work = radeon_crtc->flip_work;

	if (work == NULL) {

	if (work == NULL) {

		spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

		spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);

		return;

		return;

	}

	}





	/* Pageflip (will be) certainly completed in this vblank. Clean up. */

	/* Pageflip completed. Clean up. */

	radeon_crtc->unpin_work = NULL;

	radeon_crtc->flip_work = NULL;

	radeon_crtc->deferred_flip_completion = 0;





	/* wakeup userspace */

	/* wakeup userspace */

	if (work->event)

	if (work->event)
@@ -372,83 +355,69 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) 




	radeon_fence_unref(&work->fence);

	radeon_fence_unref(&work->fence);

	radeon_irq_kms_pflip_irq_get(rdev, work->crtc_id);

	radeon_irq_kms_pflip_irq_get(rdev, work->crtc_id);

	schedule_work(&work->work);

	queue_work(radeon_crtc->flip_queue, &work->unpin_work);

}

}





static int radeon_crtc_page_flip(struct drm_crtc *crtc,

/**

				 struct drm_framebuffer *fb,

 * radeon_flip_work_func - page flip framebuffer

				 struct drm_pending_vblank_event *event,

 *

				 uint32_t page_flip_flags)

 * @work - kernel work item

 *

 * Wait for the buffer object to become idle and do the actual page flip

 */

static void radeon_flip_work_func(struct work_struct *__work)

{

{

	struct drm_device *dev = crtc->dev;

	struct radeon_flip_work *work =

	struct radeon_device *rdev = dev->dev_private;

		container_of(__work, struct radeon_flip_work, flip_work);

	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);

	struct radeon_device *rdev = work->rdev;

	struct radeon_framebuffer *old_radeon_fb;

	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[work->crtc_id];

	struct radeon_framebuffer *new_radeon_fb;

	struct drm_gem_object *obj;

	struct radeon_bo *rbo;

	struct radeon_unpin_work *work;

	unsigned long flags;

	u32 tiling_flags, pitch_pixels;

	u64 base;

	int r;





	work = kzalloc(sizeof *work, GFP_KERNEL);

	struct drm_crtc *crtc = &radeon_crtc->base;

	if (work == NULL)

	struct drm_framebuffer *fb = work->fb;

		return -ENOMEM;





	work->event = event;

	uint32_t tiling_flags, pitch_pixels;

	work->rdev = rdev;

	uint64_t base;

	work->crtc_id = radeon_crtc->crtc_id;

	old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb);

	new_radeon_fb = to_radeon_framebuffer(fb);

	/* schedule unpin of the old buffer */

	obj = old_radeon_fb->obj;

	/* take a reference to the old object */

	drm_gem_object_reference(obj);

	rbo = gem_to_radeon_bo(obj);

	work->old_rbo = rbo;

	obj = new_radeon_fb->obj;

	rbo = gem_to_radeon_bo(obj);





	spin_lock(&rbo->tbo.bdev->fence_lock);

	unsigned long flags;

	if (rbo->tbo.sync_obj)

	int r;

		work->fence = radeon_fence_ref(rbo->tbo.sync_obj);

	spin_unlock(&rbo->tbo.bdev->fence_lock);





	INIT_WORK(&work->work, radeon_unpin_work_func);

        down_read(&rdev->exclusive_lock);

	while (work->fence) {

		r = radeon_fence_wait(work->fence, false);

		if (r == -EDEADLK) {

			up_read(&rdev->exclusive_lock);

			r = radeon_gpu_reset(rdev);

			down_read(&rdev->exclusive_lock);

		}





	/* We borrow the event spin lock for protecting unpin_work */

		if (r) {

	spin_lock_irqsave(&dev->event_lock, flags);

			DRM_ERROR("failed to wait on page flip fence (%d)!\n",

	if (radeon_crtc->unpin_work) {

				  r);

		DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");

			goto cleanup;

		r = -EBUSY;

		} else

		goto unlock_free;

			radeon_fence_unref(&work->fence);

	}

	}

	radeon_crtc->unpin_work = work;

	radeon_crtc->deferred_flip_completion = 0;

	spin_unlock_irqrestore(&dev->event_lock, flags);





	/* pin the new buffer */

	/* pin the new buffer */

	DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",

	DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",

			 work->old_rbo, rbo);

			 work->old_rbo, work->new_rbo);





	r = radeon_bo_reserve(rbo, false);

	r = radeon_bo_reserve(work->new_rbo, false);

	if (unlikely(r != 0)) {

	if (unlikely(r != 0)) {

		DRM_ERROR("failed to reserve new rbo buffer before flip\n");

		DRM_ERROR("failed to reserve new rbo buffer before flip\n");

		goto pflip_cleanup;

		goto cleanup;

	}

	}

	/* Only 27 bit offset for legacy CRTC */

	/* Only 27 bit offset for legacy CRTC */

	r = radeon_bo_pin_restricted(rbo, RADEON_GEM_DOMAIN_VRAM,

	r = radeon_bo_pin_restricted(work->new_rbo, RADEON_GEM_DOMAIN_VRAM,

				     ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, &base);

				     ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, &base);

	if (unlikely(r != 0)) {

	if (unlikely(r != 0)) {

		radeon_bo_unreserve(rbo);

		radeon_bo_unreserve(work->new_rbo);

		r = -EINVAL;

		r = -EINVAL;

		DRM_ERROR("failed to pin new rbo buffer before flip\n");

		DRM_ERROR("failed to pin new rbo buffer before flip\n");

		goto pflip_cleanup;

		goto cleanup;

	}

	}

	radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);

	radeon_bo_get_tiling_flags(work->new_rbo, &tiling_flags, NULL);

	radeon_bo_unreserve(rbo);

	radeon_bo_unreserve(work->new_rbo);





	if (!ASIC_IS_AVIVO(rdev)) {

	if (!ASIC_IS_AVIVO(rdev)) {

		/* crtc offset is from display base addr not FB location */

		/* crtc offset is from display base addr not FB location */
@@ -486,9 +455,8 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc, 
		base &= ~7;

		base &= ~7;

	}

	}





	spin_lock_irqsave(&dev->event_lock, flags);

	/* We borrow the event spin lock for protecting flip_work */

	work->new_crtc_base = base;

	spin_lock_irqsave(&crtc->dev->event_lock, flags);

	spin_unlock_irqrestore(&dev->event_lock, flags);





	/* update crtc fb */

	/* update crtc fb */

	crtc->primary->fb = fb;

	crtc->primary->fb = fb;
@@ -496,18 +464,82 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc, 
	/* set the proper interrupt */

	/* set the proper interrupt */

	radeon_irq_kms_pflip_irq_get(rdev, radeon_crtc->crtc_id);

	radeon_irq_kms_pflip_irq_get(rdev, radeon_crtc->crtc_id);





	return 0;

	/* do the flip (mmio) */

	radeon_page_flip(rdev, radeon_crtc->crtc_id, base);



	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);

	up_read(&rdev->exclusive_lock);





pflip_cleanup:

	return;

	spin_lock_irqsave(&dev->event_lock, flags);



	radeon_crtc->unpin_work = NULL;

cleanup:

unlock_free:

	drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);

	spin_unlock_irqrestore(&dev->event_lock, flags);

	drm_gem_object_unreference_unlocked(old_radeon_fb->obj);

	radeon_fence_unref(&work->fence);

	radeon_fence_unref(&work->fence);

	kfree(work);

	kfree(work);

	up_read(&rdev->exclusive_lock);

}



static int radeon_crtc_page_flip(struct drm_crtc *crtc,

				 struct drm_framebuffer *fb,

				 struct drm_pending_vblank_event *event,

				 uint32_t page_flip_flags)

{

	struct drm_device *dev = crtc->dev;

	struct radeon_device *rdev = dev->dev_private;

	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);

	struct radeon_framebuffer *old_radeon_fb;

	struct radeon_framebuffer *new_radeon_fb;

	struct drm_gem_object *obj;

	struct radeon_flip_work *work;

	unsigned long flags;



	work = kzalloc(sizeof *work, GFP_KERNEL);

	if (work == NULL)

		return -ENOMEM;



	INIT_WORK(&work->flip_work, radeon_flip_work_func);

	INIT_WORK(&work->unpin_work, radeon_unpin_work_func);





	return r;

	work->rdev = rdev;

	work->crtc_id = radeon_crtc->crtc_id;

	work->fb = fb;

	work->event = event;



	/* schedule unpin of the old buffer */

	old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb);

	obj = old_radeon_fb->obj;



	/* take a reference to the old object */

	drm_gem_object_reference(obj);

	work->old_rbo = gem_to_radeon_bo(obj);



	new_radeon_fb = to_radeon_framebuffer(fb);

	obj = new_radeon_fb->obj;

	work->new_rbo = gem_to_radeon_bo(obj);



	spin_lock(&work->new_rbo->tbo.bdev->fence_lock);

	if (work->new_rbo->tbo.sync_obj)

		work->fence = radeon_fence_ref(work->new_rbo->tbo.sync_obj);

	spin_unlock(&work->new_rbo->tbo.bdev->fence_lock);



	/* We borrow the event spin lock for protecting flip_work */

	spin_lock_irqsave(&crtc->dev->event_lock, flags);



	if (radeon_crtc->flip_work) {

		DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");

		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);

		drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);

		radeon_fence_unref(&work->fence);

		kfree(work);

		return -EBUSY;

	}

	radeon_crtc->flip_work = work;



	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);



	queue_work(radeon_crtc->flip_queue, &work->flip_work);



	return 0;

}

}





static int

static int
@@ -577,6 +609,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index) 




	drm_mode_crtc_set_gamma_size(&radeon_crtc->base, 256);

	drm_mode_crtc_set_gamma_size(&radeon_crtc->base, 256);

	radeon_crtc->crtc_id = index;

	radeon_crtc->crtc_id = index;

	radeon_crtc->flip_queue = create_singlethread_workqueue("radeon-crtc");

	rdev->mode_info.crtcs[index] = radeon_crtc;

	rdev->mode_info.crtcs[index] = radeon_crtc;





	if (rdev->family >= CHIP_BONAIRE) {

	if (rdev->family >= CHIP_BONAIRE) {

drivers/gpu/drm/radeon/radeon_mode.h

+2 −2
Original line number Original line Diff line number Diff line
@@ -325,8 +325,8 @@ struct radeon_crtc { 
	struct drm_display_mode native_mode;

	struct drm_display_mode native_mode;

	int pll_id;

	int pll_id;

	/* page flipping */

	/* page flipping */

	struct radeon_unpin_work *unpin_work;

	struct workqueue_struct *flip_queue;

	int deferred_flip_completion;

	struct radeon_flip_work *flip_work;

	/* pll sharing */

	/* pll sharing */

	struct radeon_atom_ss ss;

	struct radeon_atom_ss ss;

	bool ss_enabled;

	bool ss_enabled;