Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1f1dbe35 authored by Sushmita Susheelendra's avatar Sushmita Susheelendra
Browse files

drm/msm: Use mmu notifiers to track SVM range invalidations 



SVM buffer objects share the same virtual address on
both the CPU and GPU. Register for notifications when
SVM address ranges are unmapped on the CPU. When such
a notification is received, unmap the corresponding
SVM objects from the SMMU, after waiting on the most
recent fence that uses them. The notifier struct is
reference counted starting with the creation of the
first SVM bo in the process and is released when the
last SVM bo is freed.

Change-Id: I01f590d21fd1d146f5324539e5041f03653f858a
Signed-off-by: default avatarSushmita Susheelendra <ssusheel@codeaurora.org>
parent 0a367f63

drivers/gpu/drm/msm/Kconfig

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ config DRM_MSM 
	select QCOM_SCM

	select BACKLIGHT_CLASS_DEVICE

	select MSM_EXT_DISPLAY

	select MMU_NOTIFIER

	select INTERVAL_TREE

	default y

	help

	  DRM/KMS driver for MSM/snapdragon.

drivers/gpu/drm/msm/msm_drv.c

+2 −0
Original line number Diff line number Diff line
@@ -391,6 +391,8 @@ static int msm_load(struct drm_device *dev, unsigned long flags) 
	INIT_LIST_HEAD(&priv->vblank_ctrl.event_list);

	init_kthread_work(&priv->vblank_ctrl.work, vblank_ctrl_worker);

	spin_lock_init(&priv->vblank_ctrl.lock);

	hash_init(priv->mn_hash);

	mutex_init(&priv->mn_lock);



	drm_mode_config_init(dev);

drivers/gpu/drm/msm/msm_drv.h

+7 −1
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#include <linux/of_graph.h>

#include <linux/of_device.h>

#include <linux/sde_io_util.h>

#include <linux/hashtable.h>

#include <asm/sizes.h>

#include <linux/kthread.h>
@@ -325,6 +326,11 @@ struct msm_drm_private { 
	unsigned int num_connectors;

	struct drm_connector *connectors[MAX_CONNECTORS];



	/* hash to store mm_struct to msm_mmu_notifier mappings */

	DECLARE_HASHTABLE(mn_hash, 7);

	/* protects mn_hash and the msm_mmu_notifier for the process */

	struct mutex mn_lock;



	/* Properties */

	struct drm_property *plane_property[PLANE_PROP_COUNT];

	struct drm_property *crtc_property[CRTC_PROP_COUNT];
@@ -404,7 +410,7 @@ void msm_update_fence(struct drm_device *dev, uint32_t fence); 


void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,

		struct msm_gem_vma *vma, struct sg_table *sgt,

		void *priv);

		void *priv, bool invalidated);

int msm_gem_map_vma(struct msm_gem_address_space *aspace,

		struct msm_gem_vma *vma, struct sg_table *sgt,

		void *priv, unsigned int flags);

drivers/gpu/drm/msm/msm_gem.c

+224 −3
Original line number Diff line number Diff line
@@ -25,6 +25,129 @@ 
#include "msm_gpu.h"

#include "msm_mmu.h"



static void msm_gem_mn_free(struct kref *refcount)

{

	struct msm_mmu_notifier *msm_mn = container_of(refcount,

			struct msm_mmu_notifier, refcount);



	mmu_notifier_unregister(&msm_mn->mn, msm_mn->mm);

	hash_del(&msm_mn->node);



	kfree(msm_mn);

}



static int msm_gem_mn_get(struct msm_mmu_notifier *msm_mn)

{

	if (msm_mn)

		return kref_get_unless_zero(&msm_mn->refcount);

	return 0;

}



static void msm_gem_mn_put(struct msm_mmu_notifier *msm_mn)

{

	if (msm_mn) {

		struct msm_drm_private *msm_dev = msm_mn->msm_dev;



		mutex_lock(&msm_dev->mn_lock);

		kref_put(&msm_mn->refcount, msm_gem_mn_free);

		mutex_unlock(&msm_dev->mn_lock);

	}

}



void msm_mn_invalidate_range_start(struct mmu_notifier *mn,

		struct mm_struct *mm, unsigned long start, unsigned long end);



static const struct mmu_notifier_ops msm_mn_ops = {

	.invalidate_range_start = msm_mn_invalidate_range_start,

};



static struct msm_mmu_notifier *

msm_gem_mn_find(struct msm_drm_private *msm_dev, struct mm_struct *mm,

		struct msm_gem_address_space *aspace)

{

	struct msm_mmu_notifier *msm_mn;

	int ret = 0;



	mutex_lock(&msm_dev->mn_lock);

	hash_for_each_possible(msm_dev->mn_hash, msm_mn, node,

			(unsigned long) mm) {

		if (msm_mn->mm == mm) {

			if (!msm_gem_mn_get(msm_mn)) {

				ret = -EINVAL;

				goto fail;

			}

			mutex_unlock(&msm_dev->mn_lock);

			return msm_mn;

		}

	}



	msm_mn = kzalloc(sizeof(*msm_mn), GFP_KERNEL);

	if (!msm_mn) {

		ret = -ENOMEM;

		goto fail;

	}



	msm_mn->mm = current->mm;

	msm_mn->mn.ops = &msm_mn_ops;

	ret = mmu_notifier_register(&msm_mn->mn, msm_mn->mm);

	if (ret) {

		kfree(msm_mn);

		goto fail;

	}



	msm_mn->svm_tree = RB_ROOT;

	spin_lock_init(&msm_mn->svm_tree_lock);

	kref_init(&msm_mn->refcount);

	msm_mn->msm_dev = msm_dev;



	/* Insert the msm_mn into the hash */

	hash_add(msm_dev->mn_hash, &msm_mn->node, (unsigned long) msm_mn->mm);

	mutex_unlock(&msm_dev->mn_lock);



	return msm_mn;



fail:

	mutex_unlock(&msm_dev->mn_lock);

	return ERR_PTR(ret);

}



static int msm_gem_mn_register(struct msm_gem_svm_object *msm_svm_obj,

		struct msm_gem_address_space *aspace)

{

	struct drm_gem_object *obj = &msm_svm_obj->msm_obj_base.base;

	struct msm_drm_private *msm_dev = obj->dev->dev_private;

	struct msm_mmu_notifier *msm_mn;



	msm_svm_obj->mm = current->mm;

	msm_svm_obj->svm_node.start = msm_svm_obj->hostptr;

	msm_svm_obj->svm_node.last = msm_svm_obj->hostptr + obj->size - 1;



	msm_mn = msm_gem_mn_find(msm_dev, msm_svm_obj->mm, aspace);

	if (IS_ERR(msm_mn))

		return PTR_ERR(msm_mn);



	msm_svm_obj->msm_mn = msm_mn;



	spin_lock(&msm_mn->svm_tree_lock);

	interval_tree_insert(&msm_svm_obj->svm_node, &msm_mn->svm_tree);

	spin_unlock(&msm_mn->svm_tree_lock);



	return 0;

}



static void msm_gem_mn_unregister(struct msm_gem_svm_object *msm_svm_obj)

{

	struct msm_mmu_notifier *msm_mn = msm_svm_obj->msm_mn;



	/* invalid: bo already unregistered */

	if (!msm_mn || msm_svm_obj->invalid)

		return;



	spin_lock(&msm_mn->svm_tree_lock);

	interval_tree_remove(&msm_svm_obj->svm_node, &msm_mn->svm_tree);

	spin_unlock(&msm_mn->svm_tree_lock);

}



static int protect_pages(struct msm_gem_object *msm_obj)

{

	int perm = PERM_READ | PERM_WRITE;
@@ -354,14 +477,21 @@ static void 
put_iova(struct drm_gem_object *obj)

{

	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	struct msm_gem_svm_object *msm_svm_obj;

	struct msm_gem_vma *domain, *tmp;

	bool invalid = false;



	WARN_ON(!mutex_is_locked(&msm_obj->lock));



	if (msm_obj->flags & MSM_BO_SVM) {

		msm_svm_obj = to_msm_svm_obj(msm_obj);

		invalid = msm_svm_obj->invalid;

	}



	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {

		if (iommu_present(&platform_bus_type)) {

			msm_gem_unmap_vma(domain->aspace, domain,

				msm_obj->sgt, get_dmabuf_ptr(obj));

				msm_obj->sgt, get_dmabuf_ptr(obj), invalid);

		}



		obj_remove_domain(domain);
@@ -676,8 +806,14 @@ void msm_gem_free_object(struct drm_gem_object *obj) 


	list_del(&msm_obj->mm_list);



	mutex_lock(&msm_obj->lock);

	/* Unregister SVM object from mmu notifications */

	if (msm_obj->flags & MSM_BO_SVM) {

		msm_gem_mn_unregister(msm_svm_obj);

		msm_gem_mn_put(msm_svm_obj->msm_mn);

		msm_svm_obj->msm_mn = NULL;

	}



	mutex_lock(&msm_obj->lock);

	put_iova(obj);



	if (obj->import_attach) {
@@ -947,6 +1083,11 @@ struct drm_gem_object *msm_gem_svm_new(struct drm_device *dev, 


	msm_svm_obj = to_msm_svm_obj(msm_obj);

	msm_svm_obj->hostptr = hostptr;

	msm_svm_obj->invalid = false;



	ret = msm_gem_mn_register(msm_svm_obj, aspace);

	if (ret)

		goto fail;



	/*

	 * Get physical pages and map into smmu in the ioctl itself.
@@ -1038,7 +1179,8 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev, 
	/* OR the passed in flags */

	msm_obj->flags |= flags;



	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);

	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages,

			NULL, npages);

	if (ret) {

		mutex_unlock(&msm_obj->lock);

		goto fail;
@@ -1053,3 +1195,82 @@ fail: 


	return ERR_PTR(ret);

}



/* Timeout in ms, long enough so we are sure the GPU is hung */

#define SVM_OBJ_WAIT_TIMEOUT 10000

static void invalidate_svm_object(struct msm_gem_svm_object *msm_svm_obj)

{

	struct msm_gem_object *msm_obj = &msm_svm_obj->msm_obj_base;

	struct drm_device *dev = msm_obj->base.dev;

	struct msm_gem_vma *domain, *tmp;

	uint32_t fence;

	int ret;



	if (is_active(msm_obj)) {

		ktime_t timeout = ktime_add_ms(ktime_get(),

				SVM_OBJ_WAIT_TIMEOUT);



		/* Get the most recent fence that touches the object */

		fence = msm_gem_fence(msm_obj, MSM_PREP_READ | MSM_PREP_WRITE);



		/* Wait for the fence to retire */

		ret = msm_wait_fence(dev, fence, &timeout, true);

		if (ret)

			/* The GPU could be hung! Not much we can do */

			dev_err(dev->dev, "drm: Error (%d) waiting for svm object: 0x%llx",

					ret, msm_svm_obj->hostptr);

	}



	/* GPU is done, unmap object from SMMU */

	mutex_lock(&msm_obj->lock);

	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {

		struct msm_gem_address_space *aspace = domain->aspace;



		if (domain->iova)

			aspace->mmu->funcs->unmap(aspace->mmu,

					domain->iova, msm_obj->sgt,

					get_dmabuf_ptr(&msm_obj->base));

	}

	/* Let go of the physical pages */

	put_pages(&msm_obj->base);

	mutex_unlock(&msm_obj->lock);

}



void msm_mn_invalidate_range_start(struct mmu_notifier *mn,

		struct mm_struct *mm, unsigned long start, unsigned long end)

{

	struct msm_mmu_notifier *msm_mn =

		container_of(mn, struct msm_mmu_notifier, mn);

	struct interval_tree_node *itn = NULL;

	struct msm_gem_svm_object *msm_svm_obj;

	struct drm_gem_object *obj;

	LIST_HEAD(inv_list);



	if (!msm_gem_mn_get(msm_mn))

		return;



	spin_lock(&msm_mn->svm_tree_lock);

	itn = interval_tree_iter_first(&msm_mn->svm_tree, start, end - 1);

	while (itn) {

		msm_svm_obj = container_of(itn,

				struct msm_gem_svm_object, svm_node);

		obj = &msm_svm_obj->msm_obj_base.base;



		if (kref_get_unless_zero(&obj->refcount))

			list_add(&msm_svm_obj->lnode, &inv_list);



		itn = interval_tree_iter_next(itn, start, end - 1);

	}

	spin_unlock(&msm_mn->svm_tree_lock);



	list_for_each_entry(msm_svm_obj, &inv_list, lnode) {

		obj = &msm_svm_obj->msm_obj_base.base;

		/* Unregister SVM object from mmu notifications */

		msm_gem_mn_unregister(msm_svm_obj);

		msm_svm_obj->invalid = true;

		invalidate_svm_object(msm_svm_obj);

		drm_gem_object_unreference_unlocked(obj);

	}



	msm_gem_mn_put(msm_mn);

}

drivers/gpu/drm/msm/msm_gem.h

+18 −0
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ 


#include <linux/kref.h>

#include <linux/reservation.h>

#include <linux/mmu_notifier.h>

#include <linux/interval_tree.h>

#include "msm_drv.h"



/* Additional internal-use only BO flags: */
@@ -86,9 +88,25 @@ struct msm_gem_object { 
};

#define to_msm_bo(x) container_of(x, struct msm_gem_object, base)



struct msm_mmu_notifier {

	struct mmu_notifier mn;

	struct mm_struct *mm; /* mm_struct owning the mmu notifier mn */

	struct hlist_node node;

	struct rb_root svm_tree; /* interval tree holding all svm bos */

	spinlock_t svm_tree_lock; /* Protects svm_tree*/

	struct msm_drm_private *msm_dev;

	struct kref refcount;

};



struct msm_gem_svm_object {

	struct msm_gem_object msm_obj_base;

	uint64_t hostptr;

	struct mm_struct *mm; /* mm_struct the svm bo belongs to */

	struct interval_tree_node svm_node;

	struct msm_mmu_notifier *msm_mn;

	struct list_head lnode;

	/* bo has been unmapped on CPU, cannot be part of GPU submits */

	bool invalid;

};



#define to_msm_svm_obj(x) \