Revert "drm/amdgpu: use HMM callback to replace mmu notifier" (296bb163) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/gpu/drm/amd/amdgpu/Kconfig

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK
	config DRM_AMDGPU_USERPTR		config DRM_AMDGPU_USERPTR
	bool "Always enable userptr write support"		bool "Always enable userptr write support"
	depends on DRM_AMDGPU		depends on DRM_AMDGPU
	select HMM_MIRROR		select MMU_NOTIFIER
	help		help
	This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it		This option selects CONFIG_MMU_NOTIFIER if it isn't already
	isn't already selected to enabled full userptr support.		selected to enabled full userptr support.

	config DRM_AMDGPU_GART_DEBUGFS		config DRM_AMDGPU_GART_DEBUGFS
	bool "Allow GART access through debugfs"		bool "Allow GART access through debugfs"

drivers/gpu/drm/amd/amdgpu/Makefile

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -173,7 +173,7 @@ endif
	amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o		amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
	amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o		amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
	amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o		amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
	amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o		amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o

	include $(FULL_AMD_PATH)/powerplay/Makefile		include $(FULL_AMD_PATH)/powerplay/Makefile

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c

+93 −67

Original line number	Original line	Diff line number	Diff line
	@@ -45,7 +45,7 @@

	#include <linux/firmware.h>		#include <linux/firmware.h>
	#include <linux/module.h>		#include <linux/module.h>
	#include <linux/hmm.h>		#include <linux/mmu_notifier.h>
	#include <linux/interval_tree.h>		#include <linux/interval_tree.h>
	#include <drm/drmP.h>		#include <drm/drmP.h>
	#include <drm/drm.h>		#include <drm/drm.h>
	@@ -58,12 +58,14 @@
	*		*
	* @adev: amdgpu device pointer		* @adev: amdgpu device pointer
	* @mm: process address space		* @mm: process address space
			* @mn: MMU notifier structure
	* @type: type of MMU notifier		* @type: type of MMU notifier
	* @work: destruction work item		* @work: destruction work item
	* @node: hash table node to find structure by adev and mn		* @node: hash table node to find structure by adev and mn
	* @lock: rw semaphore protecting the notifier nodes		* @lock: rw semaphore protecting the notifier nodes
	* @objects: interval tree containing amdgpu_mn_nodes		* @objects: interval tree containing amdgpu_mn_nodes
	* @mirror: HMM mirror function support		* @read_lock: mutex for recursive locking of @lock
			* @recursion: depth of recursion
	*		*
	* Data for each amdgpu device and process address space.		* Data for each amdgpu device and process address space.
	*/		*/
	@@ -71,6 +73,7 @@ struct amdgpu_mn {
	/* constant after initialisation */		/* constant after initialisation */
	struct amdgpu_device *adev;		struct amdgpu_device *adev;
	struct mm_struct *mm;		struct mm_struct *mm;
			struct mmu_notifier mn;
	enum amdgpu_mn_type type;		enum amdgpu_mn_type type;

	/* only used on destruction */		/* only used on destruction */
	@@ -82,9 +85,8 @@ struct amdgpu_mn {
	/* objects protected by lock */		/* objects protected by lock */
	struct rw_semaphore lock;		struct rw_semaphore lock;
	struct rb_root_cached objects;		struct rb_root_cached objects;
			struct mutex read_lock;
	/* HMM mirror */		atomic_t recursion;
	struct hmm_mirror mirror;
	};		};

	/**		/**
	@@ -101,7 +103,7 @@ struct amdgpu_mn_node {
	};		};

	/**		/**
	* amdgpu_mn_destroy - destroy the HMM mirror		* amdgpu_mn_destroy - destroy the MMU notifier
	*		*
	* @work: previously sheduled work item		* @work: previously sheduled work item
	*		*
	@@ -127,26 +129,28 @@ static void amdgpu_mn_destroy(struct work_struct *work)
	}		}
	up_write(&amn->lock);		up_write(&amn->lock);
	mutex_unlock(&adev->mn_lock);		mutex_unlock(&adev->mn_lock);
			mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
	hmm_mirror_unregister(&amn->mirror);
	kfree(amn);		kfree(amn);
	}		}

	/**		/**
	* amdgpu_hmm_mirror_release - callback to notify about mm destruction		* amdgpu_mn_release - callback to notify about mm destruction
	*		*
	* @mirror: the HMM mirror (mm) this callback is about		* @mn: our notifier
			* @mm: the mm this callback is about
	*		*
	* Shedule a work item to lazy destroy HMM mirror.		* Shedule a work item to lazy destroy our notifier.
	*/		*/
	static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)		static void amdgpu_mn_release(struct mmu_notifier *mn,
			struct mm_struct *mm)
	{		{
	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);		struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);

	INIT_WORK(&amn->work, amdgpu_mn_destroy);		INIT_WORK(&amn->work, amdgpu_mn_destroy);
	schedule_work(&amn->work);		schedule_work(&amn->work);
	}		}


	/**		/**
	* amdgpu_mn_lock - take the write side lock for this notifier		* amdgpu_mn_lock - take the write side lock for this notifier
	*		*
	@@ -177,10 +181,14 @@ void amdgpu_mn_unlock(struct amdgpu_mn *mn)
	static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)		static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
	{		{
	if (blockable)		if (blockable)
	down_read(&amn->lock);		mutex_lock(&amn->read_lock);
	else if (!down_read_trylock(&amn->lock))		else if (!mutex_trylock(&amn->read_lock))
	return -EAGAIN;		return -EAGAIN;

			if (atomic_inc_return(&amn->recursion) == 1)
			down_read_non_owner(&amn->lock);
			mutex_unlock(&amn->read_lock);

	return 0;		return 0;
	}		}

	@@ -191,7 +199,8 @@ static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
	*/		*/
	static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn)		static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn)
	{		{
	up_read(&amn->lock);		if (atomic_dec_return(&amn->recursion) == 0)
			up_read_non_owner(&amn->lock);
	}		}

	/**		/**
	@@ -226,128 +235,143 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
	}		}

	/**		/**
	* amdgpu_mn_sync_pagetables_gfx - callback to notify about mm change		* amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
	*		*
	* @mirror: the hmm_mirror (mm) is about to update		* @mn: our notifier
	* @update: the update start, end address		* @range: mmu notifier context
	*		*
	* Block for operations on BOs to finish and mark pages as accessed and		* Block for operations on BOs to finish and mark pages as accessed and
	* potentially dirty.		* potentially dirty.
	*/		*/
	static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,		static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
	const struct hmm_update *update)		const struct mmu_notifier_range *range)
	{		{
	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);		struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
	unsigned long start = update->start;
	unsigned long end = update->end;
	bool blockable = update->blockable;
	struct interval_tree_node *it;		struct interval_tree_node *it;
			unsigned long end;

	/* notification is exclusive, but interval is inclusive */		/* notification is exclusive, but interval is inclusive */
	end -= 1;		end = range->end - 1;

	/* TODO we should be able to split locking for interval tree and		/* TODO we should be able to split locking for interval tree and
	* amdgpu_mn_invalidate_node		* amdgpu_mn_invalidate_node
	*/		*/
	if (amdgpu_mn_read_lock(amn, blockable))		if (amdgpu_mn_read_lock(amn, range->blockable))
	return -EAGAIN;		return -EAGAIN;

	it = interval_tree_iter_first(&amn->objects, start, end);		it = interval_tree_iter_first(&amn->objects, range->start, end);
	while (it) {		while (it) {
	struct amdgpu_mn_node *node;		struct amdgpu_mn_node *node;

	if (!blockable) {		if (!range->blockable) {
	amdgpu_mn_read_unlock(amn);		amdgpu_mn_read_unlock(amn);
	return -EAGAIN;		return -EAGAIN;
	}		}

	node = container_of(it, struct amdgpu_mn_node, it);		node = container_of(it, struct amdgpu_mn_node, it);
	it = interval_tree_iter_next(it, start, end);		it = interval_tree_iter_next(it, range->start, end);

	amdgpu_mn_invalidate_node(node, start, end);		amdgpu_mn_invalidate_node(node, range->start, end);
	}		}

	amdgpu_mn_read_unlock(amn);

	return 0;		return 0;
	}		}

	/**		/**
	* amdgpu_mn_sync_pagetables_hsa - callback to notify about mm change		* amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
	*		*
	* @mirror: the hmm_mirror (mm) is about to update		* @mn: our notifier
	* @update: the update start, end address		* @mm: the mm this callback is about
			* @start: start of updated range
			* @end: end of updated range
	*		*
	* We temporarily evict all BOs between start and end. This		* We temporarily evict all BOs between start and end. This
	* necessitates evicting all user-mode queues of the process. The BOs		* necessitates evicting all user-mode queues of the process. The BOs
	* are restorted in amdgpu_mn_invalidate_range_end_hsa.		* are restorted in amdgpu_mn_invalidate_range_end_hsa.
	*/		*/
	static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,		static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
	const struct hmm_update *update)		const struct mmu_notifier_range *range)
	{		{
	struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);		struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
	unsigned long start = update->start;
	unsigned long end = update->end;
	bool blockable = update->blockable;
	struct interval_tree_node *it;		struct interval_tree_node *it;
			unsigned long end;

	/* notification is exclusive, but interval is inclusive */		/* notification is exclusive, but interval is inclusive */
	end -= 1;		end = range->end - 1;

	if (amdgpu_mn_read_lock(amn, blockable))		if (amdgpu_mn_read_lock(amn, range->blockable))
	return -EAGAIN;		return -EAGAIN;

	it = interval_tree_iter_first(&amn->objects, start, end);		it = interval_tree_iter_first(&amn->objects, range->start, end);
	while (it) {		while (it) {
	struct amdgpu_mn_node *node;		struct amdgpu_mn_node *node;
	struct amdgpu_bo *bo;		struct amdgpu_bo *bo;

	if (!blockable) {		if (!range->blockable) {
	amdgpu_mn_read_unlock(amn);		amdgpu_mn_read_unlock(amn);
	return -EAGAIN;		return -EAGAIN;
	}		}

	node = container_of(it, struct amdgpu_mn_node, it);		node = container_of(it, struct amdgpu_mn_node, it);
	it = interval_tree_iter_next(it, start, end);		it = interval_tree_iter_next(it, range->start, end);

	list_for_each_entry(bo, &node->bos, mn_list) {		list_for_each_entry(bo, &node->bos, mn_list) {
	struct kgd_mem *mem = bo->kfd_bo;		struct kgd_mem *mem = bo->kfd_bo;

	if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,		if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
	start, end))		range->start,
	amdgpu_amdkfd_evict_userptr(mem, amn->mm);		end))
			amdgpu_amdkfd_evict_userptr(mem, range->mm);
	}		}
	}		}

	amdgpu_mn_read_unlock(amn);

	return 0;		return 0;
	}		}

	/* Low bits of any reasonable mm pointer will be unused due to struct		/**
	* alignment. Use these bits to make a unique key from the mm pointer		* amdgpu_mn_invalidate_range_end - callback to notify about mm change
	* and notifier type.		*
			* @mn: our notifier
			* @mm: the mm this callback is about
			* @start: start of updated range
			* @end: end of updated range
			*
			* Release the lock again to allow new command submissions.
	*/		*/
	#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))		static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
			const struct mmu_notifier_range *range)
			{
			struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);

			amdgpu_mn_read_unlock(amn);
			}

	static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {		static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
	[AMDGPU_MN_TYPE_GFX] = {		[AMDGPU_MN_TYPE_GFX] = {
	.sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx,		.release = amdgpu_mn_release,
	.release = amdgpu_hmm_mirror_release		.invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
			.invalidate_range_end = amdgpu_mn_invalidate_range_end,
	},		},
	[AMDGPU_MN_TYPE_HSA] = {		[AMDGPU_MN_TYPE_HSA] = {
	.sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa,		.release = amdgpu_mn_release,
	.release = amdgpu_hmm_mirror_release		.invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
			.invalidate_range_end = amdgpu_mn_invalidate_range_end,
	},		},
	};		};

			/* Low bits of any reasonable mm pointer will be unused due to struct
			* alignment. Use these bits to make a unique key from the mm pointer
			* and notifier type.
			*/
			#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))

	/**		/**
	* amdgpu_mn_get - create HMM mirror context		* amdgpu_mn_get - create notifier context
	*		*
	* @adev: amdgpu device pointer		* @adev: amdgpu device pointer
	* @type: type of MMU notifier context		* @type: type of MMU notifier context
	*		*
	* Creates a HMM mirror context for current->mm.		* Creates a notifier context for current->mm.
	*/		*/
	struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,		struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,
	enum amdgpu_mn_type type)		enum amdgpu_mn_type type)
	@@ -377,10 +401,12 @@ struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,
	amn->mm = mm;		amn->mm = mm;
	init_rwsem(&amn->lock);		init_rwsem(&amn->lock);
	amn->type = type;		amn->type = type;
			amn->mn.ops = &amdgpu_mn_ops[type];
	amn->objects = RB_ROOT_CACHED;		amn->objects = RB_ROOT_CACHED;
			mutex_init(&amn->read_lock);
			atomic_set(&amn->recursion, 0);

	amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];		r = __mmu_notifier_register(&amn->mn, mm);
	r = hmm_mirror_register(&amn->mirror, mm);
	if (r)		if (r)
	goto free_amn;		goto free_amn;

	@@ -406,7 +432,7 @@ struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,
	* @bo: amdgpu buffer object		* @bo: amdgpu buffer object
	* @addr: userptr addr we should monitor		* @addr: userptr addr we should monitor
	*		*
	* Registers an HMM mirror for the given BO at the specified address.		* Registers an MMU notifier for the given BO at the specified address.
	* Returns 0 on success, -ERRNO if anything goes wrong.		* Returns 0 on success, -ERRNO if anything goes wrong.
	*/		*/
	int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)		int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
	@@ -462,11 +488,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
	}		}

	/**		/**
	* amdgpu_mn_unregister - unregister a BO for HMM mirror updates		* amdgpu_mn_unregister - unregister a BO for notifier updates
	*		*
	* @bo: amdgpu buffer object		* @bo: amdgpu buffer object
	*		*
	* Remove any registration of HMM mirror updates from the buffer object.		* Remove any registration of MMU notifier updates from the buffer object.
	*/		*/
	void amdgpu_mn_unregister(struct amdgpu_bo *bo)		void amdgpu_mn_unregister(struct amdgpu_bo *bo)
	{		{

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -34,7 +34,7 @@ enum amdgpu_mn_type {
	AMDGPU_MN_TYPE_HSA,		AMDGPU_MN_TYPE_HSA,
	};		};

	#if defined(CONFIG_HMM_MIRROR)		#if defined(CONFIG_MMU_NOTIFIER)
	void amdgpu_mn_lock(struct amdgpu_mn *mn);		void amdgpu_mn_lock(struct amdgpu_mn *mn);
	void amdgpu_mn_unlock(struct amdgpu_mn *mn);		void amdgpu_mn_unlock(struct amdgpu_mn *mn);
	struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,		struct amdgpu_mn amdgpu_mn_get(struct amdgpu_device adev,