Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a46a2cd1 authored by Felix Kuehling's avatar Felix Kuehling Committed by Oded Gabbay
Browse files

drm/amdgpu: Add GPUVM memory management functions for KFD 



v2:
* Removed unused flags from struct kgd_mem
* Updated some comments
* Added a check to unmap_memory_from_gpu whether BO was mapped

v3: add mutex_destroy in relevant places

Signed-off-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Acked-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
parent 3c728d3a

MAINTAINERS

+1 −0
Original line number Diff line number Diff line
@@ -767,6 +767,7 @@ F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h 
F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c

F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c

F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c

F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c

F:	drivers/gpu/drm/amd/amdkfd/

F:	drivers/gpu/drm/amd/include/cik_structs.h

F:	drivers/gpu/drm/amd/include/kgd_kfd_interface.h

drivers/gpu/drm/amd/amdgpu/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -130,6 +130,7 @@ amdgpu-y += \ 
amdgpu-y += \

	 amdgpu_amdkfd.o \

	 amdgpu_amdkfd_fence.o \

	 amdgpu_amdkfd_gpuvm.o \

	 amdgpu_amdkfd_gfx_v8.o



# add cgs

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c

+1 −0
Original line number Diff line number Diff line
@@ -58,6 +58,7 @@ int amdgpu_amdkfd_init(void) 
#else

	ret = -ENOENT;

#endif

	amdgpu_amdkfd_gpuvm_init_mem_limits();



	return ret;

}

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h

+89 −2
Original line number Diff line number Diff line
@@ -28,15 +28,41 @@ 
#include <linux/types.h>

#include <linux/mmu_context.h>

#include <kgd_kfd_interface.h>

#include <drm/ttm/ttm_execbuf_util.h>

#include "amdgpu_sync.h"

#include "amdgpu_vm.h"



extern const struct kgd2kfd_calls *kgd2kfd;



struct amdgpu_device;



struct kfd_bo_va_list {

	struct list_head bo_list;

	struct amdgpu_bo_va *bo_va;

	void *kgd_dev;

	bool is_mapped;

	uint64_t va;

	uint64_t pte_flags;

};



struct kgd_mem {

	struct mutex lock;

	struct amdgpu_bo *bo;

	uint64_t gpu_addr;

	void *cpu_ptr;

	struct list_head bo_va_list;

	/* protected by amdkfd_process_info.lock */

	struct ttm_validate_buffer validate_list;

	struct ttm_validate_buffer resv_list;

	uint32_t domain;

	unsigned int mapped_to_gpu_memory;

	uint64_t va;



	uint32_t mapping_flags;



	struct amdkfd_process_info *process_info;



	struct amdgpu_sync sync;



	bool aql_queue;

};



/* KFD Memory Eviction */
@@ -52,6 +78,41 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, 
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);

struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);



struct amdkfd_process_info {

	/* List head of all VMs that belong to a KFD process */

	struct list_head vm_list_head;

	/* List head for all KFD BOs that belong to a KFD process. */

	struct list_head kfd_bo_list;

	/* Lock to protect kfd_bo_list */

	struct mutex lock;



	/* Number of VMs */

	unsigned int n_vms;

	/* Eviction Fence */

	struct amdgpu_amdkfd_fence *eviction_fence;

};



/* struct amdkfd_vm -

 * For Memory Eviction KGD requires a mechanism to keep track of all KFD BOs

 * belonging to a KFD process. All the VMs belonging to the same process point

 * to the same amdkfd_process_info.

 */

struct amdkfd_vm {

	/* Keep base as the first parameter for pointer compatibility between

	 * amdkfd_vm and amdgpu_vm.

	 */

	struct amdgpu_vm base;



	/* List node in amdkfd_process_info.vm_list_head*/

	struct list_head vm_list_node;



	struct amdgpu_device *adev;

	/* Points to the KFD process VM info*/

	struct amdkfd_process_info *process_info;



	uint64_t pd_phys_addr;

};



int amdgpu_amdkfd_init(void);

void amdgpu_amdkfd_fini(void);
@@ -96,4 +157,30 @@ uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd); 
		valid;							\

	})



/* GPUVM API */

int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,

					  void **process_info,

					  struct dma_fence **ef);

void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);

uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);

int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(

		struct kgd_dev *kgd, uint64_t va, uint64_t size,

		void *vm, struct kgd_mem **mem,

		uint64_t *offset, uint32_t flags);

int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(

		struct kgd_dev *kgd, struct kgd_mem *mem);

int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(

		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);

int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(

		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);

int amdgpu_amdkfd_gpuvm_sync_memory(

		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);

int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,

		struct kgd_mem *mem, void **kptr, uint64_t *size);

int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,

					    struct dma_fence **ef);



void amdgpu_amdkfd_gpuvm_init_mem_limits(void);

void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);



#endif /* AMDGPU_AMDKFD_H_INCLUDED */

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c

+65 −1
Original line number Diff line number Diff line
@@ -143,6 +143,10 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, 
static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);

static void set_scratch_backing_va(struct kgd_dev *kgd,

					uint64_t va, uint32_t vmid);

static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,

		uint32_t page_table_base);

static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);

static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);



/* Because of REG_GET_FIELD() being used, we put this function in the

 * asic specific file.
@@ -199,7 +203,20 @@ static const struct kfd2kgd_calls kfd2kgd = { 
	.set_scratch_backing_va = set_scratch_backing_va,

	.get_tile_config = get_tile_config,

	.get_cu_info = get_cu_info,

	.get_vram_usage = amdgpu_amdkfd_get_vram_usage

	.get_vram_usage = amdgpu_amdkfd_get_vram_usage,

	.create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,

	.destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,

	.get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,

	.set_vm_context_page_table_base = set_vm_context_page_table_base,

	.alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,

	.free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,

	.map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,

	.unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,

	.sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,

	.map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,

	.restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,

	.invalidate_tlbs = invalidate_tlbs,

	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,

};



struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -855,3 +872,50 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) 
	return hdr->common.ucode_version;

}



static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,

			uint32_t page_table_base)

{

	struct amdgpu_device *adev = get_amdgpu_device(kgd);



	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {

		pr_err("trying to set page table base for wrong VMID\n");

		return;

	}

	WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, page_table_base);

}



static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)

{

	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;

	int vmid;

	unsigned int tmp;



	for (vmid = 0; vmid < 16; vmid++) {

		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))

			continue;



		tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);

		if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) &&

			(tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) {

			WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);

			RREG32(mmVM_INVALIDATE_RESPONSE);

			break;

		}

	}



	return 0;

}



static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)

{

	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;



	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {

		pr_err("non kfd vmid\n");

		return 0;

	}



	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);

	RREG32(mmVM_INVALIDATE_RESPONSE);

	return 0;

}