Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ef568db7 authored by Felix Kuehling's avatar Felix Kuehling Committed by Oded Gabbay
Browse files

drm/amdkfd: Implement doorbell allocation for SOC15 



Allocate doorbells according to the doorbell routing information on
SOC15 ASICs (Vega10 and later). On older ASICs we continue to use the
queue_id as the doorbell ID to maintain compatibility with the Thunk.

Signed-off-by: default avatarShaoyun Liu <Shaoyun.Liu@amd.com>
Signed-off-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
parent df03ef93

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

+7 −0
Original line number Original line Diff line number Diff line
@@ -295,6 +295,13 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 
	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;

	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;

	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);

	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);

	args->doorbell_offset <<= PAGE_SHIFT;

	args->doorbell_offset <<= PAGE_SHIFT;

	if (KFD_IS_SOC15(dev->device_info->asic_family))

		/* On SOC15 ASICs, doorbell allocation must be

		 * per-device, and independent from the per-process

		 * queue_id. Return the doorbell offset within the

		 * doorbell aperture to user mode.

		 */

		args->doorbell_offset |= q_properties.doorbell_off;





	mutex_unlock(&p->mutex);

	mutex_unlock(&p->mutex);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

+78 −4
Original line number Original line Diff line number Diff line
@@ -110,6 +110,57 @@ void program_sh_mem_settings(struct device_queue_manager *dqm, 
						qpd->sh_mem_bases);

						qpd->sh_mem_bases);

}

}





static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)

{

	struct kfd_dev *dev = qpd->dqm->dev;



	if (!KFD_IS_SOC15(dev->device_info->asic_family)) {

		/* On pre-SOC15 chips we need to use the queue ID to

		 * preserve the user mode ABI.

		 */

		q->doorbell_id = q->properties.queue_id;

	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {

		/* For SDMA queues on SOC15, use static doorbell

		 * assignments based on the engine and queue.

		 */

		q->doorbell_id = dev->shared_resources.sdma_doorbell

			[q->properties.sdma_engine_id]

			[q->properties.sdma_queue_id];

	} else {

		/* For CP queues on SOC15 reserve a free doorbell ID */

		unsigned int found;



		found = find_first_zero_bit(qpd->doorbell_bitmap,

					    KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);

		if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {

			pr_debug("No doorbells available");

			return -EBUSY;

		}

		set_bit(found, qpd->doorbell_bitmap);

		q->doorbell_id = found;

	}



	q->properties.doorbell_off =

		kfd_doorbell_id_to_offset(dev, q->process,

					  q->doorbell_id);



	return 0;

}



static void deallocate_doorbell(struct qcm_process_device *qpd,

				struct queue *q)

{

	unsigned int old;

	struct kfd_dev *dev = qpd->dqm->dev;



	if (!KFD_IS_SOC15(dev->device_info->asic_family) ||

	    q->properties.type == KFD_QUEUE_TYPE_SDMA)

		return;



	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);

	WARN_ON(!old);

}



static int allocate_vmid(struct device_queue_manager *dqm,

static int allocate_vmid(struct device_queue_manager *dqm,

			struct qcm_process_device *qpd,

			struct qcm_process_device *qpd,

			struct queue *q)

			struct queue *q)
@@ -301,10 +352,14 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 
	if (retval)

	if (retval)

		return retval;

		return retval;





	retval = allocate_doorbell(qpd, q);

	if (retval)

		goto out_deallocate_hqd;



	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

				&q->gart_mqd_addr, &q->properties);

				&q->gart_mqd_addr, &q->properties);

	if (retval)

	if (retval)

		goto out_deallocate_hqd;

		goto out_deallocate_doorbell;





	pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",

	pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",

			q->pipe, q->queue);

			q->pipe, q->queue);
@@ -324,6 +379,8 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 




out_uninit_mqd:

out_uninit_mqd:

	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);

	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);

out_deallocate_doorbell:

	deallocate_doorbell(qpd, q);

out_deallocate_hqd:

out_deallocate_hqd:

	deallocate_hqd(dqm, q);

	deallocate_hqd(dqm, q);
@@ -357,6 +414,8 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 
	}

	}

	dqm->total_queue_count--;

	dqm->total_queue_count--;





	deallocate_doorbell(qpd, q);



	retval = mqd->destroy_mqd(mqd, q->mqd,

	retval = mqd->destroy_mqd(mqd, q->mqd,

				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,

				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,

				KFD_UNMAP_LATENCY_MS,

				KFD_UNMAP_LATENCY_MS,
@@ -861,6 +920,10 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 
	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;

	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;

	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;

	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;





	retval = allocate_doorbell(qpd, q);

	if (retval)

		goto out_deallocate_sdma_queue;



	pr_debug("SDMA id is:    %d\n", q->sdma_id);

	pr_debug("SDMA id is:    %d\n", q->sdma_id);

	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);

	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);

	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);

	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
@@ -869,7 +932,7 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 
	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

				&q->gart_mqd_addr, &q->properties);

				&q->gart_mqd_addr, &q->properties);

	if (retval)

	if (retval)

		goto out_deallocate_sdma_queue;

		goto out_deallocate_doorbell;





	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);

	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);

	if (retval)

	if (retval)
@@ -879,6 +942,8 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 




out_uninit_mqd:

out_uninit_mqd:

	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);

	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);

out_deallocate_doorbell:

	deallocate_doorbell(qpd, q);

out_deallocate_sdma_queue:

out_deallocate_sdma_queue:

	deallocate_sdma_queue(dqm, q->sdma_id);

	deallocate_sdma_queue(dqm, q->sdma_id);
@@ -1070,12 +1135,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 
		q->properties.sdma_engine_id =

		q->properties.sdma_engine_id =

			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;

			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;

	}

	}



	retval = allocate_doorbell(qpd, q);

	if (retval)

		goto out_deallocate_sdma_queue;



	mqd = dqm->ops.get_mqd_manager(dqm,

	mqd = dqm->ops.get_mqd_manager(dqm,

			get_mqd_type_from_queue_type(q->properties.type));

			get_mqd_type_from_queue_type(q->properties.type));





	if (!mqd) {

	if (!mqd) {

		retval = -ENOMEM;

		retval = -ENOMEM;

		goto out_deallocate_sdma_queue;

		goto out_deallocate_doorbell;

	}

	}

	/*

	/*

	 * Eviction state logic: we only mark active queues as evicted

	 * Eviction state logic: we only mark active queues as evicted
@@ -1093,7 +1163,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 
	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,

				&q->gart_mqd_addr, &q->properties);

				&q->gart_mqd_addr, &q->properties);

	if (retval)

	if (retval)

		goto out_deallocate_sdma_queue;

		goto out_deallocate_doorbell;





	list_add(&q->list, &qpd->queues_list);

	list_add(&q->list, &qpd->queues_list);

	qpd->queue_count++;

	qpd->queue_count++;
@@ -1117,6 +1187,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 
	mutex_unlock(&dqm->lock);

	mutex_unlock(&dqm->lock);

	return retval;

	return retval;





out_deallocate_doorbell:

	deallocate_doorbell(qpd, q);

out_deallocate_sdma_queue:

out_deallocate_sdma_queue:

	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)

	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)

		deallocate_sdma_queue(dqm, q->sdma_id);

		deallocate_sdma_queue(dqm, q->sdma_id);
@@ -1257,6 +1329,8 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, 
		goto failed;

		goto failed;

	}

	}





	deallocate_doorbell(qpd, q);



	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {

	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {

		dqm->sdma_queue_count--;

		dqm->sdma_queue_count--;

		deallocate_sdma_queue(dqm, q->sdma_id);

		deallocate_sdma_queue(dqm, q->sdma_id);

drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c

+4 −8
Original line number Original line Diff line number Diff line
@@ -49,7 +49,7 @@ static unsigned int max_doorbell_slices; 
 */

 */





/* # of doorbell bytes allocated for each process. */

/* # of doorbell bytes allocated for each process. */

static size_t kfd_doorbell_process_slice(struct kfd_dev *kfd)

size_t kfd_doorbell_process_slice(struct kfd_dev *kfd)

{

{

	return roundup(kfd->device_info->doorbell_size *

	return roundup(kfd->device_info->doorbell_size *

			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,

			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
@@ -214,13 +214,9 @@ void write_kernel_doorbell(void __iomem *db, u32 value) 
	}

	}

}

}





/*

unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,

 * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1

 * to doorbells with the process's doorbell page

 */

unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,

					struct kfd_process *process,

					struct kfd_process *process,

					unsigned int queue_id)

					unsigned int doorbell_id)

{

{

	/*

	/*

	 * doorbell_id_offset accounts for doorbells taken by KGD.

	 * doorbell_id_offset accounts for doorbells taken by KGD.
@@ -231,7 +227,7 @@ unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, 
	return kfd->doorbell_id_offset +

	return kfd->doorbell_id_offset +

		process->doorbell_index

		process->doorbell_index

		* kfd_doorbell_process_slice(kfd) / sizeof(u32) +

		* kfd_doorbell_process_slice(kfd) / sizeof(u32) +

		queue_id * kfd->device_info->doorbell_size / sizeof(u32);

		doorbell_id * kfd->device_info->doorbell_size / sizeof(u32);

}

}





uint64_t kfd_get_number_elems(struct kfd_dev *kfd)

uint64_t kfd_get_number_elems(struct kfd_dev *kfd)

drivers/gpu/drm/amd/amdkfd/kfd_priv.h

+9 −2
Original line number Original line Diff line number Diff line
@@ -169,6 +169,8 @@ enum cache_policy { 
	cache_policy_noncoherent

	cache_policy_noncoherent

};

};





#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)



struct kfd_event_interrupt_class {

struct kfd_event_interrupt_class {

	bool (*interrupt_isr)(struct kfd_dev *dev,

	bool (*interrupt_isr)(struct kfd_dev *dev,

				const uint32_t *ih_ring_entry);

				const uint32_t *ih_ring_entry);
@@ -449,6 +451,7 @@ struct queue { 
	uint32_t queue;

	uint32_t queue;





	unsigned int sdma_id;

	unsigned int sdma_id;

	unsigned int doorbell_id;





	struct kfd_process	*process;

	struct kfd_process	*process;

	struct kfd_dev		*device;

	struct kfd_dev		*device;
@@ -523,6 +526,9 @@ struct qcm_process_device { 
	/* IB memory */

	/* IB memory */

	uint64_t ib_base;

	uint64_t ib_base;

	void *ib_kaddr;

	void *ib_kaddr;



	/* doorbell resources per process per device */

	unsigned long *doorbell_bitmap;

};

};





/* KFD Memory Eviction */

/* KFD Memory Eviction */
@@ -747,6 +753,7 @@ unsigned int kfd_pasid_alloc(void); 
void kfd_pasid_free(unsigned int pasid);

void kfd_pasid_free(unsigned int pasid);





/* Doorbells */

/* Doorbells */

size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);

int kfd_doorbell_init(struct kfd_dev *kfd);

int kfd_doorbell_init(struct kfd_dev *kfd);

void kfd_doorbell_fini(struct kfd_dev *kfd);

void kfd_doorbell_fini(struct kfd_dev *kfd);

int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,

int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
@@ -756,9 +763,9 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, 
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);

void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);

u32 read_kernel_doorbell(u32 __iomem *db);

u32 read_kernel_doorbell(u32 __iomem *db);

void write_kernel_doorbell(void __iomem *db, u32 value);

void write_kernel_doorbell(void __iomem *db, u32 value);

unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,

unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,

					struct kfd_process *process,

					struct kfd_process *process,

					unsigned int queue_id);

					unsigned int doorbell_id);

phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,

phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,

					struct kfd_process *process);

					struct kfd_process *process);

int kfd_alloc_process_doorbells(struct kfd_process *process);

int kfd_alloc_process_doorbells(struct kfd_process *process);

drivers/gpu/drm/amd/amdkfd/kfd_process.c

+32 −0
Original line number Original line Diff line number Diff line
@@ -332,6 +332,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) 
			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,

			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,

				get_order(KFD_CWSR_TBA_TMA_SIZE));

				get_order(KFD_CWSR_TBA_TMA_SIZE));





		kfree(pdd->qpd.doorbell_bitmap);

		idr_destroy(&pdd->alloc_idr);

		idr_destroy(&pdd->alloc_idr);





		kfree(pdd);

		kfree(pdd);
@@ -586,6 +587,31 @@ static struct kfd_process *create_process(const struct task_struct *thread, 
	return ERR_PTR(err);

	return ERR_PTR(err);

}

}





static int init_doorbell_bitmap(struct qcm_process_device *qpd,

			struct kfd_dev *dev)

{

	unsigned int i;



	if (!KFD_IS_SOC15(dev->device_info->asic_family))

		return 0;



	qpd->doorbell_bitmap =

		kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,

				     BITS_PER_BYTE), GFP_KERNEL);

	if (!qpd->doorbell_bitmap)

		return -ENOMEM;



	/* Mask out any reserved doorbells */

	for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS; i++)

		if ((dev->shared_resources.reserved_doorbell_mask & i) ==

		    dev->shared_resources.reserved_doorbell_val) {

			set_bit(i, qpd->doorbell_bitmap);

			pr_debug("reserved doorbell 0x%03x\n", i);

		}



	return 0;

}



struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,

struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,

							struct kfd_process *p)

							struct kfd_process *p)

{

{
@@ -607,6 +633,12 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, 
	if (!pdd)

	if (!pdd)

		return NULL;

		return NULL;





	if (init_doorbell_bitmap(&pdd->qpd, dev)) {

		pr_err("Failed to init doorbell for process\n");

		kfree(pdd);

		return NULL;

	}



	pdd->dev = dev;

	pdd->dev = dev;

	INIT_LIST_HEAD(&pdd->qpd.queues_list);

	INIT_LIST_HEAD(&pdd->qpd.queues_list);

	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);

	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);