cxl: Driver code for powernv PCIe based cards for userspace access

/*

 * Copyright 2014 IBM Corp.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version

 * 2 of the License, or (at your option) any later version.

 */

#include <linux/module.h>

#include <linux/kernel.h>

#include <linux/bitmap.h>

#include <linux/sched.h>

#include <linux/pid.h>

#include <linux/fs.h>

#include <linux/mm.h>

#include <linux/debugfs.h>

#include <linux/slab.h>

#include <linux/idr.h>

#include <asm/cputable.h>

#include <asm/current.h>

#include <asm/copro.h>

#include "cxl.h"

/*

 * Allocates space for a CXL context.

 */

struct cxl_context *cxl_context_alloc(void)

{

	return kzalloc(sizeof(struct cxl_context), GFP_KERNEL);

}

/*

 * Initialises a CXL context.

 */

int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)

{

	int i;

	spin_lock_init(&ctx->sste_lock);

	ctx->afu = afu;

	ctx->master = master;

	ctx->pid = NULL; /* Set in start work ioctl */

	/*

	 * Allocate the segment table before we put it in the IDR so that we

	 * can always access it when dereferenced from IDR. For the same

	 * reason, the segment table is only destroyed after the context is

	 * removed from the IDR.  Access to this in the IOCTL is protected by

	 * Linux filesytem symantics (can't IOCTL until open is complete).

	 */

	i = cxl_alloc_sst(ctx);

	if (i)

		return i;

	INIT_WORK(&ctx->fault_work, cxl_handle_fault);

	init_waitqueue_head(&ctx->wq);

	spin_lock_init(&ctx->lock);

	ctx->irq_bitmap = NULL;

	ctx->pending_irq = false;

	ctx->pending_fault = false;

	ctx->pending_afu_err = false;

	/*

	 * When we have to destroy all contexts in cxl_context_detach_all() we

	 * end up with afu_release_irqs() called from inside a

	 * idr_for_each_entry(). Hence we need to make sure that anything

	 * dereferenced from this IDR is ok before we allocate the IDR here.

	 * This clears out the IRQ ranges to ensure this.

	 */

	for (i = 0; i < CXL_IRQ_RANGES; i++)

		ctx->irqs.range[i] = 0;

	mutex_init(&ctx->status_mutex);

	ctx->status = OPENED;

	/*

	 * Allocating IDR! We better make sure everything's setup that

	 * dereferences from it.

	 */

	idr_preload(GFP_KERNEL);

	spin_lock(&afu->contexts_lock);

	i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,

		      ctx->afu->num_procs, GFP_NOWAIT);

	spin_unlock(&afu->contexts_lock);

	idr_preload_end();

	if (i < 0)

		return i;

	ctx->pe = i;

	ctx->elem = &ctx->afu->spa[i];

	ctx->pe_inserted = false;

	return 0;

}

/*

 * Map a per-context mmio space into the given vma.

 */

int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma)

{

	u64 len = vma->vm_end - vma->vm_start;

	len = min(len, ctx->psn_size);

	if (ctx->afu->current_mode == CXL_MODE_DEDICATED) {

		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

		return vm_iomap_memory(vma, ctx->afu->psn_phys, ctx->afu->adapter->ps_size);

	}

	/* make sure there is a valid per process space for this AFU */

	if ((ctx->master && !ctx->afu->psa) || (!ctx->afu->pp_psa)) {

		pr_devel("AFU doesn't support mmio space\n");

		return -EINVAL;

	}

	/* Can't mmap until the AFU is enabled */

	if (!ctx->afu->enabled)

		return -EBUSY;

	pr_devel("%s: mmio physical: %llx pe: %i master:%i\n", __func__,

		 ctx->psn_phys, ctx->pe , ctx->master);

	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

	return vm_iomap_memory(vma, ctx->psn_phys, len);

}

/*

 * Detach a context from the hardware. This disables interrupts and doesn't

 * return until all outstanding interrupts for this context have completed. The

 * hardware should no longer access *ctx after this has returned.

 */

static void __detach_context(struct cxl_context *ctx)

{

	enum cxl_context_status status;

	mutex_lock(&ctx->status_mutex);

	status = ctx->status;

	ctx->status = CLOSED;

	mutex_unlock(&ctx->status_mutex);

	if (status != STARTED)

		return;

	WARN_ON(cxl_detach_process(ctx));

	afu_release_irqs(ctx);

	flush_work(&ctx->fault_work); /* Only needed for dedicated process */

	wake_up_all(&ctx->wq);

}

/*

 * Detach the given context from the AFU. This doesn't actually

 * free the context but it should stop the context running in hardware

 * (ie. prevent this context from generating any further interrupts

 * so that it can be freed).

 */

void cxl_context_detach(struct cxl_context *ctx)

{

	__detach_context(ctx);

}

/*

 * Detach all contexts on the given AFU.

 */

void cxl_context_detach_all(struct cxl_afu *afu)

{

	struct cxl_context *ctx;

	int tmp;

	rcu_read_lock();

	idr_for_each_entry(&afu->contexts_idr, ctx, tmp)

		/*

		 * Anything done in here needs to be setup before the IDR is

		 * created and torn down after the IDR removed

		 */

		__detach_context(ctx);

	rcu_read_unlock();

}

void cxl_context_free(struct cxl_context *ctx)

{

	spin_lock(&ctx->afu->contexts_lock);

	idr_remove(&ctx->afu->contexts_idr, ctx->pe);

	spin_unlock(&ctx->afu->contexts_lock);

	synchronize_rcu();

	free_page((u64)ctx->sstp);

	ctx->sstp = NULL;

	put_pid(ctx->pid);

	kfree(ctx);

}

/*

 * Copyright 2014 IBM Corp.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version

 * 2 of the License, or (at your option) any later version.

 */

#include <linux/debugfs.h>

#include <linux/kernel.h>

#include <linux/slab.h>

#include "cxl.h"

static struct dentry *cxl_debugfs;

void cxl_stop_trace(struct cxl *adapter)

{

	int slice;

	/* Stop the trace */

	cxl_p1_write(adapter, CXL_PSL_TRACE, 0x8000000000000017LL);

	/* Stop the slice traces */

	spin_lock(&adapter->afu_list_lock);

	for (slice = 0; slice < adapter->slices; slice++) {

		if (adapter->afu[slice])

			cxl_p1n_write(adapter->afu[slice], CXL_PSL_SLICE_TRACE, 0x8000000000000000LL);

	}

	spin_unlock(&adapter->afu_list_lock);

}

/* Helpers to export CXL mmaped IO registers via debugfs */

static int debugfs_io_u64_get(void *data, u64 *val)

{

	*val = in_be64((u64 __iomem *)data);

	return 0;

}

static int debugfs_io_u64_set(void *data, u64 val)

{

	out_be64((u64 __iomem *)data, val);

	return 0;

}

DEFINE_SIMPLE_ATTRIBUTE(fops_io_x64, debugfs_io_u64_get, debugfs_io_u64_set, "0x%016llx\n");

static struct dentry *debugfs_create_io_x64(const char *name, umode_t mode,

					    struct dentry *parent, u64 __iomem *value)

{

	return debugfs_create_file(name, mode, parent, (void *)value, &fops_io_x64);

}

int cxl_debugfs_adapter_add(struct cxl *adapter)

{

	struct dentry *dir;

	char buf[32];

	if (!cxl_debugfs)

		return -ENODEV;

	snprintf(buf, 32, "card%i", adapter->adapter_num);

	dir = debugfs_create_dir(buf, cxl_debugfs);

	if (IS_ERR(dir))

		return PTR_ERR(dir);

	adapter->debugfs = dir;

	debugfs_create_io_x64("fir1",     S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR1));

	debugfs_create_io_x64("fir2",     S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR2));

	debugfs_create_io_x64("fir_cntl", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR_CNTL));

	debugfs_create_io_x64("err_ivte", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_ErrIVTE));

	debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));

	return 0;

}

void cxl_debugfs_adapter_remove(struct cxl *adapter)

{

	debugfs_remove_recursive(adapter->debugfs);

}

int cxl_debugfs_afu_add(struct cxl_afu *afu)

{

	struct dentry *dir;

	char buf[32];

	if (!afu->adapter->debugfs)

		return -ENODEV;

	snprintf(buf, 32, "psl%i.%i", afu->adapter->adapter_num, afu->slice);

	dir = debugfs_create_dir(buf, afu->adapter->debugfs);

	if (IS_ERR(dir))

		return PTR_ERR(dir);

	afu->debugfs = dir;

	debugfs_create_io_x64("fir",        S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_FIR_SLICE_An));

	debugfs_create_io_x64("serr",       S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SERR_An));

	debugfs_create_io_x64("afu_debug",  S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_AFU_DEBUG_An));

	debugfs_create_io_x64("sr",         S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SR_An));

	debugfs_create_io_x64("dsisr",      S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DSISR_An));

	debugfs_create_io_x64("dar",        S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DAR_An));

	debugfs_create_io_x64("sstp0",      S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP0_An));

	debugfs_create_io_x64("sstp1",      S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP1_An));

	debugfs_create_io_x64("err_status", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_ErrStat_An));

	debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SLICE_TRACE));

	return 0;

}

void cxl_debugfs_afu_remove(struct cxl_afu *afu)

{

	debugfs_remove_recursive(afu->debugfs);

}

int __init cxl_debugfs_init(void)

{

	struct dentry *ent;

	ent = debugfs_create_dir("cxl", NULL);

	if (IS_ERR(ent))

		return PTR_ERR(ent);

	cxl_debugfs = ent;

	return 0;

}

void cxl_debugfs_exit(void)

{

	debugfs_remove_recursive(cxl_debugfs);

}

/*

 * Copyright 2014 IBM Corp.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version

 * 2 of the License, or (at your option) any later version.

 */

#include <linux/workqueue.h>

#include <linux/sched.h>

#include <linux/pid.h>

#include <linux/mm.h>

#include <linux/moduleparam.h>

#undef MODULE_PARAM_PREFIX

#define MODULE_PARAM_PREFIX "cxl" "."

#include <asm/current.h>

#include <asm/copro.h>

#include <asm/mmu.h>

#include "cxl.h"

static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group,

				       bool sec_hash,

				       struct cxl_sste *secondary_group,

				       unsigned int *lru)

{

	unsigned int i, entry;

	struct cxl_sste *sste, *group = primary_group;

	for (i = 0; i < 2; i++) {

		for (entry = 0; entry < 8; entry++) {

			sste = group + entry;

			if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V))

				return sste;

		}

		if (!sec_hash)

			break;

		group = secondary_group;

	}

	/* Nothing free, select an entry to cast out */

	if (sec_hash && (*lru & 0x8))

		sste = secondary_group + (*lru & 0x7);

	else

		sste = primary_group + (*lru & 0x7);

	*lru = (*lru + 1) & 0xf;

	return sste;

}

static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb)

{

	/* mask is the group index, we search primary and secondary here. */

	unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */

	bool sec_hash = 1;

	struct cxl_sste *sste;

	unsigned int hash;

	unsigned long flags;

	sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC);

	if (slb->vsid & SLB_VSID_B_1T)

		hash = (slb->esid >> SID_SHIFT_1T) & mask;

	else /* 256M */

		hash = (slb->esid >> SID_SHIFT) & mask;

	spin_lock_irqsave(&ctx->sste_lock, flags);

	sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash,

			      ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru);

	pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",

			sste - ctx->sstp, slb->vsid, slb->esid);

	sste->vsid_data = cpu_to_be64(slb->vsid);

	sste->esid_data = cpu_to_be64(slb->esid);

	spin_unlock_irqrestore(&ctx->sste_lock, flags);

}

static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm,

			     u64 ea)

{

	struct copro_slb slb = {0,0};

	int rc;

	if (!(rc = copro_calculate_slb(mm, ea, &slb))) {

		cxl_load_segment(ctx, &slb);

	}

	return rc;

}

static void cxl_ack_ae(struct cxl_context *ctx)

{

	unsigned long flags;

	cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0);

	spin_lock_irqsave(&ctx->lock, flags);

	ctx->pending_fault = true;

	ctx->fault_addr = ctx->dar;

	ctx->fault_dsisr = ctx->dsisr;

	spin_unlock_irqrestore(&ctx->lock, flags);

	wake_up_all(&ctx->wq);

}

static int cxl_handle_segment_miss(struct cxl_context *ctx,

				   struct mm_struct *mm, u64 ea)

{

	int rc;

	pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea);

	if ((rc = cxl_fault_segment(ctx, mm, ea)))

		cxl_ack_ae(ctx);

	else {

		mb(); /* Order seg table write to TFC MMIO write */

		cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);

	}

	return IRQ_HANDLED;

}

static void cxl_handle_page_fault(struct cxl_context *ctx,

				  struct mm_struct *mm, u64 dsisr, u64 dar)

{

	unsigned flt = 0;

	int result;

	unsigned long access, flags;

	if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {

		pr_devel("copro_handle_mm_fault failed: %#x\n", result);

		return cxl_ack_ae(ctx);

	}

	/*

	 * update_mmu_cache() will not have loaded the hash since current->trap

	 * is not a 0x400 or 0x300, so just call hash_page_mm() here.

	 */

	access = _PAGE_PRESENT;

	if (dsisr & CXL_PSL_DSISR_An_S)

		access |= _PAGE_RW;

	if ((!ctx->kernel) || ~(dar & (1ULL << 63)))

		access |= _PAGE_USER;

	local_irq_save(flags);

	hash_page_mm(mm, dar, access, 0x300);

	local_irq_restore(flags);

	pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);

	cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);

}

void cxl_handle_fault(struct work_struct *fault_work)

{

	struct cxl_context *ctx =

		container_of(fault_work, struct cxl_context, fault_work);

	u64 dsisr = ctx->dsisr;

	u64 dar = ctx->dar;

	struct task_struct *task;

	struct mm_struct *mm;

	if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||

	    cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar ||

	    cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) {

		/* Most likely explanation is harmless - a dedicated process

		 * has detached and these were cleared by the PSL purge, but

		 * warn about it just in case */

		dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n");

		return;

	}

	pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "

		"DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);

	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {

		pr_devel("cxl_handle_fault unable to get task %i\n",

			 pid_nr(ctx->pid));

		cxl_ack_ae(ctx);

		return;

	}

	if (!(mm = get_task_mm(task))) {

		pr_devel("cxl_handle_fault unable to get mm %i\n",

			 pid_nr(ctx->pid));

		cxl_ack_ae(ctx);

		goto out;

	}

	if (dsisr & CXL_PSL_DSISR_An_DS)

		cxl_handle_segment_miss(ctx, mm, dar);

	else if (dsisr & CXL_PSL_DSISR_An_DM)

		cxl_handle_page_fault(ctx, mm, dsisr, dar);

	else

		WARN(1, "cxl_handle_fault has nothing to handle\n");

	mmput(mm);

out:

	put_task_struct(task);

}

static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)

{

	int rc;

	struct task_struct *task;

	struct mm_struct *mm;

	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {

		pr_devel("cxl_prefault_one unable to get task %i\n",

			 pid_nr(ctx->pid));

		return;

	}

	if (!(mm = get_task_mm(task))) {

		pr_devel("cxl_prefault_one unable to get mm %i\n",

			 pid_nr(ctx->pid));

		put_task_struct(task);

		return;

	}

	rc = cxl_fault_segment(ctx, mm, ea);

	mmput(mm);

	put_task_struct(task);

}

static u64 next_segment(u64 ea, u64 vsid)

{

	if (vsid & SLB_VSID_B_1T)

		ea |= (1ULL << 40) - 1;

	else

		ea |= (1ULL << 28) - 1;

	return ea + 1;

}

static void cxl_prefault_vma(struct cxl_context *ctx)

{

	u64 ea, last_esid = 0;

	struct copro_slb slb;

	struct vm_area_struct *vma;

	int rc;

	struct task_struct *task;

	struct mm_struct *mm;

	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {

		pr_devel("cxl_prefault_vma unable to get task %i\n",

			 pid_nr(ctx->pid));

		return;

	}

	if (!(mm = get_task_mm(task))) {

		pr_devel("cxl_prefault_vm unable to get mm %i\n",

			 pid_nr(ctx->pid));

		goto out1;

	}

	down_read(&mm->mmap_sem);

	for (vma = mm->mmap; vma; vma = vma->vm_next) {

		for (ea = vma->vm_start; ea < vma->vm_end;

				ea = next_segment(ea, slb.vsid)) {

			rc = copro_calculate_slb(mm, ea, &slb);

			if (rc)

				continue;

			if (last_esid == slb.esid)

				continue;

			cxl_load_segment(ctx, &slb);

			last_esid = slb.esid;

		}

	}

	up_read(&mm->mmap_sem);

	mmput(mm);

out1:

	put_task_struct(task);

}

void cxl_prefault(struct cxl_context *ctx, u64 wed)

{

	switch (ctx->afu->prefault_mode) {

	case CXL_PREFAULT_WED:

		cxl_prefault_one(ctx, wed);

		break;

	case CXL_PREFAULT_ALL:

		cxl_prefault_vma(ctx);

		break;

	default:

		break;

	}

}