IB/ipath: Performance improvements via mmap of queues

	ipath_cq.o \

	ipath_keys.o \

	ipath_mad.o \

	ipath_mmap.o \

	ipath_mr.o \

	ipath_qp.o \

	ipath_rc.o \

 * @entry: work completion entry to add

 * @sig: true if @entry is a solicitated entry

 *

 * This may be called with one of the qp->s_lock or qp->r_rq.lock held.

 * This may be called with qp->s_lock held.

 */

void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)

{

	struct ipath_cq_wc *wc = cq->queue;

	unsigned long flags;

	u32 head;

	u32 next;

	spin_lock_irqsave(&cq->lock, flags);

	if (cq->head == cq->ibcq.cqe)

	/*

	 * Note that the head pointer might be writable by user processes.

	 * Take care to verify it is a sane value.

	 */

	head = wc->head;

	if (head >= (unsigned) cq->ibcq.cqe) {

		head = cq->ibcq.cqe;

		next = 0;

	else

		next = cq->head + 1;

	if (unlikely(next == cq->tail)) {

	} else

		next = head + 1;

	if (unlikely(next == wc->tail)) {

		spin_unlock_irqrestore(&cq->lock, flags);

		if (cq->ibcq.event_handler) {

			struct ib_event ev;

		}

		return;

	}

	cq->queue[cq->head] = *entry;

	cq->head = next;

	wc->queue[head] = *entry;

	wc->head = next;

	if (cq->notify == IB_CQ_NEXT_COMP ||

	    (cq->notify == IB_CQ_SOLICITED && solicited)) {

int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)

{

	struct ipath_cq *cq = to_icq(ibcq);

	struct ipath_cq_wc *wc = cq->queue;

	unsigned long flags;

	int npolled;

	spin_lock_irqsave(&cq->lock, flags);

	for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {

		if (cq->tail == cq->head)

		if (wc->tail == wc->head)

			break;

		*entry = cq->queue[cq->tail];

		if (cq->tail == cq->ibcq.cqe)

			cq->tail = 0;

		*entry = wc->queue[wc->tail];

		if (wc->tail >= cq->ibcq.cqe)

			wc->tail = 0;

		else

			cq->tail++;

			wc->tail++;

	}

	spin_unlock_irqrestore(&cq->lock, flags);

{

	struct ipath_ibdev *dev = to_idev(ibdev);

	struct ipath_cq *cq;

	struct ib_wc *wc;

	struct ipath_cq_wc *wc;

	struct ib_cq *ret;

	if (entries > ib_ipath_max_cqes) {

		ret = ERR_PTR(-EINVAL);

		goto bail;

		goto done;

	}

	if (dev->n_cqs_allocated == ib_ipath_max_cqs) {

		ret = ERR_PTR(-ENOMEM);

		goto bail;

		goto done;

	}

	/*

	 * Need to use vmalloc() if we want to support large #s of

	 * entries.

	 */

	/* Allocate the completion queue structure. */

	cq = kmalloc(sizeof(*cq), GFP_KERNEL);

	if (!cq) {

		ret = ERR_PTR(-ENOMEM);

		goto bail;

		goto done;

	}

	/*

	 * Need to use vmalloc() if we want to support large #s of entries.

	 * Allocate the completion queue entries and head/tail pointers.

	 * This is allocated separately so that it can be resized and

	 * also mapped into user space.

	 * We need to use vmalloc() in order to support mmap and large

	 * numbers of entries.

	 */

	wc = vmalloc(sizeof(*wc) * (entries + 1));

	wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * entries);

	if (!wc) {

		kfree(cq);

		ret = ERR_PTR(-ENOMEM);

		goto bail;

		goto bail_cq;

	}

	/*

	 * Return the address of the WC as the offset to mmap.

	 * See ipath_mmap() for details.

	 */

	if (udata && udata->outlen >= sizeof(__u64)) {

		struct ipath_mmap_info *ip;

		__u64 offset = (__u64) wc;

		int err;

		err = ib_copy_to_udata(udata, &offset, sizeof(offset));

		if (err) {

			ret = ERR_PTR(err);

			goto bail_wc;

		}

		/* Allocate info for ipath_mmap(). */

		ip = kmalloc(sizeof(*ip), GFP_KERNEL);

		if (!ip) {

			ret = ERR_PTR(-ENOMEM);

			goto bail_wc;

		}

		cq->ip = ip;

		ip->context = context;

		ip->obj = wc;

		kref_init(&ip->ref);

		ip->mmap_cnt = 0;

		ip->size = PAGE_ALIGN(sizeof(*wc) +

				      sizeof(struct ib_wc) * entries);

		spin_lock_irq(&dev->pending_lock);

		ip->next = dev->pending_mmaps;

		dev->pending_mmaps = ip;

		spin_unlock_irq(&dev->pending_lock);

	} else

		cq->ip = NULL;

	/*

	 * ib_create_cq() will initialize cq->ibcq except for cq->ibcq.cqe.

	 * The number of entries should be >= the number requested or return

	cq->triggered = 0;

	spin_lock_init(&cq->lock);

	tasklet_init(&cq->comptask, send_complete, (unsigned long)cq);

	cq->head = 0;

	cq->tail = 0;

	wc->head = 0;

	wc->tail = 0;

	cq->queue = wc;

	ret = &cq->ibcq;

	dev->n_cqs_allocated++;

	goto done;

bail:

bail_wc:

	vfree(wc);

bail_cq:

	kfree(cq);

done:

	return ret;

}

	tasklet_kill(&cq->comptask);

	dev->n_cqs_allocated--;

	if (cq->ip)

		kref_put(&cq->ip->ref, ipath_release_mmap_info);

	else

		vfree(cq->queue);

	kfree(cq);

	spin_lock_irqsave(&cq->lock, flags);

	/*

	 * Don't change IB_CQ_NEXT_COMP to IB_CQ_SOLICITED but allow

	 * any other transitions.

	 * any other transitions (see C11-31 and C11-32 in ch. 11.4.2.2).

	 */

	if (cq->notify != IB_CQ_NEXT_COMP)

		cq->notify = notify;

int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)

{

	struct ipath_cq *cq = to_icq(ibcq);

	struct ib_wc *wc, *old_wc;

	u32 n;

	struct ipath_cq_wc *old_wc = cq->queue;

	struct ipath_cq_wc *wc;

	u32 head, tail, n;

	int ret;

	/*

	 * Need to use vmalloc() if we want to support large #s of entries.

	 */

	wc = vmalloc(sizeof(*wc) * (cqe + 1));

	wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * cqe);

	if (!wc) {

		ret = -ENOMEM;

		goto bail;

	}

	/*

	 * Return the address of the WC as the offset to mmap.

	 * See ipath_mmap() for details.

	 */

	if (udata && udata->outlen >= sizeof(__u64)) {

		__u64 offset = (__u64) wc;

		ret = ib_copy_to_udata(udata, &offset, sizeof(offset));

		if (ret)

			goto bail;

	}

	spin_lock_irq(&cq->lock);

	if (cq->head < cq->tail)

		n = cq->ibcq.cqe + 1 + cq->head - cq->tail;

	/*

	 * Make sure head and tail are sane since they

	 * might be user writable.

	 */

	head = old_wc->head;

	if (head > (u32) cq->ibcq.cqe)

		head = (u32) cq->ibcq.cqe;

	tail = old_wc->tail;

	if (tail > (u32) cq->ibcq.cqe)

		tail = (u32) cq->ibcq.cqe;

	if (head < tail)

		n = cq->ibcq.cqe + 1 + head - tail;

	else

		n = cq->head - cq->tail;

		n = head - tail;

	if (unlikely((u32)cqe < n)) {

		spin_unlock_irq(&cq->lock);

		vfree(wc);

		ret = -EOVERFLOW;

		goto bail;

	}

	for (n = 0; cq->tail != cq->head; n++) {

		wc[n] = cq->queue[cq->tail];

		if (cq->tail == cq->ibcq.cqe)

			cq->tail = 0;

	for (n = 0; tail != head; n++) {

		wc->queue[n] = old_wc->queue[tail];

		if (tail == (u32) cq->ibcq.cqe)

			tail = 0;

		else

			cq->tail++;

			tail++;

	}

	cq->ibcq.cqe = cqe;

	cq->head = n;

	cq->tail = 0;

	old_wc = cq->queue;

	wc->head = n;

	wc->tail = 0;

	cq->queue = wc;

	spin_unlock_irq(&cq->lock);

	vfree(old_wc);

	if (cq->ip) {

		struct ipath_ibdev *dev = to_idev(ibcq->device);

		struct ipath_mmap_info *ip = cq->ip;

		ip->obj = wc;

		ip->size = PAGE_ALIGN(sizeof(*wc) +

				      sizeof(struct ib_wc) * cqe);

		spin_lock_irq(&dev->pending_lock);

		ip->next = dev->pending_mmaps;

		dev->pending_mmaps = ip;

		spin_unlock_irq(&dev->pending_lock);

	}

	ret = 0;

bail:

/*

 * Copyright (c) 2006 QLogic, Inc. All rights reserved.

 *

 * This software is available to you under a choice of one of two

 * licenses.  You may choose to be licensed under the terms of the GNU

 * General Public License (GPL) Version 2, available from the file

 * COPYING in the main directory of this source tree, or the

 * OpenIB.org BSD license below:

 *

 *     Redistribution and use in source and binary forms, with or

 *     without modification, are permitted provided that the following

 *     conditions are met:

 *

 *      - Redistributions of source code must retain the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer.

 *

 *      - Redistributions in binary form must reproduce the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer in the documentation and/or other materials

 *        provided with the distribution.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

 * SOFTWARE.

 */

#include <linux/config.h>

#include <linux/module.h>

#include <linux/vmalloc.h>

#include <linux/mm.h>

#include <linux/errno.h>

#include <asm/pgtable.h>

#include "ipath_verbs.h"

/**

 * ipath_release_mmap_info - free mmap info structure

 * @ref: a pointer to the kref within struct ipath_mmap_info

 */

void ipath_release_mmap_info(struct kref *ref)

{

	struct ipath_mmap_info *ip =

		container_of(ref, struct ipath_mmap_info, ref);

	vfree(ip->obj);

	kfree(ip);

}

/*

 * open and close keep track of how many times the CQ is mapped,

 * to avoid releasing it.

 */

static void ipath_vma_open(struct vm_area_struct *vma)

{

	struct ipath_mmap_info *ip = vma->vm_private_data;

	kref_get(&ip->ref);

	ip->mmap_cnt++;

}

static void ipath_vma_close(struct vm_area_struct *vma)

{

	struct ipath_mmap_info *ip = vma->vm_private_data;

	ip->mmap_cnt--;

	kref_put(&ip->ref, ipath_release_mmap_info);

}

static struct vm_operations_struct ipath_vm_ops = {

	.open =     ipath_vma_open,

	.close =    ipath_vma_close,

};

/**

 * ipath_mmap - create a new mmap region

 * @context: the IB user context of the process making the mmap() call

 * @vma: the VMA to be initialized

 * Return zero if the mmap is OK. Otherwise, return an errno.

 */

int ipath_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)

{

	struct ipath_ibdev *dev = to_idev(context->device);

	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;

	unsigned long size = vma->vm_end - vma->vm_start;

	struct ipath_mmap_info *ip, **pp;

	int ret = -EINVAL;

	/*

	 * Search the device's list of objects waiting for a mmap call.

	 * Normally, this list is very short since a call to create a

	 * CQ, QP, or SRQ is soon followed by a call to mmap().

	 */

	spin_lock_irq(&dev->pending_lock);

	for (pp = &dev->pending_mmaps; (ip = *pp); pp = &ip->next) {

		/* Only the creator is allowed to mmap the object */

		if (context != ip->context || (void *) offset != ip->obj)

			continue;

		/* Don't allow a mmap larger than the object. */

		if (size > ip->size)

			break;

		*pp = ip->next;

		spin_unlock_irq(&dev->pending_lock);

		ret = remap_vmalloc_range(vma, ip->obj, 0);

		if (ret)

			goto done;

		vma->vm_ops = &ipath_vm_ops;

		vma->vm_private_data = ip;

		ipath_vma_open(vma);

		goto done;

	}

	spin_unlock_irq(&dev->pending_lock);

done:

	return ret;

}

#include <linux/vmalloc.h>

#include "ipath_verbs.h"

#include "ipath_common.h"

#include "ipath_kernel.h"

#define BITS_PER_PAGE		(PAGE_SIZE*BITS_PER_BYTE)

#define BITS_PER_PAGE_MASK	(BITS_PER_PAGE-1)

#define find_next_offset(map, off) find_next_zero_bit((map)->page, \

						      BITS_PER_PAGE, off)

#define TRANS_INVALID	0

#define TRANS_ANY2RST	1

#define TRANS_RST2INIT	2

#define TRANS_INIT2INIT	3

#define TRANS_INIT2RTR	4

#define TRANS_RTR2RTS	5

#define TRANS_RTS2RTS	6

#define TRANS_SQERR2RTS	7

#define TRANS_ANY2ERR	8

#define TRANS_RTS2SQD	9  /* XXX Wait for expected ACKs & signal event */

#define TRANS_SQD2SQD	10 /* error if not drained & parameter change */

#define TRANS_SQD2RTS	11 /* error if not drained */

/*

 * Convert the AETH credit code into the number of credits.

 */

	qp->s_last = 0;

	qp->s_ssn = 1;

	qp->s_lsn = 0;

	qp->r_rq.head = 0;

	qp->r_rq.tail = 0;

	if (qp->r_rq.wq) {

		qp->r_rq.wq->head = 0;

		qp->r_rq.wq->tail = 0;

	}

	qp->r_reuse_sge = 0;

}

	qp->s_hdrwords = 0;

	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;

	wc.opcode = IB_WC_RECV;

	if (qp->r_rq.wq) {

		struct ipath_rwq *wq;

		u32 head;

		u32 tail;

		spin_lock(&qp->r_rq.lock);

	while (qp->r_rq.tail != qp->r_rq.head) {

		wc.wr_id = get_rwqe_ptr(&qp->r_rq, qp->r_rq.tail)->wr_id;

		if (++qp->r_rq.tail >= qp->r_rq.size)

			qp->r_rq.tail = 0;

		/* sanity check pointers before trusting them */

		wq = qp->r_rq.wq;

		head = wq->head;

		if (head >= qp->r_rq.size)

			head = 0;

		tail = wq->tail;

		if (tail >= qp->r_rq.size)

			tail = 0;

		wc.opcode = IB_WC_RECV;

		while (tail != head) {

			wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;

			if (++tail >= qp->r_rq.size)

				tail = 0;

			ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);

		}

		wq->tail = tail;

		spin_unlock(&qp->r_rq.lock);

	}

}

/**

 * ipath_modify_qp - modify the attributes of a queue pair

	attr->dest_qp_num = qp->remote_qpn;

	attr->qp_access_flags = qp->qp_access_flags;

	attr->cap.max_send_wr = qp->s_size - 1;

	attr->cap.max_recv_wr = qp->r_rq.size - 1;

	attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;

	attr->cap.max_send_sge = qp->s_max_sge;

	attr->cap.max_recv_sge = qp->r_rq.max_sge;

	attr->cap.max_inline_data = 0;

	} else {

		u32 min, max, x;

		u32 credits;

		struct ipath_rwq *wq = qp->r_rq.wq;

		u32 head;

		u32 tail;

		/* sanity check pointers before trusting them */

		head = wq->head;

		if (head >= qp->r_rq.size)

			head = 0;

		tail = wq->tail;

		if (tail >= qp->r_rq.size)

			tail = 0;

		/*

		 * Compute the number of credits available (RWQEs).

		 * XXX Not holding the r_rq.lock here so there is a small

		 * chance that the pair of reads are not atomic.

		 */

		credits = qp->r_rq.head - qp->r_rq.tail;

		credits = head - tail;

		if ((int)credits < 0)

			credits += qp->r_rq.size;

		/*

	case IB_QPT_UD:

	case IB_QPT_SMI:

	case IB_QPT_GSI:

		qp = kmalloc(sizeof(*qp), GFP_KERNEL);

		sz = sizeof(*qp);

		if (init_attr->srq) {

			struct ipath_srq *srq = to_isrq(init_attr->srq);

			sz += sizeof(*qp->r_sg_list) *

				srq->rq.max_sge;

		} else

			sz += sizeof(*qp->r_sg_list) *

				init_attr->cap.max_recv_sge;

		qp = kmalloc(sz, GFP_KERNEL);

		if (!qp) {

			vfree(swq);

			ret = ERR_PTR(-ENOMEM);

			goto bail;

			goto bail_swq;

		}

		if (init_attr->srq) {

			sz = 0;

			qp->r_rq.size = 0;

			qp->r_rq.max_sge = 0;

			qp->r_rq.wq = NULL;

			init_attr->cap.max_recv_wr = 0;

			init_attr->cap.max_recv_sge = 0;

		} else {

			qp->r_rq.size = init_attr->cap.max_recv_wr + 1;

			qp->r_rq.max_sge = init_attr->cap.max_recv_sge;

			sz = (sizeof(struct ipath_sge) * qp->r_rq.max_sge) +

			sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +

				sizeof(struct ipath_rwqe);

			qp->r_rq.wq = vmalloc(qp->r_rq.size * sz);

			qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) +

					      qp->r_rq.size * sz);

			if (!qp->r_rq.wq) {

				kfree(qp);

				vfree(swq);

				ret = ERR_PTR(-ENOMEM);

				goto bail;

				goto bail_qp;

			}

		}

		err = ipath_alloc_qpn(&dev->qp_table, qp,

				      init_attr->qp_type);

		if (err) {

			vfree(swq);

			vfree(qp->r_rq.wq);

			kfree(qp);

			ret = ERR_PTR(err);

			goto bail;

			goto bail_rwq;

		}

		qp->ip = NULL;

		ipath_reset_qp(qp);

		/* Tell the core driver that the kernel SMA is present. */

	init_attr->cap.max_inline_data = 0;

	/*

	 * Return the address of the RWQ as the offset to mmap.

	 * See ipath_mmap() for details.

	 */

	if (udata && udata->outlen >= sizeof(__u64)) {

		struct ipath_mmap_info *ip;

		__u64 offset = (__u64) qp->r_rq.wq;

		int err;

		err = ib_copy_to_udata(udata, &offset, sizeof(offset));

		if (err) {

			ret = ERR_PTR(err);

			goto bail_rwq;

		}

		if (qp->r_rq.wq) {

			/* Allocate info for ipath_mmap(). */

			ip = kmalloc(sizeof(*ip), GFP_KERNEL);

			if (!ip) {