IB/ipath: Fix many locking issues when switching to error state (e509be89) · Commits · e / devices / android_kernel_xiaomi_nabu

drivers/infiniband/hw/ipath/ipath_qp.c

+91 −92

Original line number	Original line	Diff line number	Diff line
	@@ -242,7 +242,6 @@ static void ipath_free_qp(struct ipath_qp_table qpt, struct ipath_qp qp)
	{		{
	struct ipath_qp q, *qpp;		struct ipath_qp q, *qpp;
	unsigned long flags;		unsigned long flags;
	int fnd = 0;

	spin_lock_irqsave(&qpt->lock, flags);		spin_lock_irqsave(&qpt->lock, flags);

	@@ -253,51 +252,40 @@ static void ipath_free_qp(struct ipath_qp_table qpt, struct ipath_qp qp)
	*qpp = qp->next;		*qpp = qp->next;
	qp->next = NULL;		qp->next = NULL;
	atomic_dec(&qp->refcount);		atomic_dec(&qp->refcount);
	fnd = 1;
	break;		break;
	}		}
	}		}

	spin_unlock_irqrestore(&qpt->lock, flags);		spin_unlock_irqrestore(&qpt->lock, flags);

	if (!fnd)
	return;

	free_qpn(qpt, qp->ibqp.qp_num);

	wait_event(qp->wait, !atomic_read(&qp->refcount));
	}		}

	/**		/**
	* ipath_free_all_qps - remove all QPs from the table		* ipath_free_all_qps - check for QPs still in use
	* @qpt: the QP table to empty		* @qpt: the QP table to empty
			*
			* There should not be any QPs still in use.
			* Free memory for table.
	*/		*/
	void ipath_free_all_qps(struct ipath_qp_table *qpt)		unsigned ipath_free_all_qps(struct ipath_qp_table *qpt)
	{		{
	unsigned long flags;		unsigned long flags;
	struct ipath_qp qp, nqp;		struct ipath_qp *qp;
	u32 n;		u32 n, qp_inuse = 0;

	for (n = 0; n < qpt->max; n++) {
	spin_lock_irqsave(&qpt->lock, flags);		spin_lock_irqsave(&qpt->lock, flags);
			for (n = 0; n < qpt->max; n++) {
	qp = qpt->table[n];		qp = qpt->table[n];
	qpt->table[n] = NULL;		qpt->table[n] = NULL;
	spin_unlock_irqrestore(&qpt->lock, flags);

	while (qp) {		for (; qp; qp = qp->next)
	nqp = qp->next;		qp_inuse++;
	free_qpn(qpt, qp->ibqp.qp_num);
	if (!atomic_dec_and_test(&qp->refcount) \|\|
	!ipath_destroy_qp(&qp->ibqp))
	ipath_dbg("QP memory leak!\n");
	qp = nqp;
	}
	}		}
			spin_unlock_irqrestore(&qpt->lock, flags);

	for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {		for (n = 0; n < ARRAY_SIZE(qpt->map); n++)
	if (qpt->map[n].page)		if (qpt->map[n].page)
	free_page((unsigned long) qpt->map[n].page);		free_page((unsigned long) qpt->map[n].page);
	}		return qp_inuse;
	}		}

	/**		/**
	@@ -336,11 +324,12 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
	qp->remote_qpn = 0;		qp->remote_qpn = 0;
	qp->qkey = 0;		qp->qkey = 0;
	qp->qp_access_flags = 0;		qp->qp_access_flags = 0;
	qp->s_busy = 0;		atomic_set(&qp->s_dma_busy, 0);
	qp->s_flags &= IPATH_S_SIGNAL_REQ_WR;		qp->s_flags &= IPATH_S_SIGNAL_REQ_WR;
	qp->s_hdrwords = 0;		qp->s_hdrwords = 0;
	qp->s_wqe = NULL;		qp->s_wqe = NULL;
	qp->s_pkt_delay = 0;		qp->s_pkt_delay = 0;
			qp->s_draining = 0;
	qp->s_psn = 0;		qp->s_psn = 0;
	qp->r_psn = 0;		qp->r_psn = 0;
	qp->r_msn = 0;		qp->r_msn = 0;
	@@ -353,7 +342,8 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
	}		}
	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;		qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
	qp->r_nak_state = 0;		qp->r_nak_state = 0;
	qp->r_wrid_valid = 0;		qp->r_aflags = 0;
			qp->r_flags = 0;
	qp->s_rnr_timeout = 0;		qp->s_rnr_timeout = 0;
	qp->s_head = 0;		qp->s_head = 0;
	qp->s_tail = 0;		qp->s_tail = 0;
	@@ -361,7 +351,6 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
	qp->s_last = 0;		qp->s_last = 0;
	qp->s_ssn = 1;		qp->s_ssn = 1;
	qp->s_lsn = 0;		qp->s_lsn = 0;
	qp->s_wait_credit = 0;
	memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));		memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));
	qp->r_head_ack_queue = 0;		qp->r_head_ack_queue = 0;
	qp->s_tail_ack_queue = 0;		qp->s_tail_ack_queue = 0;
	@@ -370,7 +359,6 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
	qp->r_rq.wq->head = 0;		qp->r_rq.wq->head = 0;
	qp->r_rq.wq->tail = 0;		qp->r_rq.wq->tail = 0;
	}		}
	qp->r_reuse_sge = 0;
	}		}

	/**		/**
	@@ -402,39 +390,21 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
	list_del_init(&qp->piowait);		list_del_init(&qp->piowait);
	spin_unlock(&dev->pending_lock);		spin_unlock(&dev->pending_lock);

	wc.vendor_err = 0;		/* Schedule the sending tasklet to drain the send work queue. */
	wc.byte_len = 0;		if (qp->s_last != qp->s_head)
	wc.imm_data = 0;		ipath_schedule_send(qp);

			memset(&wc, 0, sizeof(wc));
	wc.qp = &qp->ibqp;		wc.qp = &qp->ibqp;
	wc.src_qp = 0;
	wc.wc_flags = 0;
	wc.pkey_index = 0;
	wc.slid = 0;
	wc.sl = 0;
	wc.dlid_path_bits = 0;
	wc.port_num = 0;
	if (qp->r_wrid_valid) {
	qp->r_wrid_valid = 0;
	wc.wr_id = qp->r_wr_id;
	wc.opcode = IB_WC_RECV;		wc.opcode = IB_WC_RECV;

			if (test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) {
			wc.wr_id = qp->r_wr_id;
	wc.status = err;		wc.status = err;
	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
	}		}
	wc.status = IB_WC_WR_FLUSH_ERR;		wc.status = IB_WC_WR_FLUSH_ERR;

	while (qp->s_last != qp->s_head) {
	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);

	wc.wr_id = wqe->wr.wr_id;
	wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
	if (++qp->s_last >= qp->s_size)
	qp->s_last = 0;
	ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
	}
	qp->s_cur = qp->s_tail = qp->s_head;
	qp->s_hdrwords = 0;
	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;

	if (qp->r_rq.wq) {		if (qp->r_rq.wq) {
	struct ipath_rwq *wq;		struct ipath_rwq *wq;
	u32 head;		u32 head;
	@@ -450,7 +420,6 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
	tail = wq->tail;		tail = wq->tail;
	if (tail >= qp->r_rq.size)		if (tail >= qp->r_rq.size)
	tail = 0;		tail = 0;
	wc.opcode = IB_WC_RECV;
	while (tail != head) {		while (tail != head) {
	wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;		wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
	if (++tail >= qp->r_rq.size)		if (++tail >= qp->r_rq.size)
	@@ -482,11 +451,10 @@ int ipath_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	struct ipath_ibdev *dev = to_idev(ibqp->device);		struct ipath_ibdev *dev = to_idev(ibqp->device);
	struct ipath_qp *qp = to_iqp(ibqp);		struct ipath_qp *qp = to_iqp(ibqp);
	enum ib_qp_state cur_state, new_state;		enum ib_qp_state cur_state, new_state;
	unsigned long flags;
	int lastwqe = 0;		int lastwqe = 0;
	int ret;		int ret;

	spin_lock_irqsave(&qp->s_lock, flags);		spin_lock_irq(&qp->s_lock);

	cur_state = attr_mask & IB_QP_CUR_STATE ?		cur_state = attr_mask & IB_QP_CUR_STATE ?
	attr->cur_qp_state : qp->state;		attr->cur_qp_state : qp->state;
	@@ -539,16 +507,42 @@ int ipath_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,

	switch (new_state) {		switch (new_state) {
	case IB_QPS_RESET:		case IB_QPS_RESET:
			if (qp->state != IB_QPS_RESET) {
			qp->state = IB_QPS_RESET;
			spin_lock(&dev->pending_lock);
			if (!list_empty(&qp->timerwait))
			list_del_init(&qp->timerwait);
			if (!list_empty(&qp->piowait))
			list_del_init(&qp->piowait);
			spin_unlock(&dev->pending_lock);
			qp->s_flags &= ~IPATH_S_ANY_WAIT;
			spin_unlock_irq(&qp->s_lock);
			/* Stop the sending tasklet */
			tasklet_kill(&qp->s_task);
			wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
			spin_lock_irq(&qp->s_lock);
			}
	ipath_reset_qp(qp, ibqp->qp_type);		ipath_reset_qp(qp, ibqp->qp_type);
	break;		break;

			case IB_QPS_SQD:
			qp->s_draining = qp->s_last != qp->s_cur;
			qp->state = new_state;
			break;

			case IB_QPS_SQE:
			if (qp->ibqp.qp_type == IB_QPT_RC)
			goto inval;
			qp->state = new_state;
			break;

	case IB_QPS_ERR:		case IB_QPS_ERR:
	lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);		lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
	break;		break;

	default:		default:
			qp->state = new_state;
	break;		break;

	}		}

	if (attr_mask & IB_QP_PKEY_INDEX)		if (attr_mask & IB_QP_PKEY_INDEX)
	@@ -601,8 +595,7 @@ int ipath_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)		if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
	qp->s_max_rd_atomic = attr->max_rd_atomic;		qp->s_max_rd_atomic = attr->max_rd_atomic;

	qp->state = new_state;		spin_unlock_irq(&qp->s_lock);
	spin_unlock_irqrestore(&qp->s_lock, flags);

	if (lastwqe) {		if (lastwqe) {
	struct ib_event ev;		struct ib_event ev;
	@@ -616,7 +609,7 @@ int ipath_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	goto bail;		goto bail;

	inval:		inval:
	spin_unlock_irqrestore(&qp->s_lock, flags);		spin_unlock_irq(&qp->s_lock);
	ret = -EINVAL;		ret = -EINVAL;

	bail:		bail:
	@@ -647,7 +640,7 @@ int ipath_query_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	attr->pkey_index = qp->s_pkey_index;		attr->pkey_index = qp->s_pkey_index;
	attr->alt_pkey_index = 0;		attr->alt_pkey_index = 0;
	attr->en_sqd_async_notify = 0;		attr->en_sqd_async_notify = 0;
	attr->sq_draining = 0;		attr->sq_draining = qp->s_draining;
	attr->max_rd_atomic = qp->s_max_rd_atomic;		attr->max_rd_atomic = qp->s_max_rd_atomic;
	attr->max_dest_rd_atomic = qp->r_max_rd_atomic;		attr->max_dest_rd_atomic = qp->r_max_rd_atomic;
	attr->min_rnr_timer = qp->r_min_rnr_timer;		attr->min_rnr_timer = qp->r_min_rnr_timer;
	@@ -837,6 +830,7 @@ struct ib_qp ipath_create_qp(struct ib_pd ibpd,
	spin_lock_init(&qp->r_rq.lock);		spin_lock_init(&qp->r_rq.lock);
	atomic_set(&qp->refcount, 0);		atomic_set(&qp->refcount, 0);
	init_waitqueue_head(&qp->wait);		init_waitqueue_head(&qp->wait);
			init_waitqueue_head(&qp->wait_dma);
	tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp);		tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp);
	INIT_LIST_HEAD(&qp->piowait);		INIT_LIST_HEAD(&qp->piowait);
	INIT_LIST_HEAD(&qp->timerwait);		INIT_LIST_HEAD(&qp->timerwait);
	@@ -930,6 +924,7 @@ struct ib_qp ipath_create_qp(struct ib_pd ibpd,
	else		else
	vfree(qp->r_rq.wq);		vfree(qp->r_rq.wq);
	ipath_free_qp(&dev->qp_table, qp);		ipath_free_qp(&dev->qp_table, qp);
			free_qpn(&dev->qp_table, qp->ibqp.qp_num);
	bail_qp:		bail_qp:
	kfree(qp);		kfree(qp);
	bail_swq:		bail_swq:
	@@ -951,41 +946,44 @@ int ipath_destroy_qp(struct ib_qp *ibqp)
	{		{
	struct ipath_qp *qp = to_iqp(ibqp);		struct ipath_qp *qp = to_iqp(ibqp);
	struct ipath_ibdev *dev = to_idev(ibqp->device);		struct ipath_ibdev *dev = to_idev(ibqp->device);
	unsigned long flags;

	spin_lock_irqsave(&qp->s_lock, flags);		/* Make sure HW and driver activity is stopped. */
	qp->state = IB_QPS_ERR;		spin_lock_irq(&qp->s_lock);
	spin_unlock_irqrestore(&qp->s_lock, flags);		if (qp->state != IB_QPS_RESET) {
	spin_lock(&dev->n_qps_lock);		qp->state = IB_QPS_RESET;
	dev->n_qps_allocated--;		spin_lock(&dev->pending_lock);
	spin_unlock(&dev->n_qps_lock);		if (!list_empty(&qp->timerwait))
			list_del_init(&qp->timerwait);
	/* Stop the sending tasklet. */		if (!list_empty(&qp->piowait))
			list_del_init(&qp->piowait);
			spin_unlock(&dev->pending_lock);
			qp->s_flags &= ~IPATH_S_ANY_WAIT;
			spin_unlock_irq(&qp->s_lock);
			/* Stop the sending tasklet */
	tasklet_kill(&qp->s_task);		tasklet_kill(&qp->s_task);
			wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
			} else
			spin_unlock_irq(&qp->s_lock);

			ipath_free_qp(&dev->qp_table, qp);

	if (qp->s_tx) {		if (qp->s_tx) {
	atomic_dec(&qp->refcount);		atomic_dec(&qp->refcount);
	if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)		if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)
	kfree(qp->s_tx->txreq.map_addr);		kfree(qp->s_tx->txreq.map_addr);
			spin_lock_irq(&dev->pending_lock);
			list_add(&qp->s_tx->txreq.list, &dev->txreq_free);
			spin_unlock_irq(&dev->pending_lock);
			qp->s_tx = NULL;
	}		}

	/* Make sure the QP isn't on the timeout list. */		wait_event(qp->wait, !atomic_read(&qp->refcount));
	spin_lock_irqsave(&dev->pending_lock, flags);
	if (!list_empty(&qp->timerwait))
	list_del_init(&qp->timerwait);
	if (!list_empty(&qp->piowait))
	list_del_init(&qp->piowait);
	if (qp->s_tx)
	list_add(&qp->s_tx->txreq.list, &dev->txreq_free);
	spin_unlock_irqrestore(&dev->pending_lock, flags);

	/*		/* all user's cleaned up, mark it available */
	* Make sure that the QP is not in the QPN table so receive		free_qpn(&dev->qp_table, qp->ibqp.qp_num);
	* interrupts will discard packets for this QP. XXX Also remove QP		spin_lock(&dev->n_qps_lock);
	* from multicast table.		dev->n_qps_allocated--;
	*/		spin_unlock(&dev->n_qps_lock);
	if (atomic_read(&qp->refcount) != 0)
	ipath_free_qp(&dev->qp_table, qp);

	if (qp->ip)		if (qp->ip)
	kref_put(&qp->ip->ref, ipath_release_mmap_info);		kref_put(&qp->ip->ref, ipath_release_mmap_info);
	@@ -1055,9 +1053,10 @@ void ipath_get_credit(struct ipath_qp *qp, u32 aeth)
	}		}

	/* Restart sending if it was blocked due to lack of credits. */		/* Restart sending if it was blocked due to lack of credits. */
	if (qp->s_cur != qp->s_head &&		if ((qp->s_flags & IPATH_S_WAIT_SSN_CREDIT) &&
			qp->s_cur != qp->s_head &&
	(qp->s_lsn == (u32) -1 \|\|		(qp->s_lsn == (u32) -1 \|\|
	ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,		ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,
	qp->s_lsn + 1) <= 0))		qp->s_lsn + 1) <= 0))
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);
	}		}

drivers/infiniband/hw/ipath/ipath_rc.c

+97 −54

Original line number	Original line	Diff line number	Diff line
	@@ -92,6 +92,10 @@ static int ipath_make_rc_ack(struct ipath_ibdev dev, struct ipath_qp qp,
	u32 bth0;		u32 bth0;
	u32 bth2;		u32 bth2;

			/* Don't send an ACK if we aren't supposed to. */
			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
			goto bail;

	/* header size in 32-bit words LRH+BTH = (8+12)/4. */		/* header size in 32-bit words LRH+BTH = (8+12)/4. */
	hwords = 5;		hwords = 5;

	@@ -238,14 +242,25 @@ int ipath_make_rc_req(struct ipath_qp *qp)
	ipath_make_rc_ack(dev, qp, ohdr, pmtu))		ipath_make_rc_ack(dev, qp, ohdr, pmtu))
	goto done;		goto done;

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) \|\|		if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {
	qp->s_rnr_timeout \|\| qp->s_wait_credit)		if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
			goto bail;
			/* We are in the error state, flush the work request. */
			if (qp->s_last == qp->s_head)
	goto bail;		goto bail;
			/* If DMAs are in progress, we can't flush immediately. */
			if (atomic_read(&qp->s_dma_busy)) {
			qp->s_flags \|= IPATH_S_WAIT_DMA;
			goto bail;
			}
			wqe = get_swqe_ptr(qp, qp->s_last);
			ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
			goto done;
			}

	/* Limit the number of packets sent without an ACK. */		/* Leave BUSY set until RNR timeout. */
	if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) {		if (qp->s_rnr_timeout) {
	qp->s_wait_credit = 1;		qp->s_flags \|= IPATH_S_WAITING;
	dev->n_rc_stalls++;
	goto bail;		goto bail;
	}		}

	@@ -257,6 +272,9 @@ int ipath_make_rc_req(struct ipath_qp *qp)
	wqe = get_swqe_ptr(qp, qp->s_cur);		wqe = get_swqe_ptr(qp, qp->s_cur);
	switch (qp->s_state) {		switch (qp->s_state) {
	default:		default:
			if (!(ib_ipath_state_ops[qp->state] &
			IPATH_PROCESS_NEXT_SEND_OK))
			goto bail;
	/*		/*
	* Resend an old request or start a new one.		* Resend an old request or start a new one.
	*		*
	@@ -294,8 +312,10 @@ int ipath_make_rc_req(struct ipath_qp *qp)
	case IB_WR_SEND_WITH_IMM:		case IB_WR_SEND_WITH_IMM:
	/* If no credit, return. */		/* If no credit, return. */
	if (qp->s_lsn != (u32) -1 &&		if (qp->s_lsn != (u32) -1 &&
	ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)		ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
			qp->s_flags \|= IPATH_S_WAIT_SSN_CREDIT;
	goto bail;		goto bail;
			}
	wqe->lpsn = wqe->psn;		wqe->lpsn = wqe->psn;
	if (len > pmtu) {		if (len > pmtu) {
	wqe->lpsn += (len - 1) / pmtu;		wqe->lpsn += (len - 1) / pmtu;
	@@ -325,8 +345,10 @@ int ipath_make_rc_req(struct ipath_qp *qp)
	case IB_WR_RDMA_WRITE_WITH_IMM:		case IB_WR_RDMA_WRITE_WITH_IMM:
	/* If no credit, return. */		/* If no credit, return. */
	if (qp->s_lsn != (u32) -1 &&		if (qp->s_lsn != (u32) -1 &&
	ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)		ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
			qp->s_flags \|= IPATH_S_WAIT_SSN_CREDIT;
	goto bail;		goto bail;
			}
	ohdr->u.rc.reth.vaddr =		ohdr->u.rc.reth.vaddr =
	cpu_to_be64(wqe->wr.wr.rdma.remote_addr);		cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
	ohdr->u.rc.reth.rkey =		ohdr->u.rc.reth.rkey =
	@@ -570,7 +592,11 @@ int ipath_make_rc_req(struct ipath_qp *qp)
	ipath_make_ruc_header(dev, qp, ohdr, bth0 \| (qp->s_state << 24), bth2);		ipath_make_ruc_header(dev, qp, ohdr, bth0 \| (qp->s_state << 24), bth2);
	done:		done:
	ret = 1;		ret = 1;
			goto unlock;

	bail:		bail:
			qp->s_flags &= ~IPATH_S_BUSY;
			unlock:
	spin_unlock_irqrestore(&qp->s_lock, flags);		spin_unlock_irqrestore(&qp->s_lock, flags);
	return ret;		return ret;
	}		}
	@@ -606,7 +632,11 @@ static void send_rc_ack(struct ipath_qp *qp)

	spin_unlock_irqrestore(&qp->s_lock, flags);		spin_unlock_irqrestore(&qp->s_lock, flags);

			/* Don't try to send ACKs if the link isn't ACTIVE */
	dd = dev->dd;		dd = dev->dd;
			if (!(dd->ipath_flags & IPATH_LINKACTIVE))
			goto done;

	piobuf = ipath_getpiobuf(dd, 0, NULL);		piobuf = ipath_getpiobuf(dd, 0, NULL);
	if (!piobuf) {		if (!piobuf) {
	/*		/*
	@@ -668,15 +698,16 @@ static void send_rc_ack(struct ipath_qp *qp)
	goto done;		goto done;

	queue_ack:		queue_ack:
			if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK) {
	dev->n_rc_qacks++;		dev->n_rc_qacks++;
	qp->s_flags \|= IPATH_S_ACK_PENDING;		qp->s_flags \|= IPATH_S_ACK_PENDING;
	qp->s_nak_state = qp->r_nak_state;		qp->s_nak_state = qp->r_nak_state;
	qp->s_ack_psn = qp->r_ack_psn;		qp->s_ack_psn = qp->r_ack_psn;
	spin_unlock_irqrestore(&qp->s_lock, flags);

	/* Call ipath_do_rc_send() in another thread. */
	tasklet_hi_schedule(&qp->s_task);

			/* Schedule the send tasklet. */
			ipath_schedule_send(qp);
			}
			spin_unlock_irqrestore(&qp->s_lock, flags);
	done:		done:
	return;		return;
	}		}
	@@ -735,7 +766,7 @@ static void reset_psn(struct ipath_qp *qp, u32 psn)
	/*		/*
	* Set the state to restart in the middle of a request.		* Set the state to restart in the middle of a request.
	* Don't change the s_sge, s_cur_sge, or s_cur_size.		* Don't change the s_sge, s_cur_sge, or s_cur_size.
	* See ipath_do_rc_send().		* See ipath_make_rc_req().
	*/		*/
	switch (opcode) {		switch (opcode) {
	case IB_WR_SEND:		case IB_WR_SEND:
	@@ -801,7 +832,7 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn)
	dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK;		dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK;

	reset_psn(qp, psn);		reset_psn(qp, psn);
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);

	bail:		bail:
	return;		return;
	@@ -809,13 +840,7 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn)

	static inline void update_last_psn(struct ipath_qp *qp, u32 psn)		static inline void update_last_psn(struct ipath_qp *qp, u32 psn)
	{		{
	if (qp->s_last_psn != psn) {
	qp->s_last_psn = psn;		qp->s_last_psn = psn;
	if (qp->s_wait_credit) {
	qp->s_wait_credit = 0;
	tasklet_hi_schedule(&qp->s_task);
	}
	}
	}		}

	/**		/**
	@@ -915,14 +940,10 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
	wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {		wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
	qp->s_num_rd_atomic--;		qp->s_num_rd_atomic--;
	/* Restart sending task if fence is complete */		/* Restart sending task if fence is complete */
	if ((qp->s_flags & IPATH_S_FENCE_PENDING) &&		if (((qp->s_flags & IPATH_S_FENCE_PENDING) &&
	!qp->s_num_rd_atomic) {		!qp->s_num_rd_atomic) \|\|
	qp->s_flags &= ~IPATH_S_FENCE_PENDING;		qp->s_flags & IPATH_S_RDMAR_PENDING)
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);
	} else if (qp->s_flags & IPATH_S_RDMAR_PENDING) {
	qp->s_flags &= ~IPATH_S_RDMAR_PENDING;
	tasklet_hi_schedule(&qp->s_task);
	}
	}		}
	/* Post a send completion queue entry if requested. */		/* Post a send completion queue entry if requested. */
	if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) \|\|		if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) \|\|
	@@ -956,6 +977,8 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
	} else {		} else {
	if (++qp->s_last >= qp->s_size)		if (++qp->s_last >= qp->s_size)
	qp->s_last = 0;		qp->s_last = 0;
			if (qp->state == IB_QPS_SQD && qp->s_last == qp->s_cur)
			qp->s_draining = 0;
	if (qp->s_last == qp->s_tail)		if (qp->s_last == qp->s_tail)
	break;		break;
	wqe = get_swqe_ptr(qp, qp->s_last);		wqe = get_swqe_ptr(qp, qp->s_last);
	@@ -979,7 +1002,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
	*/		*/
	if (ipath_cmp24(qp->s_psn, psn) <= 0) {		if (ipath_cmp24(qp->s_psn, psn) <= 0) {
	reset_psn(qp, psn + 1);		reset_psn(qp, psn + 1);
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);
	}		}
	} else if (ipath_cmp24(qp->s_psn, psn) <= 0) {		} else if (ipath_cmp24(qp->s_psn, psn) <= 0) {
	qp->s_state = OP(SEND_LAST);		qp->s_state = OP(SEND_LAST);
	@@ -1018,6 +1041,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode,
	ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) &		ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) &
	IPATH_AETH_CREDIT_MASK];		IPATH_AETH_CREDIT_MASK];
	ipath_insert_rnr_queue(qp);		ipath_insert_rnr_queue(qp);
			ipath_schedule_send(qp);
	goto bail;		goto bail;

	case 3: /* NAK */		case 3: /* NAK */
	@@ -1108,6 +1132,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,

	spin_lock_irqsave(&qp->s_lock, flags);		spin_lock_irqsave(&qp->s_lock, flags);

			/* Double check we can process this now that we hold the s_lock. */
			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
			goto ack_done;

	/* Ignore invalid responses. */		/* Ignore invalid responses. */
	if (ipath_cmp24(psn, qp->s_next_psn) >= 0)		if (ipath_cmp24(psn, qp->s_next_psn) >= 0)
	goto ack_done;		goto ack_done;
	@@ -1343,7 +1371,12 @@ static inline int ipath_rc_rcv_error(struct ipath_ibdev *dev,
	psn &= IPATH_PSN_MASK;		psn &= IPATH_PSN_MASK;
	e = NULL;		e = NULL;
	old_req = 1;		old_req = 1;

	spin_lock_irqsave(&qp->s_lock, flags);		spin_lock_irqsave(&qp->s_lock, flags);
			/* Double check we can process this now that we hold the s_lock. */
			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
			goto unlock_done;

	for (i = qp->r_head_ack_queue; ; i = prev) {		for (i = qp->r_head_ack_queue; ; i = prev) {
	if (i == qp->s_tail_ack_queue)		if (i == qp->s_tail_ack_queue)
	old_req = 0;		old_req = 0;
	@@ -1471,7 +1504,7 @@ static inline int ipath_rc_rcv_error(struct ipath_ibdev *dev,
	break;		break;
	}		}
	qp->r_nak_state = 0;		qp->r_nak_state = 0;
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);

	unlock_done:		unlock_done:
	spin_unlock_irqrestore(&qp->s_lock, flags);		spin_unlock_irqrestore(&qp->s_lock, flags);
	@@ -1503,18 +1536,15 @@ void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err)

	static inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n)		static inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n)
	{		{
	unsigned long flags;
	unsigned next;		unsigned next;

	next = n + 1;		next = n + 1;
	if (next > IPATH_MAX_RDMA_ATOMIC)		if (next > IPATH_MAX_RDMA_ATOMIC)
	next = 0;		next = 0;
	spin_lock_irqsave(&qp->s_lock, flags);
	if (n == qp->s_tail_ack_queue) {		if (n == qp->s_tail_ack_queue) {
	qp->s_tail_ack_queue = next;		qp->s_tail_ack_queue = next;
	qp->s_ack_state = OP(ACKNOWLEDGE);		qp->s_ack_state = OP(ACKNOWLEDGE);
	}		}
	spin_unlock_irqrestore(&qp->s_lock, flags);
	}		}

	/**		/**
	@@ -1543,6 +1573,7 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	int diff;		int diff;
	struct ib_reth *reth;		struct ib_reth *reth;
	int header_in_data;		int header_in_data;
			unsigned long flags;

	/* Validate the SLID. See Ch. 9.6.1.5 */		/* Validate the SLID. See Ch. 9.6.1.5 */
	if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid))		if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid))
	@@ -1690,9 +1721,8 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	goto nack_inv;		goto nack_inv;
	ipath_copy_sge(&qp->r_sge, data, tlen);		ipath_copy_sge(&qp->r_sge, data, tlen);
	qp->r_msn++;		qp->r_msn++;
	if (!qp->r_wrid_valid)		if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
	break;		break;
	qp->r_wrid_valid = 0;
	wc.wr_id = qp->r_wr_id;		wc.wr_id = qp->r_wr_id;
	wc.status = IB_WC_SUCCESS;		wc.status = IB_WC_SUCCESS;
	if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) \|\|		if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) \|\|
	@@ -1764,9 +1794,13 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	next = qp->r_head_ack_queue + 1;		next = qp->r_head_ack_queue + 1;
	if (next > IPATH_MAX_RDMA_ATOMIC)		if (next > IPATH_MAX_RDMA_ATOMIC)
	next = 0;		next = 0;
			spin_lock_irqsave(&qp->s_lock, flags);
			/* Double check we can process this while holding the s_lock. */
			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
			goto unlock;
	if (unlikely(next == qp->s_tail_ack_queue)) {		if (unlikely(next == qp->s_tail_ack_queue)) {
	if (!qp->s_ack_queue[next].sent)		if (!qp->s_ack_queue[next].sent)
	goto nack_inv;		goto nack_inv_unlck;
	ipath_update_ack_queue(qp, next);		ipath_update_ack_queue(qp, next);
	}		}
	e = &qp->s_ack_queue[qp->r_head_ack_queue];		e = &qp->s_ack_queue[qp->r_head_ack_queue];
	@@ -1787,7 +1821,7 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr,		ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr,
	rkey, IB_ACCESS_REMOTE_READ);		rkey, IB_ACCESS_REMOTE_READ);
	if (unlikely(!ok))		if (unlikely(!ok))
	goto nack_acc;		goto nack_acc_unlck;
	/*		/*
	* Update the next expected PSN. We add 1 later		* Update the next expected PSN. We add 1 later
	* below, so only add the remainder here.		* below, so only add the remainder here.
	@@ -1814,13 +1848,12 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	qp->r_psn++;		qp->r_psn++;
	qp->r_state = opcode;		qp->r_state = opcode;
	qp->r_nak_state = 0;		qp->r_nak_state = 0;
	barrier();
	qp->r_head_ack_queue = next;		qp->r_head_ack_queue = next;

	/* Call ipath_do_rc_send() in another thread. */		/* Schedule the send tasklet. */
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);

	goto done;		goto unlock;
	}		}

	case OP(COMPARE_SWAP):		case OP(COMPARE_SWAP):
	@@ -1839,9 +1872,13 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	next = qp->r_head_ack_queue + 1;		next = qp->r_head_ack_queue + 1;
	if (next > IPATH_MAX_RDMA_ATOMIC)		if (next > IPATH_MAX_RDMA_ATOMIC)
	next = 0;		next = 0;
			spin_lock_irqsave(&qp->s_lock, flags);
			/* Double check we can process this while holding the s_lock. */
			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))
			goto unlock;
	if (unlikely(next == qp->s_tail_ack_queue)) {		if (unlikely(next == qp->s_tail_ack_queue)) {
	if (!qp->s_ack_queue[next].sent)		if (!qp->s_ack_queue[next].sent)
	goto nack_inv;		goto nack_inv_unlck;
	ipath_update_ack_queue(qp, next);		ipath_update_ack_queue(qp, next);
	}		}
	if (!header_in_data)		if (!header_in_data)
	@@ -1851,13 +1888,13 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) \|		vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) \|
	be32_to_cpu(ateth->vaddr[1]);		be32_to_cpu(ateth->vaddr[1]);
	if (unlikely(vaddr & (sizeof(u64) - 1)))		if (unlikely(vaddr & (sizeof(u64) - 1)))
	goto nack_inv;		goto nack_inv_unlck;
	rkey = be32_to_cpu(ateth->rkey);		rkey = be32_to_cpu(ateth->rkey);
	/* Check rkey & NAK */		/* Check rkey & NAK */
	if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge,		if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge,
	sizeof(u64), vaddr, rkey,		sizeof(u64), vaddr, rkey,
	IB_ACCESS_REMOTE_ATOMIC)))		IB_ACCESS_REMOTE_ATOMIC)))
	goto nack_acc;		goto nack_acc_unlck;
	/* Perform atomic OP and save result. */		/* Perform atomic OP and save result. */
	maddr = (atomic64_t *) qp->r_sge.sge.vaddr;		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
	sdata = be64_to_cpu(ateth->swap_data);		sdata = be64_to_cpu(ateth->swap_data);
	@@ -1874,13 +1911,12 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	qp->r_psn++;		qp->r_psn++;
	qp->r_state = opcode;		qp->r_state = opcode;
	qp->r_nak_state = 0;		qp->r_nak_state = 0;
	barrier();
	qp->r_head_ack_queue = next;		qp->r_head_ack_queue = next;

	/* Call ipath_do_rc_send() in another thread. */		/* Schedule the send tasklet. */
	tasklet_hi_schedule(&qp->s_task);		ipath_schedule_send(qp);

	goto done;		goto unlock;
	}		}

	default:		default:
	@@ -1901,19 +1937,26 @@ void ipath_rc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	qp->r_ack_psn = qp->r_psn;		qp->r_ack_psn = qp->r_psn;
	goto send_ack;		goto send_ack;

			nack_inv_unlck:
			spin_unlock_irqrestore(&qp->s_lock, flags);
	nack_inv:		nack_inv:
	ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR);		ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
	qp->r_nak_state = IB_NAK_INVALID_REQUEST;		qp->r_nak_state = IB_NAK_INVALID_REQUEST;
	qp->r_ack_psn = qp->r_psn;		qp->r_ack_psn = qp->r_psn;
	goto send_ack;		goto send_ack;

			nack_acc_unlck:
			spin_unlock_irqrestore(&qp->s_lock, flags);
	nack_acc:		nack_acc:
	ipath_rc_error(qp, IB_WC_LOC_PROT_ERR);		ipath_rc_error(qp, IB_WC_LOC_PROT_ERR);
	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;		qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
	qp->r_ack_psn = qp->r_psn;		qp->r_ack_psn = qp->r_psn;
	send_ack:		send_ack:
	send_rc_ack(qp);		send_rc_ack(qp);
			goto done;

			unlock:
			spin_unlock_irqrestore(&qp->s_lock, flags);
	done:		done:
	return;		return;
	}		}

drivers/infiniband/hw/ipath/ipath_ruc.c

+115 −53

File changed.

Preview size limit exceeded, changes collapsed.

drivers/infiniband/hw/ipath/ipath_uc.c

+38 −19

Original line number	Original line	Diff line number	Diff line
	/*		/*
	* Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.		* Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
	* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.		* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
	*		*
	* This software is available to you under a choice of one of two		* This software is available to you under a choice of one of two
	@@ -47,14 +47,30 @@ int ipath_make_uc_req(struct ipath_qp *qp)
	{		{
	struct ipath_other_headers *ohdr;		struct ipath_other_headers *ohdr;
	struct ipath_swqe *wqe;		struct ipath_swqe *wqe;
			unsigned long flags;
	u32 hwords;		u32 hwords;
	u32 bth0;		u32 bth0;
	u32 len;		u32 len;
	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);		u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
	int ret = 0;		int ret = 0;

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK))		spin_lock_irqsave(&qp->s_lock, flags);

			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {
			if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
			goto bail;
			/* We are in the error state, flush the work request. */
			if (qp->s_last == qp->s_head)
			goto bail;
			/* If DMAs are in progress, we can't flush immediately. */
			if (atomic_read(&qp->s_dma_busy)) {
			qp->s_flags \|= IPATH_S_WAIT_DMA;
			goto bail;
			}
			wqe = get_swqe_ptr(qp, qp->s_last);
			ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
	goto done;		goto done;
			}

	ohdr = &qp->s_hdr.u.oth;		ohdr = &qp->s_hdr.u.oth;
	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)		if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
	@@ -69,9 +85,12 @@ int ipath_make_uc_req(struct ipath_qp *qp)
	qp->s_wqe = NULL;		qp->s_wqe = NULL;
	switch (qp->s_state) {		switch (qp->s_state) {
	default:		default:
			if (!(ib_ipath_state_ops[qp->state] &
			IPATH_PROCESS_NEXT_SEND_OK))
			goto bail;
	/* Check if send work queue is empty. */		/* Check if send work queue is empty. */
	if (qp->s_cur == qp->s_head)		if (qp->s_cur == qp->s_head)
	goto done;		goto bail;
	/*		/*
	* Start a new request.		* Start a new request.
	*/		*/
	@@ -134,7 +153,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
	break;		break;

	default:		default:
	goto done;		goto bail;
	}		}
	break;		break;

	@@ -194,9 +213,14 @@ int ipath_make_uc_req(struct ipath_qp *qp)
	ipath_make_ruc_header(to_idev(qp->ibqp.device),		ipath_make_ruc_header(to_idev(qp->ibqp.device),
	qp, ohdr, bth0 \| (qp->s_state << 24),		qp, ohdr, bth0 \| (qp->s_state << 24),
	qp->s_next_psn++ & IPATH_PSN_MASK);		qp->s_next_psn++ & IPATH_PSN_MASK);
			done:
	ret = 1;		ret = 1;
			goto unlock;

	done:		bail:
			qp->s_flags &= ~IPATH_S_BUSY;
			unlock:
			spin_unlock_irqrestore(&qp->s_lock, flags);
	return ret;		return ret;
	}		}

	@@ -258,8 +282,7 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	*/		*/
	opcode = be32_to_cpu(ohdr->bth[0]) >> 24;		opcode = be32_to_cpu(ohdr->bth[0]) >> 24;

	wc.imm_data = 0;		memset(&wc, 0, sizeof wc);
	wc.wc_flags = 0;

	/* Compare the PSN verses the expected PSN. */		/* Compare the PSN verses the expected PSN. */
	if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) {		if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) {
	@@ -322,8 +345,8 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	case OP(SEND_ONLY):		case OP(SEND_ONLY):
	case OP(SEND_ONLY_WITH_IMMEDIATE):		case OP(SEND_ONLY_WITH_IMMEDIATE):
	send_first:		send_first:
	if (qp->r_reuse_sge) {		if (qp->r_flags & IPATH_R_REUSE_SGE) {
	qp->r_reuse_sge = 0;		qp->r_flags &= ~IPATH_R_REUSE_SGE;
	qp->r_sge = qp->s_rdma_read_sge;		qp->r_sge = qp->s_rdma_read_sge;
	} else if (!ipath_get_rwqe(qp, 0)) {		} else if (!ipath_get_rwqe(qp, 0)) {
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	@@ -340,13 +363,13 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	case OP(SEND_MIDDLE):		case OP(SEND_MIDDLE):
	/* Check for invalid length PMTU or posted rwqe len. */		/* Check for invalid length PMTU or posted rwqe len. */
	if (unlikely(tlen != (hdrsize + pmtu + 4))) {		if (unlikely(tlen != (hdrsize + pmtu + 4))) {
	qp->r_reuse_sge = 1;		qp->r_flags \|= IPATH_R_REUSE_SGE;
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;
	}		}
	qp->r_rcv_len += pmtu;		qp->r_rcv_len += pmtu;
	if (unlikely(qp->r_rcv_len > qp->r_len)) {		if (unlikely(qp->r_rcv_len > qp->r_len)) {
	qp->r_reuse_sge = 1;		qp->r_flags \|= IPATH_R_REUSE_SGE;
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;
	}		}
	@@ -372,7 +395,7 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	/* Check for invalid length. */		/* Check for invalid length. */
	/* XXX LAST len should be >= 1 */		/* XXX LAST len should be >= 1 */
	if (unlikely(tlen < (hdrsize + pad + 4))) {		if (unlikely(tlen < (hdrsize + pad + 4))) {
	qp->r_reuse_sge = 1;		qp->r_flags \|= IPATH_R_REUSE_SGE;
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;
	}		}
	@@ -380,7 +403,7 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	tlen -= (hdrsize + pad + 4);		tlen -= (hdrsize + pad + 4);
	wc.byte_len = tlen + qp->r_rcv_len;		wc.byte_len = tlen + qp->r_rcv_len;
	if (unlikely(wc.byte_len > qp->r_len)) {		if (unlikely(wc.byte_len > qp->r_len)) {
	qp->r_reuse_sge = 1;		qp->r_flags \|= IPATH_R_REUSE_SGE;
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;
	}		}
	@@ -390,14 +413,10 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	wc.wr_id = qp->r_wr_id;		wc.wr_id = qp->r_wr_id;
	wc.status = IB_WC_SUCCESS;		wc.status = IB_WC_SUCCESS;
	wc.opcode = IB_WC_RECV;		wc.opcode = IB_WC_RECV;
	wc.vendor_err = 0;
	wc.qp = &qp->ibqp;		wc.qp = &qp->ibqp;
	wc.src_qp = qp->remote_qpn;		wc.src_qp = qp->remote_qpn;
	wc.pkey_index = 0;
	wc.slid = qp->remote_ah_attr.dlid;		wc.slid = qp->remote_ah_attr.dlid;
	wc.sl = qp->remote_ah_attr.sl;		wc.sl = qp->remote_ah_attr.sl;
	wc.dlid_path_bits = 0;
	wc.port_num = 0;
	/* Signal completion event if the solicited bit is set. */		/* Signal completion event if the solicited bit is set. */
	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
	(ohdr->bth[0] &		(ohdr->bth[0] &
	@@ -488,8 +507,8 @@ void ipath_uc_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;
	}		}
	if (qp->r_reuse_sge)		if (qp->r_flags & IPATH_R_REUSE_SGE)
	qp->r_reuse_sge = 0;		qp->r_flags &= ~IPATH_R_REUSE_SGE;
	else if (!ipath_get_rwqe(qp, 1)) {		else if (!ipath_get_rwqe(qp, 1)) {
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto done;		goto done;

drivers/infiniband/hw/ipath/ipath_ud.c

+48 −18

Original line number	Original line	Diff line number	Diff line
	@@ -65,9 +65,9 @@ static void ipath_ud_loopback(struct ipath_qp sqp, struct ipath_swqe swqe)
	u32 length;		u32 length;

	qp = ipath_lookup_qpn(&dev->qp_table, swqe->wr.wr.ud.remote_qpn);		qp = ipath_lookup_qpn(&dev->qp_table, swqe->wr.wr.ud.remote_qpn);
	if (!qp) {		if (!qp \|\| !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto send_comp;		goto done;
	}		}

	rsge.sg_list = NULL;		rsge.sg_list = NULL;
	@@ -91,14 +91,12 @@ static void ipath_ud_loopback(struct ipath_qp sqp, struct ipath_swqe swqe)
	* present on the wire.		* present on the wire.
	*/		*/
	length = swqe->length;		length = swqe->length;
			memset(&wc, 0, sizeof wc);
	wc.byte_len = length + sizeof(struct ib_grh);		wc.byte_len = length + sizeof(struct ib_grh);

	if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {		if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
	wc.wc_flags = IB_WC_WITH_IMM;		wc.wc_flags = IB_WC_WITH_IMM;
	wc.imm_data = swqe->wr.ex.imm_data;		wc.imm_data = swqe->wr.ex.imm_data;
	} else {
	wc.wc_flags = 0;
	wc.imm_data = 0;
	}		}

	/*		/*
	@@ -229,7 +227,6 @@ static void ipath_ud_loopback(struct ipath_qp sqp, struct ipath_swqe swqe)
	}		}
	wc.status = IB_WC_SUCCESS;		wc.status = IB_WC_SUCCESS;
	wc.opcode = IB_WC_RECV;		wc.opcode = IB_WC_RECV;
	wc.vendor_err = 0;
	wc.qp = &qp->ibqp;		wc.qp = &qp->ibqp;
	wc.src_qp = sqp->ibqp.qp_num;		wc.src_qp = sqp->ibqp.qp_num;
	/* XXX do we know which pkey matched? Only needed for GSI. */		/* XXX do we know which pkey matched? Only needed for GSI. */
	@@ -248,8 +245,7 @@ static void ipath_ud_loopback(struct ipath_qp sqp, struct ipath_swqe swqe)
	kfree(rsge.sg_list);		kfree(rsge.sg_list);
	if (atomic_dec_and_test(&qp->refcount))		if (atomic_dec_and_test(&qp->refcount))
	wake_up(&qp->wait);		wake_up(&qp->wait);
	send_comp:		done:;
	ipath_send_complete(sqp, swqe, IB_WC_SUCCESS);
	}		}

	/**		/**
	@@ -264,6 +260,7 @@ int ipath_make_ud_req(struct ipath_qp *qp)
	struct ipath_other_headers *ohdr;		struct ipath_other_headers *ohdr;
	struct ib_ah_attr *ah_attr;		struct ib_ah_attr *ah_attr;
	struct ipath_swqe *wqe;		struct ipath_swqe *wqe;
			unsigned long flags;
	u32 nwords;		u32 nwords;
	u32 extra_bytes;		u32 extra_bytes;
	u32 bth0;		u32 bth0;
	@@ -271,13 +268,30 @@ int ipath_make_ud_req(struct ipath_qp *qp)
	u16 lid;		u16 lid;
	int ret = 0;		int ret = 0;

	if (unlikely(!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)))		spin_lock_irqsave(&qp->s_lock, flags);

			if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {
			if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
			goto bail;
			/* We are in the error state, flush the work request. */
			if (qp->s_last == qp->s_head)
			goto bail;
			/* If DMAs are in progress, we can't flush immediately. */
			if (atomic_read(&qp->s_dma_busy)) {
			qp->s_flags \|= IPATH_S_WAIT_DMA;
	goto bail;		goto bail;
			}
			wqe = get_swqe_ptr(qp, qp->s_last);
			ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
			goto done;
			}

	if (qp->s_cur == qp->s_head)		if (qp->s_cur == qp->s_head)
	goto bail;		goto bail;

	wqe = get_swqe_ptr(qp, qp->s_cur);		wqe = get_swqe_ptr(qp, qp->s_cur);
			if (++qp->s_cur >= qp->s_size)
			qp->s_cur = 0;

	/* Construct the header. */		/* Construct the header. */
	ah_attr = &to_iah(wqe->wr.wr.ud.ah)->attr;		ah_attr = &to_iah(wqe->wr.wr.ud.ah)->attr;
	@@ -288,10 +302,23 @@ int ipath_make_ud_req(struct ipath_qp *qp)
	dev->n_unicast_xmit++;		dev->n_unicast_xmit++;
	} else {		} else {
	dev->n_unicast_xmit++;		dev->n_unicast_xmit++;
	lid = ah_attr->dlid &		lid = ah_attr->dlid & ~((1 << dev->dd->ipath_lmc) - 1);
	~((1 << dev->dd->ipath_lmc) - 1);
	if (unlikely(lid == dev->dd->ipath_lid)) {		if (unlikely(lid == dev->dd->ipath_lid)) {
			/*
			* If DMAs are in progress, we can't generate
			* a completion for the loopback packet since
			* it would be out of order.
			* XXX Instead of waiting, we could queue a
			* zero length descriptor so we get a callback.
			*/
			if (atomic_read(&qp->s_dma_busy)) {
			qp->s_flags \|= IPATH_S_WAIT_DMA;
			goto bail;
			}
			spin_unlock_irqrestore(&qp->s_lock, flags);
	ipath_ud_loopback(qp, wqe);		ipath_ud_loopback(qp, wqe);
			spin_lock_irqsave(&qp->s_lock, flags);
			ipath_send_complete(qp, wqe, IB_WC_SUCCESS);
	goto done;		goto done;
	}		}
	}		}
	@@ -368,11 +395,13 @@ int ipath_make_ud_req(struct ipath_qp *qp)
	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);		ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);

	done:		done:
	if (++qp->s_cur >= qp->s_size)
	qp->s_cur = 0;
	ret = 1;		ret = 1;
			goto unlock;

	bail:		bail:
			qp->s_flags &= ~IPATH_S_BUSY;
			unlock:
			spin_unlock_irqrestore(&qp->s_lock, flags);
	return ret;		return ret;
	}		}

	@@ -506,8 +535,8 @@ void ipath_ud_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	/*		/*
	* Get the next work request entry to find where to put the data.		* Get the next work request entry to find where to put the data.
	*/		*/
	if (qp->r_reuse_sge)		if (qp->r_flags & IPATH_R_REUSE_SGE)
	qp->r_reuse_sge = 0;		qp->r_flags &= ~IPATH_R_REUSE_SGE;
	else if (!ipath_get_rwqe(qp, 0)) {		else if (!ipath_get_rwqe(qp, 0)) {
	/*		/*
	* Count VL15 packets dropped due to no receive buffer.		* Count VL15 packets dropped due to no receive buffer.
	@@ -523,7 +552,7 @@ void ipath_ud_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	}		}
	/* Silently drop packets which are too big. */		/* Silently drop packets which are too big. */
	if (wc.byte_len > qp->r_len) {		if (wc.byte_len > qp->r_len) {
	qp->r_reuse_sge = 1;		qp->r_flags \|= IPATH_R_REUSE_SGE;
	dev->n_pkt_drops++;		dev->n_pkt_drops++;
	goto bail;		goto bail;
	}		}
	@@ -535,7 +564,8 @@ void ipath_ud_rcv(struct ipath_ibdev dev, struct ipath_ib_header hdr,
	ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));		ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));
	ipath_copy_sge(&qp->r_sge, data,		ipath_copy_sge(&qp->r_sge, data,
	wc.byte_len - sizeof(struct ib_grh));		wc.byte_len - sizeof(struct ib_grh));
	qp->r_wrid_valid = 0;		if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
			goto bail;
	wc.wr_id = qp->r_wr_id;		wc.wr_id = qp->r_wr_id;
	wc.status = IB_WC_SUCCESS;		wc.status = IB_WC_SUCCESS;
	wc.opcode = IB_WC_RECV;		wc.opcode = IB_WC_RECV;