Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 019f118b authored by Brian Welty's avatar Brian Welty Committed by Jason Gunthorpe
Browse files

IB/{hfi1, qib, rdmavt}: Move copy SGE logic into rdmavt 



This patch moves hfi1_copy_sge() into rdmavt for sharing with qib.
This patch also moves all the wss_*() functions into rdmavt as
several wss_*() functions are called from hfi1_copy_sge()

When SGE copy mode is adaptive, cacheless copy may be done in some cases
for performance reasons. In those cases, X86 cacheless copy function
is called since the drivers that use rdmavt and may set SGE copy mode
to adaptive are X86 only. For this reason, this patch adds
"depends on X86_64" to rdmavt/Kconfig.

Reviewed-by: default avatarAshutosh Dixit <ashutosh.dixit@intel.com>
Reviewed-by: default avatarMichael J. Ruhl <michael.j.ruhl@intel.com>
Reviewed-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Reviewed-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarBrian Welty <brian.welty@intel.com>
Signed-off-by: default avatarHarish Chegondi <harish.chegondi@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent b56511c1

drivers/infiniband/hw/hfi1/init.c

+0 −6
Original line number Diff line number Diff line
@@ -1504,9 +1504,6 @@ static int __init hfi1_mod_init(void) 
	idr_init(&hfi1_unit_table);



	hfi1_dbg_init();

	ret = hfi1_wss_init();

	if (ret < 0)

		goto bail_wss;

	ret = pci_register_driver(&hfi1_pci_driver);

	if (ret < 0) {

		pr_err("Unable to register driver: error %d\n", -ret);
@@ -1515,8 +1512,6 @@ static int __init hfi1_mod_init(void) 
	goto bail; /* all OK */



bail_dev:

	hfi1_wss_exit();

bail_wss:

	hfi1_dbg_exit();

	idr_destroy(&hfi1_unit_table);

	dev_cleanup();
@@ -1533,7 +1528,6 @@ static void __exit hfi1_mod_cleanup(void) 
{

	pci_unregister_driver(&hfi1_pci_driver);

	node_affinity_destroy_all();

	hfi1_wss_exit();

	hfi1_dbg_exit();



	idr_destroy(&hfi1_unit_table);

drivers/infiniband/hw/hfi1/rc.c

+6 −4
Original line number Diff line number Diff line
@@ -1644,7 +1644,8 @@ static void rc_rcv_resp(struct hfi1_packet *packet) 
		qp->s_rdma_read_len -= pmtu;

		update_last_psn(qp, psn);

		spin_unlock_irqrestore(&qp->s_lock, flags);

		hfi1_copy_sge(&qp->s_rdma_read_sge, data, pmtu, false, false);

		rvt_copy_sge(qp, &qp->s_rdma_read_sge,

			     data, pmtu, false, false);

		goto bail;



	case OP(RDMA_READ_RESPONSE_ONLY):
@@ -1684,7 +1685,8 @@ static void rc_rcv_resp(struct hfi1_packet *packet) 
		if (unlikely(tlen != qp->s_rdma_read_len))

			goto ack_len_err;

		aeth = be32_to_cpu(ohdr->u.aeth);

		hfi1_copy_sge(&qp->s_rdma_read_sge, data, tlen, false, false);

		rvt_copy_sge(qp, &qp->s_rdma_read_sge,

			     data, tlen, false, false);

		WARN_ON(qp->s_rdma_read_sge.num_sge);

		(void)do_rc_ack(qp, aeth, psn,

				 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
@@ -2144,7 +2146,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto nack_inv;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);

		break;



	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -2200,7 +2202,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) 
		wc.byte_len = tlen + qp->r_rcv_len;

		if (unlikely(wc.byte_len > qp->r_len))

			goto nack_inv;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, copy_last);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last);

		rvt_put_ss(&qp->r_sge);

		qp->r_msn++;

		if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))

drivers/infiniband/hw/hfi1/ruc.c

+2 −1
Original line number Diff line number Diff line
@@ -361,7 +361,8 @@ static void ruc_loopback(struct rvt_qp *sqp) 
		if (len > sge->sge_length)

			len = sge->sge_length;

		WARN_ON_ONCE(len == 0);

		hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, release, copy_last);

		rvt_copy_sge(qp, &qp->r_sge, sge->vaddr,

			     len, release, copy_last);

		sge->vaddr += len;

		sge->length -= len;

		sge->sge_length -= len;

drivers/infiniband/hw/hfi1/uc.c

+5 −5
Original line number Diff line number Diff line
@@ -426,7 +426,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto rewind;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, false, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false);

		break;



	case OP(SEND_LAST_WITH_IMMEDIATE):
@@ -449,7 +449,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) 
		if (unlikely(wc.byte_len > qp->r_len))

			goto rewind;

		wc.opcode = IB_WC_RECV;

		hfi1_copy_sge(&qp->r_sge, data, tlen, false, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false);

		rvt_put_ss(&qp->s_rdma_read_sge);

last_imm:

		wc.wr_id = qp->r_wr_id;
@@ -523,7 +523,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto drop;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);

		break;



	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -550,7 +550,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) 
		}

		wc.byte_len = qp->r_len;

		wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);

		rvt_put_ss(&qp->r_sge);

		goto last_imm;
@@ -564,7 +564,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) 
		tlen -= (hdrsize + extra_bytes);

		if (unlikely(tlen + qp->r_rcv_len != qp->r_len))

			goto drop;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);

		rvt_put_ss(&qp->r_sge);

		break;

drivers/infiniband/hw/hfi1/ud.c

+9 −9
Original line number Diff line number Diff line
@@ -210,7 +210,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) 
		}



		hfi1_make_grh(ibp, &grh, &grd, 0, 0);

		hfi1_copy_sge(&qp->r_sge, &grh,

		rvt_copy_sge(qp, &qp->r_sge, &grh,

			     sizeof(grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else {
@@ -228,7 +228,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) 
		if (len > sge->sge_length)

			len = sge->sge_length;

		WARN_ON_ONCE(len == 0);

		hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, true, false);

		rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false);

		sge->vaddr += len;

		sge->length -= len;

		sge->sge_length -= len;
@@ -1019,7 +1019,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) 
		goto drop;

	}

	if (packet->grh) {

		hfi1_copy_sge(&qp->r_sge, packet->grh,

		rvt_copy_sge(qp, &qp->r_sge, packet->grh,

			     sizeof(struct ib_grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else if (packet->etype == RHF_RCV_TYPE_BYPASS) {
@@ -1030,13 +1030,13 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) 
		 * out when creating 16B, add back the GRH here.

		 */

		hfi1_make_ext_grh(packet, &grh, slid, dlid);

		hfi1_copy_sge(&qp->r_sge, &grh,

		rvt_copy_sge(qp, &qp->r_sge, &grh,

			     sizeof(struct ib_grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else {

		rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);

	}

	hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),

	rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),

		     true, false);

	rvt_put_ss(&qp->r_sge);

	if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))