Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4cae780e authored by Al Viro's avatar Al Viro Committed by Greg Kroah-Hartman
Browse files

lustre: introduce lnet_copy_{k, }iov2iter(), kill lnet_copy_{k, }iov2{k, }iov() 



Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
Signed-off-by: default avatarJames Simmons <jsimmons@infradead.org>
Signed-off-by: default avatarOleg Drokin <green@linuxhacker.ru>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent c1b7b8eb

drivers/staging/lustre/include/linux/lnet/lib-lnet.h

+2 −55
Original line number Diff line number Diff line
@@ -613,65 +613,12 @@ int lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst, 
		      int src_niov, const lnet_kiov_t *src,

		      unsigned int offset, unsigned int len);



void lnet_copy_iov2iov(unsigned int ndiov, const struct kvec *diov,

		       unsigned int doffset,

void lnet_copy_iov2iter(struct iov_iter *to,

			unsigned int nsiov, const struct kvec *siov,

			unsigned int soffset, unsigned int nob);

void lnet_copy_kiov2iov(unsigned int niov, const struct kvec *iov,

			unsigned int iovoffset,

void lnet_copy_kiov2iter(struct iov_iter *to,

			 unsigned int nkiov, const lnet_kiov_t *kiov,

			 unsigned int kiovoffset, unsigned int nob);

void lnet_copy_iov2kiov(unsigned int nkiov, const lnet_kiov_t *kiov,

			unsigned int kiovoffset,

			 unsigned int niov, const struct kvec *iov,

			 unsigned int iovoffset, unsigned int nob);

void lnet_copy_kiov2kiov(unsigned int ndkiov, const lnet_kiov_t *dkiov,

			 unsigned int doffset,

			  unsigned int nskiov, const lnet_kiov_t *skiov,

			  unsigned int soffset, unsigned int nob);



static inline void

lnet_copy_iov2flat(int dlen, void *dest, unsigned int doffset,

		   unsigned int nsiov, const struct kvec *siov, unsigned int soffset,

		   unsigned int nob)

{

	struct kvec diov = {/*.iov_base = */ dest, /*.iov_len = */ dlen};



	lnet_copy_iov2iov(1, &diov, doffset,

			  nsiov, siov, soffset, nob);

}



static inline void

lnet_copy_kiov2flat(int dlen, void *dest, unsigned int doffset,

		    unsigned int nsiov, const lnet_kiov_t *skiov,

		    unsigned int soffset, unsigned int nob)

{

	struct kvec diov = {/* .iov_base = */ dest, /* .iov_len = */ dlen};



	lnet_copy_kiov2iov(1, &diov, doffset,

			   nsiov, skiov, soffset, nob);

}



static inline void

lnet_copy_flat2iov(unsigned int ndiov, const struct kvec *diov, unsigned int doffset,

		   int slen, void *src, unsigned int soffset, unsigned int nob)

{

	struct kvec siov = {/*.iov_base = */ src, /*.iov_len = */slen};



	lnet_copy_iov2iov(ndiov, diov, doffset,

			  1, &siov, soffset, nob);

}



static inline void

lnet_copy_flat2kiov(unsigned int ndiov, const lnet_kiov_t *dkiov,

		    unsigned int doffset, int slen, void *src,

		    unsigned int soffset, unsigned int nob)

{

	struct kvec siov = {/* .iov_base = */ src, /* .iov_len = */ slen};



	lnet_copy_iov2kiov(ndiov, dkiov, doffset,

			   1, &siov, soffset, nob);

}



void lnet_me_unlink(lnet_me_t *me);

drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c

+16 −21
Original line number Diff line number Diff line
@@ -1491,6 +1491,7 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg) 
	lnet_kiov_t *payload_kiov = lntmsg->msg_kiov;

	unsigned int payload_offset = lntmsg->msg_offset;

	unsigned int payload_nob = lntmsg->msg_len;

	struct iov_iter from;

	struct kib_msg *ibmsg;

	struct kib_rdma_desc  *rd;

	struct kib_tx *tx;
@@ -1510,6 +1511,17 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg) 
	/* payload is either all vaddrs or all pages */

	LASSERT(!(payload_kiov && payload_iov));



	if (payload_kiov)

		iov_iter_bvec(&from, ITER_BVEC | WRITE,

			      payload_kiov, payload_niov,

			      payload_nob + payload_offset);

	else

		iov_iter_kvec(&from, ITER_KVEC | WRITE,

			      payload_iov, payload_niov,

			      payload_nob + payload_offset);



	iov_iter_advance(&from, payload_offset);



	switch (type) {

	default:

		LBUG();
@@ -1629,17 +1641,8 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg) 
	ibmsg = tx->tx_msg;

	ibmsg->ibm_u.immediate.ibim_hdr = *hdr;



	if (payload_kiov)

		lnet_copy_kiov2flat(IBLND_MSG_SIZE, ibmsg,

				    offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),

				    payload_niov, payload_kiov,

				    payload_offset, payload_nob);

	else

		lnet_copy_iov2flat(IBLND_MSG_SIZE, ibmsg,

				   offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),

				   payload_niov, payload_iov,

				   payload_offset, payload_nob);



	copy_from_iter(&ibmsg->ibm_u.immediate.ibim_payload, IBLND_MSG_SIZE,

		       &from);

	nob = offsetof(struct kib_immediate_msg, ibim_payload[payload_nob]);

	kiblnd_init_tx_msg(ni, tx, IBLND_MSG_IMMEDIATE, nob);
@@ -1739,16 +1742,8 @@ kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed, 
			break;

		}



		if (to->type & ITER_BVEC)

			lnet_copy_flat2kiov(to->nr_segs, to->bvec, to->iov_offset,

					    IBLND_MSG_SIZE, rxmsg,

					    offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),

					    iov_iter_count(to));

		else

			lnet_copy_flat2iov(to->nr_segs, to->kvec, to->iov_offset,

					   IBLND_MSG_SIZE, rxmsg,

					   offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),

					   iov_iter_count(to));

		copy_to_iter(&rxmsg->ibm_u.immediate.ibim_payload,

			     IBLND_MSG_SIZE, to);

		lnet_finalize(ni, lntmsg, 0);

		break;

drivers/staging/lustre/lnet/lnet/lib-move.c

+56 −258
Original line number Diff line number Diff line
@@ -166,25 +166,17 @@ lnet_iov_nob(unsigned int niov, struct kvec *iov) 
EXPORT_SYMBOL(lnet_iov_nob);



void

lnet_copy_iov2iov(unsigned int ndiov, const struct kvec *diov, unsigned int doffset,

		  unsigned int nsiov, const struct kvec *siov, unsigned int soffset,

		  unsigned int nob)

lnet_copy_iov2iter(struct iov_iter *to,

		   unsigned int nsiov, const struct kvec *siov,

		   unsigned int soffset, unsigned int nob)

{

	/* NB diov, siov are READ-ONLY */

	unsigned int this_nob;

	const char *s;

	size_t left;



	if (!nob)

		return;



	/* skip complete frags before 'doffset' */

	LASSERT(ndiov > 0);

	while (doffset >= diov->iov_len) {

		doffset -= diov->iov_len;

		diov++;

		ndiov--;

		LASSERT(ndiov > 0);

	}



	/* skip complete frags before 'soffset' */

	LASSERT(nsiov > 0);

	while (soffset >= siov->iov_len) {
@@ -194,35 +186,64 @@ lnet_copy_iov2iov(unsigned int ndiov, const struct kvec *diov, unsigned int doff 
		LASSERT(nsiov > 0);

	}



	s = (char *)siov->iov_base + soffset;

	left = siov->iov_len - soffset;

	do {

		LASSERT(ndiov > 0);

		size_t n, copy = left;

		LASSERT(nsiov > 0);

		this_nob = min(diov->iov_len - doffset,

			       siov->iov_len - soffset);

		this_nob = min(this_nob, nob);



		memcpy((char *)diov->iov_base + doffset,

		       (char *)siov->iov_base + soffset, this_nob);

		nob -= this_nob;

		if (copy > nob)

			copy = nob;

		n = copy_to_iter(s, copy, to);

		if (n != copy)

			return;

		nob -= n;



		if (diov->iov_len > doffset + this_nob) {

			doffset += this_nob;

		} else {

			diov++;

			ndiov--;

			doffset = 0;

		siov++;

		s = (char *)siov->iov_base;

		left = siov->iov_len;

		nsiov--;

	} while (nob > 0);

}

EXPORT_SYMBOL(lnet_copy_iov2iter);



		if (siov->iov_len > soffset + this_nob) {

			soffset += this_nob;

		} else {

void

lnet_copy_kiov2iter(struct iov_iter *to,

		    unsigned int nsiov, const lnet_kiov_t *siov,

		    unsigned int soffset, unsigned int nob)

{

	if (!nob)

		return;



	LASSERT(!in_interrupt());



	LASSERT(nsiov > 0);

	while (soffset >= siov->bv_len) {

		soffset -= siov->bv_len;

		siov++;

		nsiov--;

			soffset = 0;

		LASSERT(nsiov > 0);

	}



	do {

		size_t copy = siov->bv_len - soffset, n;



		LASSERT(nsiov > 0);



		if (copy > nob)

			copy = nob;

		n = copy_page_to_iter(siov->bv_page,

				      siov->bv_offset + soffset,

				      copy, to);

		if (n != copy)

			return;

		nob -= n;

		siov++;

		nsiov--;

		soffset = 0;

	} while (nob > 0);

}

EXPORT_SYMBOL(lnet_copy_iov2iov);

EXPORT_SYMBOL(lnet_copy_kiov2iter);



int

lnet_extract_iov(int dst_niov, struct kvec *dst,
@@ -286,229 +307,6 @@ lnet_kiov_nob(unsigned int niov, lnet_kiov_t *kiov) 
}

EXPORT_SYMBOL(lnet_kiov_nob);



void

lnet_copy_kiov2kiov(unsigned int ndiov, const lnet_kiov_t *diov, unsigned int doffset,

		    unsigned int nsiov, const lnet_kiov_t *siov, unsigned int soffset,

		    unsigned int nob)

{

	/* NB diov, siov are READ-ONLY */

	unsigned int this_nob;

	char *daddr = NULL;

	char *saddr = NULL;



	if (!nob)

		return;



	LASSERT(!in_interrupt());



	LASSERT(ndiov > 0);

	while (doffset >= diov->bv_len) {

		doffset -= diov->bv_len;

		diov++;

		ndiov--;

		LASSERT(ndiov > 0);

	}



	LASSERT(nsiov > 0);

	while (soffset >= siov->bv_len) {

		soffset -= siov->bv_len;

		siov++;

		nsiov--;

		LASSERT(nsiov > 0);

	}



	do {

		LASSERT(ndiov > 0);

		LASSERT(nsiov > 0);

		this_nob = min(diov->bv_len - doffset,

			       siov->bv_len - soffset);

		this_nob = min(this_nob, nob);



		if (!daddr)

			daddr = ((char *)kmap(diov->bv_page)) +

				diov->bv_offset + doffset;

		if (!saddr)

			saddr = ((char *)kmap(siov->bv_page)) +

				siov->bv_offset + soffset;



		/*

		 * Vanishing risk of kmap deadlock when mapping 2 pages.

		 * However in practice at least one of the kiovs will be mapped

		 * kernel pages and the map/unmap will be NOOPs

		 */

		memcpy(daddr, saddr, this_nob);

		nob -= this_nob;



		if (diov->bv_len > doffset + this_nob) {

			daddr += this_nob;

			doffset += this_nob;

		} else {

			kunmap(diov->bv_page);

			daddr = NULL;

			diov++;

			ndiov--;

			doffset = 0;

		}



		if (siov->bv_len > soffset + this_nob) {

			saddr += this_nob;

			soffset += this_nob;

		} else {

			kunmap(siov->bv_page);

			saddr = NULL;

			siov++;

			nsiov--;

			soffset = 0;

		}

	} while (nob > 0);



	if (daddr)

		kunmap(diov->bv_page);

	if (saddr)

		kunmap(siov->bv_page);

}

EXPORT_SYMBOL(lnet_copy_kiov2kiov);



void

lnet_copy_kiov2iov(unsigned int niov, const struct kvec *iov, unsigned int iovoffset,

		   unsigned int nkiov, const lnet_kiov_t *kiov,

		   unsigned int kiovoffset, unsigned int nob)

{

	/* NB iov, kiov are READ-ONLY */

	unsigned int this_nob;

	char *addr = NULL;



	if (!nob)

		return;



	LASSERT(!in_interrupt());



	LASSERT(niov > 0);

	while (iovoffset >= iov->iov_len) {

		iovoffset -= iov->iov_len;

		iov++;

		niov--;

		LASSERT(niov > 0);

	}



	LASSERT(nkiov > 0);

	while (kiovoffset >= kiov->bv_len) {

		kiovoffset -= kiov->bv_len;

		kiov++;

		nkiov--;

		LASSERT(nkiov > 0);

	}



	do {

		LASSERT(niov > 0);

		LASSERT(nkiov > 0);

		this_nob = min(iov->iov_len - iovoffset,

			       (__kernel_size_t)kiov->bv_len - kiovoffset);

		this_nob = min(this_nob, nob);



		if (!addr)

			addr = ((char *)kmap(kiov->bv_page)) +

				kiov->bv_offset + kiovoffset;



		memcpy((char *)iov->iov_base + iovoffset, addr, this_nob);

		nob -= this_nob;



		if (iov->iov_len > iovoffset + this_nob) {

			iovoffset += this_nob;

		} else {

			iov++;

			niov--;

			iovoffset = 0;

		}



		if (kiov->bv_len > kiovoffset + this_nob) {

			addr += this_nob;

			kiovoffset += this_nob;

		} else {

			kunmap(kiov->bv_page);

			addr = NULL;

			kiov++;

			nkiov--;

			kiovoffset = 0;

		}



	} while (nob > 0);



	if (addr)

		kunmap(kiov->bv_page);

}

EXPORT_SYMBOL(lnet_copy_kiov2iov);



void

lnet_copy_iov2kiov(unsigned int nkiov, const lnet_kiov_t *kiov,

		   unsigned int kiovoffset, unsigned int niov,

		   const struct kvec *iov, unsigned int iovoffset,

		   unsigned int nob)

{

	/* NB kiov, iov are READ-ONLY */

	unsigned int this_nob;

	char *addr = NULL;



	if (!nob)

		return;



	LASSERT(!in_interrupt());



	LASSERT(nkiov > 0);

	while (kiovoffset >= kiov->bv_len) {

		kiovoffset -= kiov->bv_len;

		kiov++;

		nkiov--;

		LASSERT(nkiov > 0);

	}



	LASSERT(niov > 0);

	while (iovoffset >= iov->iov_len) {

		iovoffset -= iov->iov_len;

		iov++;

		niov--;

		LASSERT(niov > 0);

	}



	do {

		LASSERT(nkiov > 0);

		LASSERT(niov > 0);

		this_nob = min((__kernel_size_t)kiov->bv_len - kiovoffset,

			       iov->iov_len - iovoffset);

		this_nob = min(this_nob, nob);



		if (!addr)

			addr = ((char *)kmap(kiov->bv_page)) +

				kiov->bv_offset + kiovoffset;



		memcpy(addr, (char *)iov->iov_base + iovoffset, this_nob);

		nob -= this_nob;



		if (kiov->bv_len > kiovoffset + this_nob) {

			addr += this_nob;

			kiovoffset += this_nob;

		} else {

			kunmap(kiov->bv_page);

			addr = NULL;

			kiov++;

			nkiov--;

			kiovoffset = 0;

		}



		if (iov->iov_len > iovoffset + this_nob) {

			iovoffset += this_nob;

		} else {

			iov++;

			niov--;

			iovoffset = 0;

		}

	} while (nob > 0);



	if (addr)

		kunmap(kiov->bv_page);

}

EXPORT_SYMBOL(lnet_copy_iov2kiov);



int

lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,

		  int src_niov, const lnet_kiov_t *src,

drivers/staging/lustre/lnet/lnet/lo.c

+12 −23
Original line number Diff line number Diff line
@@ -47,29 +47,18 @@ lolnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, 
	lnet_msg_t *sendmsg = private;



	if (lntmsg) {		   /* not discarding */

		if (sendmsg->msg_iov) {

			if (to->type & ITER_KVEC)

				lnet_copy_iov2iov(to->nr_segs, to->kvec, to->iov_offset,

		if (sendmsg->msg_iov)

			lnet_copy_iov2iter(to,

					   sendmsg->msg_niov,

					   sendmsg->msg_iov,

						  sendmsg->msg_offset, iov_iter_count(to));

					   sendmsg->msg_offset,

					   iov_iter_count(to));

		else

				lnet_copy_iov2kiov(to->nr_segs, to->bvec, to->iov_offset,

						   sendmsg->msg_niov,

						   sendmsg->msg_iov,

						   sendmsg->msg_offset, iov_iter_count(to));

		} else {

			if (to->type & ITER_KVEC)

				lnet_copy_kiov2iov(to->nr_segs, to->kvec, to->iov_offset,

						   sendmsg->msg_niov,

						   sendmsg->msg_kiov,

						   sendmsg->msg_offset, iov_iter_count(to));

			else

				lnet_copy_kiov2kiov(to->nr_segs, to->bvec, to->iov_offset,

			lnet_copy_kiov2iter(to,

					    sendmsg->msg_niov,

					    sendmsg->msg_kiov,

						    sendmsg->msg_offset, iov_iter_count(to));

		}

					    sendmsg->msg_offset,

					    iov_iter_count(to));



		lnet_finalize(ni, lntmsg, 0);

	}