Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b2875d4c authored by Michael S. Tsirkin's avatar Michael S. Tsirkin Committed by Roland Dreier
Browse files

IB/mthca: Always fill MTTs from CPU 



Speed up memory registration by filling in MTTs directly when the CPU
can write directly to the whole table (all mem-free cards, and to
Tavor mode on 64-bit systems with the patch I posted earlier).  This
reduces the number of FW commands needed to register an MR by at least
a factor of 2 and speeds up memory registration significantly.

Signed-off-by: default avatarMichael S. Tsirkin <mst@mellanox.co.il>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent c20e20ab

drivers/infiniband/hw/mthca/mthca_dev.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -464,6 +464,8 @@ void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar); 
int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd);

int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd);

void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);

void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);





int mthca_write_mtt_size(struct mthca_dev *dev);



struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size);

struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size);

void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt);

void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt);

int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,

int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,

drivers/infiniband/hw/mthca/mthca_mr.c

+80 −2
Original line number Original line Diff line number Diff line
@@ -243,7 +243,7 @@ void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) 
	kfree(mtt);

	kfree(mtt);

}

}





int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,

static int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,

			     int start_index, u64 *buffer_list, int list_len)

			     int start_index, u64 *buffer_list, int list_len)

{

{

	struct mthca_mailbox *mailbox;

	struct mthca_mailbox *mailbox;
@@ -295,6 +295,84 @@ int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, 
	return err;

	return err;

}

}





int mthca_write_mtt_size(struct mthca_dev *dev)

{

	if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy)

		/*

		 * Be friendly to WRITE_MTT command

		 * and leave two empty slots for the

		 * index and reserved fields of the

		 * mailbox.

		 */

		return PAGE_SIZE / sizeof (u64) - 2;



	/* For Arbel, all MTTs must fit in the same page. */

	return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff;

}



void mthca_tavor_write_mtt_seg(struct mthca_dev *dev, struct mthca_mtt *mtt,

			      int start_index, u64 *buffer_list, int list_len)

{

	u64 __iomem *mtts;

	int i;



	mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * MTHCA_MTT_SEG_SIZE +

		start_index * sizeof (u64);

	for (i = 0; i < list_len; ++i)

		mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT),

				  mtts + i);

}



void mthca_arbel_write_mtt_seg(struct mthca_dev *dev, struct mthca_mtt *mtt,

			      int start_index, u64 *buffer_list, int list_len)

{

	__be64 *mtts;

	dma_addr_t dma_handle;

	int i;

	int s = start_index * sizeof (u64);



	/* For Arbel, all MTTs must fit in the same page. */

	BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE);

	/* Require full segments */

	BUG_ON(s % MTHCA_MTT_SEG_SIZE);



	mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg +

				s / MTHCA_MTT_SEG_SIZE, &dma_handle);



	BUG_ON(!mtts);



	for (i = 0; i < list_len; ++i)

		mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT);



	dma_sync_single(&dev->pdev->dev, dma_handle, list_len * sizeof (u64), DMA_TO_DEVICE);

}



int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,

		    int start_index, u64 *buffer_list, int list_len)

{

	int size = mthca_write_mtt_size(dev);

	int chunk;



	if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy)

		return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len);



	while (list_len > 0) {

		chunk = min(size, list_len);

		if (mthca_is_memfree(dev))

			mthca_arbel_write_mtt_seg(dev, mtt, start_index,

						  buffer_list, chunk);

		else

			mthca_tavor_write_mtt_seg(dev, mtt, start_index,

						  buffer_list, chunk);



		list_len    -= chunk;

		start_index += chunk;

		buffer_list += chunk;

	}



	return 0;

}



static inline u32 tavor_hw_index_to_key(u32 ind)

static inline u32 tavor_hw_index_to_key(u32 ind)

{

{

	return ind;

	return ind;

drivers/infiniband/hw/mthca/mthca_provider.c

+7 −7
Original line number Original line Diff line number Diff line
@@ -1015,6 +1015,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region, 
	int shift, n, len;

	int shift, n, len;

	int i, j, k;

	int i, j, k;

	int err = 0;

	int err = 0;

	int write_mtt_size;





	shift = ffs(region->page_size) - 1;

	shift = ffs(region->page_size) - 1;
@@ -1040,6 +1041,8 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region, 




	i = n = 0;

	i = n = 0;





	write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages));



	list_for_each_entry(chunk, &region->chunk_list, list)

	list_for_each_entry(chunk, &region->chunk_list, list)

		for (j = 0; j < chunk->nmap; ++j) {

		for (j = 0; j < chunk->nmap; ++j) {

			len = sg_dma_len(&chunk->page_list[j]) >> shift;

			len = sg_dma_len(&chunk->page_list[j]) >> shift;
@@ -1047,14 +1050,11 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region, 
				pages[i++] = sg_dma_address(&chunk->page_list[j]) +

				pages[i++] = sg_dma_address(&chunk->page_list[j]) +

					region->page_size * k;

					region->page_size * k;

				/*

				/*

				 * Be friendly to WRITE_MTT command

				 * Be friendly to write_mtt and pass it chunks

				 * and leave two empty slots for the

				 * of appropriate size.

				 * index and reserved fields of the

				 * mailbox.

				 */

				 */

				if (i == PAGE_SIZE / sizeof (u64) - 2) {

				if (i == write_mtt_size) {

					err = mthca_write_mtt(dev, mr->mtt,

					err = mthca_write_mtt(dev, mr->mtt, n, pages, i);

							      n, pages, i);

					if (err)

					if (err)

						goto mtt_done;

						goto mtt_done;

					n += i;

					n += i;