Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit eeb8461e authored by Yishai Hadas's avatar Yishai Hadas Committed by Roland Dreier
Browse files

IB: Refactor umem to use linear SG table 



This patch refactors the IB core umem code and vendor drivers to use a
linear (chained) SG table instead of chunk list.  With this change the
relevant code becomes clearer—no need for nested loops to build and
use umem.

Signed-off-by: default avatarShachar Raindel <raindel@mellanox.com>
Signed-off-by: default avatarYishai Hadas <yishaih@mellanox.com>
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent cfbf8d48

drivers/infiniband/core/umem.c

+56 −64
Original line number Diff line number Diff line
@@ -42,29 +42,29 @@ 


#include "uverbs.h"



#define IB_UMEM_MAX_PAGE_CHUNK						\

	((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) /	\

	 ((void *) &((struct ib_umem_chunk *) 0)->page_list[1] -	\

	  (void *) &((struct ib_umem_chunk *) 0)->page_list[0]))



static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)

{

	struct ib_umem_chunk *chunk, *tmp;

	struct scatterlist *sg;

	struct page *page;

	int i;



	list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) {

		ib_dma_unmap_sg(dev, chunk->page_list,

				chunk->nents, DMA_BIDIRECTIONAL);

		for (i = 0; i < chunk->nents; ++i) {

			struct page *page = sg_page(&chunk->page_list[i]);

	if (umem->nmap > 0)

		ib_dma_unmap_sg(dev, umem->sg_head.sgl,

				umem->nmap,

				DMA_BIDIRECTIONAL);



	for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {



		page = sg_page(sg);

		if (umem->writable && dirty)

			set_page_dirty_lock(page);

		put_page(page);

	}



		kfree(chunk);

	}

	sg_free_table(&umem->sg_head);

	return;



}



/**
@@ -81,15 +81,15 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, 
	struct ib_umem *umem;

	struct page **page_list;

	struct vm_area_struct **vma_list;

	struct ib_umem_chunk *chunk;

	unsigned long locked;

	unsigned long lock_limit;

	unsigned long cur_base;

	unsigned long npages;

	int ret;

	int off;

	int i;

	DEFINE_DMA_ATTRS(attrs);

	struct scatterlist *sg, *sg_list_start;

	int need_release = 0;



	if (dmasync)

		dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs);
@@ -97,7 +97,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, 
	if (!can_do_mlock())

		return ERR_PTR(-EPERM);



	umem = kmalloc(sizeof *umem, GFP_KERNEL);

	umem = kzalloc(sizeof *umem, GFP_KERNEL);

	if (!umem)

		return ERR_PTR(-ENOMEM);
@@ -117,8 +117,6 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, 
	/* We assume the memory is from hugetlb until proved otherwise */

	umem->hugetlb   = 1;



	INIT_LIST_HEAD(&umem->chunk_list);



	page_list = (struct page **) __get_free_page(GFP_KERNEL);

	if (!page_list) {

		kfree(umem);
@@ -147,7 +145,18 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, 


	cur_base = addr & PAGE_MASK;



	ret = 0;

	if (npages == 0) {

		ret = -EINVAL;

		goto out;

	}



	ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL);

	if (ret)

		goto out;



	need_release = 1;

	sg_list_start = umem->sg_head.sgl;



	while (npages) {

		ret = get_user_pages(current, current->mm, cur_base,

				     min_t(unsigned long, npages,
@@ -157,53 +166,37 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, 
		if (ret < 0)

			goto out;



		umem->npages += ret;

		cur_base += ret * PAGE_SIZE;

		npages   -= ret;



		off = 0;

		for_each_sg(sg_list_start, sg, ret, i) {

			if (vma_list && !is_vm_hugetlb_page(vma_list[i]))

				umem->hugetlb = 0;



		while (ret) {

			chunk = kmalloc(sizeof *chunk + sizeof (struct scatterlist) *

					min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK),

					GFP_KERNEL);

			if (!chunk) {

				ret = -ENOMEM;

				goto out;

			sg_set_page(sg, page_list[i], PAGE_SIZE, 0);

		}



			chunk->nents = min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK);

			sg_init_table(chunk->page_list, chunk->nents);

			for (i = 0; i < chunk->nents; ++i) {

				if (vma_list &&

				    !is_vm_hugetlb_page(vma_list[i + off]))

					umem->hugetlb = 0;

				sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0);

		/* preparing for next loop */

		sg_list_start = sg;

	}



			chunk->nmap = ib_dma_map_sg_attrs(context->device,

							  &chunk->page_list[0],

							  chunk->nents,

	umem->nmap = ib_dma_map_sg_attrs(context->device,

				  umem->sg_head.sgl,

				  umem->npages,

				  DMA_BIDIRECTIONAL,

				  &attrs);

			if (chunk->nmap <= 0) {

				for (i = 0; i < chunk->nents; ++i)

					put_page(sg_page(&chunk->page_list[i]));

				kfree(chunk);



	if (umem->nmap <= 0) {

		ret = -ENOMEM;

		goto out;

	}



			ret -= chunk->nents;

			off += chunk->nents;

			list_add_tail(&chunk->list, &umem->chunk_list);

		}



	ret = 0;

	}



out:

	if (ret < 0) {

		if (need_release)

			__ib_umem_release(context->device, umem, 0);

		kfree(umem);

	} else
@@ -278,17 +271,16 @@ EXPORT_SYMBOL(ib_umem_release); 


int ib_umem_page_count(struct ib_umem *umem)

{

	struct ib_umem_chunk *chunk;

	int shift;

	int i;

	int n;

	struct scatterlist *sg;



	shift = ilog2(umem->page_size);



	n = 0;

	list_for_each_entry(chunk, &umem->chunk_list, list)

		for (i = 0; i < chunk->nmap; ++i)

			n += sg_dma_len(&chunk->page_list[i]) >> shift;

	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)

		n += sg_dma_len(sg) >> shift;



	return n;

}

drivers/infiniband/hw/amso1100/c2_provider.c

+9 −14
Original line number Diff line number Diff line
@@ -431,9 +431,9 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
	u64 *pages;

	u64 kva = 0;

	int shift, n, len;

	int i, j, k;

	int i, k, entry;

	int err = 0;

	struct ib_umem_chunk *chunk;

	struct scatterlist *sg;

	struct c2_pd *c2pd = to_c2pd(pd);

	struct c2_mr *c2mr;
@@ -452,10 +452,7 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
	}



	shift = ffs(c2mr->umem->page_size) - 1;



	n = 0;

	list_for_each_entry(chunk, &c2mr->umem->chunk_list, list)

		n += chunk->nents;

	n = c2mr->umem->nmap;



	pages = kmalloc(n * sizeof(u64), GFP_KERNEL);

	if (!pages) {
@@ -464,16 +461,14 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
	}



	i = 0;

	list_for_each_entry(chunk, &c2mr->umem->chunk_list, list) {

		for (j = 0; j < chunk->nmap; ++j) {

			len = sg_dma_len(&chunk->page_list[j]) >> shift;

	for_each_sg(c2mr->umem->sg_head.sgl, sg, c2mr->umem->nmap, entry) {

		len = sg_dma_len(sg) >> shift;

		for (k = 0; k < len; ++k) {

			pages[i++] =

					sg_dma_address(&chunk->page_list[j]) +

				sg_dma_address(sg) +

				(c2mr->umem->page_size * k);

		}

	}

	}



	kva = virt;

  	err = c2_nsmr_register_phys_kern(to_c2dev(pd->device),

drivers/infiniband/hw/cxgb3/iwch_provider.c

+7 −12
Original line number Diff line number Diff line
@@ -618,14 +618,13 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
{

	__be64 *pages;

	int shift, n, len;

	int i, j, k;

	int i, k, entry;

	int err = 0;

	struct ib_umem_chunk *chunk;

	struct iwch_dev *rhp;

	struct iwch_pd *php;

	struct iwch_mr *mhp;

	struct iwch_reg_user_mr_resp uresp;



	struct scatterlist *sg;

	PDBG("%s ib_pd %p\n", __func__, pd);



	php = to_iwch_pd(pd);
@@ -645,9 +644,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 


	shift = ffs(mhp->umem->page_size) - 1;



	n = 0;

	list_for_each_entry(chunk, &mhp->umem->chunk_list, list)

		n += chunk->nents;

	n = mhp->umem->nmap;



	err = iwch_alloc_pbl(mhp, n);

	if (err)
@@ -661,12 +658,10 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 


	i = n = 0;



	list_for_each_entry(chunk, &mhp->umem->chunk_list, list)

		for (j = 0; j < chunk->nmap; ++j) {

			len = sg_dma_len(&chunk->page_list[j]) >> shift;

	for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {

			len = sg_dma_len(sg) >> shift;

			for (k = 0; k < len; ++k) {

				pages[i++] = cpu_to_be64(sg_dma_address(

					&chunk->page_list[j]) +

				pages[i++] = cpu_to_be64(sg_dma_address(sg) +

					mhp->umem->page_size * k);

				if (i == PAGE_SIZE / sizeof *pages) {

					err = iwch_write_pbl(mhp, pages, i, n);

drivers/infiniband/hw/cxgb4/mem.c

+17 −22
Original line number Diff line number Diff line
@@ -678,9 +678,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
{

	__be64 *pages;

	int shift, n, len;

	int i, j, k;

	int i, k, entry;

	int err = 0;

	struct ib_umem_chunk *chunk;

	struct scatterlist *sg;

	struct c4iw_dev *rhp;

	struct c4iw_pd *php;

	struct c4iw_mr *mhp;
@@ -710,10 +710,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 


	shift = ffs(mhp->umem->page_size) - 1;



	n = 0;

	list_for_each_entry(chunk, &mhp->umem->chunk_list, list)

		n += chunk->nents;



	n = mhp->umem->nmap;

	err = alloc_pbl(mhp, n);

	if (err)

		goto err;
@@ -726,12 +723,10 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 


	i = n = 0;



	list_for_each_entry(chunk, &mhp->umem->chunk_list, list)

		for (j = 0; j < chunk->nmap; ++j) {

			len = sg_dma_len(&chunk->page_list[j]) >> shift;

	for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {

		len = sg_dma_len(sg) >> shift;

		for (k = 0; k < len; ++k) {

				pages[i++] = cpu_to_be64(sg_dma_address(

					&chunk->page_list[j]) +

			pages[i++] = cpu_to_be64(sg_dma_address(sg) +

				mhp->umem->page_size * k);

			if (i == PAGE_SIZE / sizeof *pages) {

				err = write_pbl(&mhp->rhp->rdev,

drivers/infiniband/hw/ehca/ehca_classes.h

+1 −1
Original line number Diff line number Diff line
@@ -322,7 +322,7 @@ struct ehca_mr_pginfo { 
		} phy;

		struct { /* type EHCA_MR_PGI_USER section */

			struct ib_umem *region;

			struct ib_umem_chunk *next_chunk;

			struct scatterlist *next_sg;

			u64 next_nmap;

		} usr;

		struct { /* type EHCA_MR_PGI_FMR section */