Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 356e88eb authored by Jason Cai (Xiang Feng)'s avatar Jason Cai (Xiang Feng) Committed by Alex Williamson
Browse files

vfio/type1: Improve memory pinning process for raw PFN mapping 



When using vfio to pass through a PCIe device (e.g. a GPU card) that
has a huge BAR (e.g. 16GB), a lot of cycles are wasted on memory
pinning because PFNs of PCI BAR are not backed by struct page, and
the corresponding VMA has flag VM_PFNMAP.

With this change, when pinning a region which is a raw PFN mapping,
it can skip unnecessary user memory pinning process, and thus, can
significantly improve VM's boot up time when passing through devices
via VFIO. In my test on a Xeon E5 2.6GHz, the time mapping a 16GB
BAR was reduced from about 0.4s to 1.5us.

Signed-off-by: default avatarJason Cai (Xiang Feng) <jason.cai@linux.alibaba.com>
Signed-off-by: default avatarAlex Williamson <alex.williamson@redhat.com>
parent c9f89c3f

drivers/vfio/vfio_iommu_type1.c

+15 −10
Original line number Original line Diff line number Diff line
@@ -404,7 +404,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 
{

{

	unsigned long pfn = 0;

	unsigned long pfn = 0;

	long ret, pinned = 0, lock_acct = 0;

	long ret, pinned = 0, lock_acct = 0;

	bool rsvd;

	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;

	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;





	/* This code path is only user initiated */

	/* This code path is only user initiated */
@@ -415,14 +414,23 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 
	if (ret)

	if (ret)

		return ret;

		return ret;





	if (is_invalid_reserved_pfn(*pfn_base)) {

		struct vm_area_struct *vma;



		down_read(&current->mm->mmap_sem);

		vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);

		pinned = min_t(long, npage, vma_pages(vma));

		up_read(&current->mm->mmap_sem);

		return pinned;

	}



	pinned++;

	pinned++;

	rsvd = is_invalid_reserved_pfn(*pfn_base);





	/*

	/*

	 * Reserved pages aren't counted against the user, externally pinned

	 * Reserved pages aren't counted against the user, externally pinned

	 * pages are already counted against the user.

	 * pages are already counted against the user.

	 */

	 */

	if (!rsvd && !vfio_find_vpfn(dma, iova)) {

	if (!vfio_find_vpfn(dma, iova)) {

		if (!lock_cap && current->mm->locked_vm + 1 > limit) {

		if (!lock_cap && current->mm->locked_vm + 1 > limit) {

			put_pfn(*pfn_base, dma->prot);

			put_pfn(*pfn_base, dma->prot);

			pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,

			pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
@@ -442,13 +450,12 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 
		if (ret)

		if (ret)

			break;

			break;





		if (pfn != *pfn_base + pinned ||

		if (pfn != *pfn_base + pinned) {

		    rsvd != is_invalid_reserved_pfn(pfn)) {

			put_pfn(pfn, dma->prot);

			put_pfn(pfn, dma->prot);

			break;

			break;

		}

		}





		if (!rsvd && !vfio_find_vpfn(dma, iova)) {

		if (!vfio_find_vpfn(dma, iova)) {

			if (!lock_cap &&

			if (!lock_cap &&

			    current->mm->locked_vm + lock_acct + 1 > limit) {

			    current->mm->locked_vm + lock_acct + 1 > limit) {

				put_pfn(pfn, dma->prot);

				put_pfn(pfn, dma->prot);
@@ -466,10 +473,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 




unpin_out:

unpin_out:

	if (ret) {

	if (ret) {

		if (!rsvd) {

		for (pfn = *pfn_base ; pinned ; pfn++, pinned--)

		for (pfn = *pfn_base ; pinned ; pfn++, pinned--)

			put_pfn(pfn, dma->prot);

			put_pfn(pfn, dma->prot);

		}





		return ret;

		return ret;

	}

	}