Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a9869b83 authored by Naoya Horiguchi's avatar Naoya Horiguchi Committed by Andi Kleen
Browse files

hugetlb: move refcounting in hugepage allocation inside hugetlb_lock 



Currently alloc_huge_page() raises page refcount outside hugetlb_lock.
but it causes race when dequeue_hwpoison_huge_page() runs concurrently
with alloc_huge_page().
To avoid it, this patch moves set_page_refcounted() in hugetlb_lock.

Signed-off-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: default avatarWu Fengguang <fengguang.wu@intel.com>
Acked-by: default avatarMel Gorman <mel@csn.ul.ie>
Reviewed-by: default avatarChristoph Lameter <cl@linux.com>
Signed-off-by: default avatarAndi Kleen <ak@linux.intel.com>
parent 6de2b1aa

mm/hugetlb.c

+13 −22
Original line number Original line Diff line number Diff line
@@ -509,6 +509,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid) 
		return NULL;

		return NULL;

	page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);

	page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);

	list_del(&page->lru);

	list_del(&page->lru);

	set_page_refcounted(page);

	h->free_huge_pages--;

	h->free_huge_pages--;

	h->free_huge_pages_node[nid]--;

	h->free_huge_pages_node[nid]--;

	return page;

	return page;
@@ -868,12 +869,6 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid) 




	spin_lock(&hugetlb_lock);

	spin_lock(&hugetlb_lock);

	if (page) {

	if (page) {

		/*

		 * This page is now managed by the hugetlb allocator and has

		 * no users -- drop the buddy allocator's reference.

		 */

		put_page_testzero(page);

		VM_BUG_ON(page_count(page));

		r_nid = page_to_nid(page);

		r_nid = page_to_nid(page);

		set_compound_page_dtor(page, free_huge_page);

		set_compound_page_dtor(page, free_huge_page);

		/*

		/*
@@ -936,16 +931,13 @@ static int gather_surplus_pages(struct hstate *h, int delta) 
	spin_unlock(&hugetlb_lock);

	spin_unlock(&hugetlb_lock);

	for (i = 0; i < needed; i++) {

	for (i = 0; i < needed; i++) {

		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);

		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);

		if (!page) {

		if (!page)

			/*

			/*

			 * We were not able to allocate enough pages to

			 * We were not able to allocate enough pages to

			 * satisfy the entire reservation so we free what

			 * satisfy the entire reservation so we free what

			 * we've allocated so far.

			 * we've allocated so far.

			 */

			 */

			spin_lock(&hugetlb_lock);

			needed = 0;

			goto free;

			goto free;

		}





		list_add(&page->lru, &surplus_list);

		list_add(&page->lru, &surplus_list);

	}

	}
@@ -972,31 +964,31 @@ static int gather_surplus_pages(struct hstate *h, int delta) 
	needed += allocated;

	needed += allocated;

	h->resv_huge_pages += delta;

	h->resv_huge_pages += delta;

	ret = 0;

	ret = 0;

free:



	spin_unlock(&hugetlb_lock);

	/* Free the needed pages to the hugetlb pool */

	/* Free the needed pages to the hugetlb pool */

	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

		if ((--needed) < 0)

		if ((--needed) < 0)

			break;

			break;

		list_del(&page->lru);

		list_del(&page->lru);

		/*

		 * This page is now managed by the hugetlb allocator and has

		 * no users -- drop the buddy allocator's reference.

		 */

		put_page_testzero(page);

		VM_BUG_ON(page_count(page));

		enqueue_huge_page(h, page);

		enqueue_huge_page(h, page);

	}

	}





	/* Free unnecessary surplus pages to the buddy allocator */

	/* Free unnecessary surplus pages to the buddy allocator */

free:

	if (!list_empty(&surplus_list)) {

	if (!list_empty(&surplus_list)) {

		spin_unlock(&hugetlb_lock);

		list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

		list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

			list_del(&page->lru);

			list_del(&page->lru);

			/*

			put_page(page);

			 * The page has a reference count of zero already, so

			 * call free_huge_page directly instead of using

			 * put_page.  This must be done with hugetlb_lock

			 * unlocked which is safe because free_huge_page takes

			 * hugetlb_lock before deciding how to free the page.

			 */

			free_huge_page(page);

		}

		}

		spin_lock(&hugetlb_lock);

	}

	}

	spin_lock(&hugetlb_lock);





	return ret;

	return ret;

}

}
@@ -1123,7 +1115,6 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma, 
		}

		}

	}

	}





	set_page_refcounted(page);

	set_page_private(page, (unsigned long) mapping);

	set_page_private(page, (unsigned long) mapping);





	vma_commit_reservation(h, vma, addr);

	vma_commit_reservation(h, vma, addr);