Commit 48480dc3 authored Apr 24, 2019 by Roman Gushchin Committed by Vijayanand Jitta Apr 25, 2019

mm/vmalloc.c: refactor __vunmap() to avoid duplicated call to find_vm_area()

__vunmap() calls find_vm_area() twice without an obvious reason: first
directly to get the area pointer, second indirectly by calling
remove_vm_area(), which is again searching for the area.

To remove this redundancy, let's split remove_vm_area() into
__remove_vm_area(struct vmap_area *), which performs the actual area
removal, and remove_vm_area(const void *addr) wrapper, which can be used
everywhere, where it has been used before.

On my test setup, I've got 5-10% speed up on vfree()'ing 1000000 of
4-pages vmalloc blocks.

Perf report before:
  22.64%  cat      [kernel.vmlinux]  [k] free_pcppages_bulk
  10.30%  cat      [kernel.vmlinux]  [k] __vunmap
   9.80%  cat      [kernel.vmlinux]  [k] find_vmap_area
   8.11%  cat      [kernel.vmlinux]  [k] vunmap_page_range
   4.20%  cat      [kernel.vmlinux]  [k] __slab_free
   3.56%  cat      [kernel.vmlinux]  [k] __list_del_entry_valid
   3.46%  cat      [kernel.vmlinux]  [k] smp_call_function_many
   3.33%  cat      [kernel.vmlinux]  [k] kfree
   3.32%  cat      [kernel.vmlinux]  [k] free_unref_page

Perf report after:
  23.01%  cat      [kernel.kallsyms]  [k] free_pcppages_bulk
   9.46%  cat      [kernel.kallsyms]  [k] __vunmap
   9.15%  cat      [kernel.kallsyms]  [k] vunmap_page_range
   6.17%  cat      [kernel.kallsyms]  [k] __slab_free
   5.61%  cat      [kernel.kallsyms]  [k] kfree
   4.86%  cat      [kernel.kallsyms]  [k] bad_range
   4.67%  cat      [kernel.kallsyms]  [k] free_unref_page_commit
   4.24%  cat      [kernel.kallsyms]  [k] __list_del_entry_valid
   3.68%  cat      [kernel.kallsyms]  [k] free_unref_page
   3.65%  cat      [kernel.kallsyms]  [k] __list_add_valid
   3.19%  cat      [kernel.kallsyms]  [k] __purge_vmap_area_lazy
   3.10%  cat      [kernel.kallsyms]  [k] find_vmap_area
   3.05%  cat      [kernel.kallsyms]  [k] rcu_cblist_dequeue

Link: http://lkml.kernel.org/r/20190417194002.12369-2-guro@fb.com


Signed-off-by: Roman Gushchin <guro@fb.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Matthew Wilcox <willy@infradead.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
Git-commit: e97ad926639fef3930abb72b1f5588d655ee4d74
Git-repo: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git


Change-Id: Id734ceca4b12c9445bf5166794aa49bea92ec402
Signed-off-by: Vijayanand Jitta <vjitta@codeaurora.org>

parent f74ff626

mm/vmalloc.c

+29 −22

Original line number	Diff line number	Diff line
		@@ -1462,24 +1462,12 @@ struct vm_struct find_vm_area(const void addr)
		return NULL;
		}

		/**
		* remove_vm_area - find and remove a continuous kernel virtual area
		* @addr: base address
		*
		* Search for the kernel VM area starting at @addr, and remove it.
		* This function returns the found VM area, but using it is NOT safe
		* on SMP machines, except for its size or flags.
		*/
		struct vm_struct remove_vm_area(const void addr)
		static struct vm_struct __remove_vm_area(struct vmap_area va)
		{
		struct vmap_area *va;
		struct vm_struct *vm = va->vm;

		might_sleep();

		va = find_vmap_area((unsigned long)addr);
		if (va && va->flags & VM_VM_AREA) {
		struct vm_struct *vm = va->vm;

		spin_lock(&vmap_area_lock);
		va->vm = NULL;
		va->flags &= ~VM_VM_AREA;
		@@ -1491,12 +1479,31 @@ struct vm_struct remove_vm_area(const void addr)

		return vm;
		}
		return NULL;

		/**
		* remove_vm_area - find and remove a continuous kernel virtual area
		* @addr: base address
		*
		* Search for the kernel VM area starting at @addr, and remove it.
		* This function returns the found VM area, but using it is NOT safe
		* on SMP machines, except for its size or flags.
		*/
		struct vm_struct remove_vm_area(const void addr)
		{
		struct vm_struct *vm = NULL;
		struct vmap_area *va;

		va = find_vmap_area((unsigned long)addr);
		if (va && va->flags & VM_VM_AREA)
		vm = __remove_vm_area(va);

		return vm;
		}

		static void __vunmap(const void *addr, int deallocate_pages)
		{
		struct vm_struct *area;
		struct vmap_area *va;

		if (!addr)
		return;
		@@ -1505,17 +1512,18 @@ static void __vunmap(const void *addr, int deallocate_pages)
		addr))
		return;

		area = find_vmap_area((unsigned long)addr)->vm;
		if (unlikely(!area)) {
		va = find_vmap_area((unsigned long)addr);
		if (unlikely(!va \|\| !(va->flags & VM_VM_AREA))) {
		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
		addr);
		return;
		}

		debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
		debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
		area = va->vm;
		debug_check_no_locks_freed(addr, get_vm_area_size(area));
		debug_check_no_obj_freed(addr, get_vm_area_size(area));

		remove_vm_area(addr);
		__remove_vm_area(va);
		if (deallocate_pages) {
		int i;

		@@ -1530,7 +1538,6 @@ static void __vunmap(const void *addr, int deallocate_pages)
		}

		kfree(area);
		return;
		}

		static inline void __vfree_deferred(const void *addr)