Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0316bec2 authored by Andrey Ryabinin's avatar Andrey Ryabinin Committed by Linus Torvalds
Browse files

mm: slub: add kernel address sanitizer support for slub allocator 



With this patch kasan will be able to catch bugs in memory allocated by
slub.  Initially all objects in newly allocated slab page, marked as
redzone.  Later, when allocation of slub object happens, requested by
caller number of bytes marked as accessible, and the rest of the object
(including slub's metadata) marked as redzone (inaccessible).

We also mark object as accessible if ksize was called for this object.
There is some places in kernel where ksize function is called to inquire
size of really allocated area.  Such callers could validly access whole
allocated memory, so it should be marked as accessible.

Code in slub.c and slab_common.c files could validly access to object's
metadata, so instrumentation for this files are disabled.

Signed-off-by: default avatarAndrey Ryabinin <a.ryabinin@samsung.com>
Signed-off-by: default avatarDmitry Chernenkov <dmitryc@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Konstantin Serebryany <kcc@google.com>
Signed-off-by: default avatarAndrey Konovalov <adech.fo@gmail.com>
Cc: Yuri Gribov <tetra2005@gmail.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Sasha Levin <sasha.levin@oracle.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a79316c6

include/linux/kasan.h

+27 −0
Original line number Diff line number Diff line
@@ -37,6 +37,18 @@ void kasan_unpoison_shadow(const void *address, size_t size); 
void kasan_alloc_pages(struct page *page, unsigned int order);

void kasan_free_pages(struct page *page, unsigned int order);



void kasan_poison_slab(struct page *page);

void kasan_unpoison_object_data(struct kmem_cache *cache, void *object);

void kasan_poison_object_data(struct kmem_cache *cache, void *object);



void kasan_kmalloc_large(const void *ptr, size_t size);

void kasan_kfree_large(const void *ptr);

void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size);

void kasan_krealloc(const void *object, size_t new_size);



void kasan_slab_alloc(struct kmem_cache *s, void *object);

void kasan_slab_free(struct kmem_cache *s, void *object);



#else /* CONFIG_KASAN */



static inline void kasan_unpoison_shadow(const void *address, size_t size) {}
@@ -47,6 +59,21 @@ static inline void kasan_disable_current(void) {} 
static inline void kasan_alloc_pages(struct page *page, unsigned int order) {}

static inline void kasan_free_pages(struct page *page, unsigned int order) {}



static inline void kasan_poison_slab(struct page *page) {}

static inline void kasan_unpoison_object_data(struct kmem_cache *cache,

					void *object) {}

static inline void kasan_poison_object_data(struct kmem_cache *cache,

					void *object) {}



static inline void kasan_kmalloc_large(void *ptr, size_t size) {}

static inline void kasan_kfree_large(const void *ptr) {}

static inline void kasan_kmalloc(struct kmem_cache *s, const void *object,

				size_t size) {}

static inline void kasan_krealloc(const void *object, size_t new_size) {}



static inline void kasan_slab_alloc(struct kmem_cache *s, void *object) {}

static inline void kasan_slab_free(struct kmem_cache *s, void *object) {}



#endif /* CONFIG_KASAN */



#endif /* LINUX_KASAN_H */

include/linux/slab.h

+9 −2
Original line number Diff line number Diff line
@@ -104,6 +104,7 @@ 
				(unsigned long)ZERO_SIZE_PTR)



#include <linux/kmemleak.h>

#include <linux/kasan.h>



struct mem_cgroup;

/*
@@ -325,7 +326,10 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, 
static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,

		gfp_t flags, size_t size)

{

	return kmem_cache_alloc(s, flags);

	void *ret = kmem_cache_alloc(s, flags);



	kasan_kmalloc(s, ret, size);

	return ret;

}



static __always_inline void *
@@ -333,7 +337,10 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, 
			      gfp_t gfpflags,

			      int node, size_t size)

{

	return kmem_cache_alloc_node(s, gfpflags, node);

	void *ret = kmem_cache_alloc_node(s, gfpflags, node);



	kasan_kmalloc(s, ret, size);

	return ret;

}

#endif /* CONFIG_TRACING */

lib/Kconfig.kasan

+1 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ if HAVE_ARCH_KASAN 


config KASAN

	bool "KASan: runtime memory debugger"

	depends on SLUB_DEBUG

	help

	  Enables kernel address sanitizer - runtime memory debugger,

	  designed to find out-of-bounds accesses and use-after-free bugs.

mm/Makefile

+3 −0
Original line number Diff line number Diff line
@@ -2,6 +2,9 @@ 
# Makefile for the linux memory manager.

#



KASAN_SANITIZE_slab_common.o := n

KASAN_SANITIZE_slub.o := n



mmu-y			:= nommu.o

mmu-$(CONFIG_MMU)	:= gup.o highmem.o memory.o mincore.o \

			   mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \

mm/kasan/kasan.c

+98 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ 
#include <linux/kasan.h>



#include "kasan.h"

#include "../slab.h"



/*

 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
@@ -268,6 +269,103 @@ void kasan_free_pages(struct page *page, unsigned int order) 
				KASAN_FREE_PAGE);

}



void kasan_poison_slab(struct page *page)

{

	kasan_poison_shadow(page_address(page),

			PAGE_SIZE << compound_order(page),

			KASAN_KMALLOC_REDZONE);

}



void kasan_unpoison_object_data(struct kmem_cache *cache, void *object)

{

	kasan_unpoison_shadow(object, cache->object_size);

}



void kasan_poison_object_data(struct kmem_cache *cache, void *object)

{

	kasan_poison_shadow(object,

			round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),

			KASAN_KMALLOC_REDZONE);

}



void kasan_slab_alloc(struct kmem_cache *cache, void *object)

{

	kasan_kmalloc(cache, object, cache->object_size);

}



void kasan_slab_free(struct kmem_cache *cache, void *object)

{

	unsigned long size = cache->object_size;

	unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);



	/* RCU slabs could be legally used after free within the RCU period */

	if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))

		return;



	kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);

}



void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size)

{

	unsigned long redzone_start;

	unsigned long redzone_end;



	if (unlikely(object == NULL))

		return;



	redzone_start = round_up((unsigned long)(object + size),

				KASAN_SHADOW_SCALE_SIZE);

	redzone_end = round_up((unsigned long)object + cache->object_size,

				KASAN_SHADOW_SCALE_SIZE);



	kasan_unpoison_shadow(object, size);

	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,

		KASAN_KMALLOC_REDZONE);

}

EXPORT_SYMBOL(kasan_kmalloc);



void kasan_kmalloc_large(const void *ptr, size_t size)

{

	struct page *page;

	unsigned long redzone_start;

	unsigned long redzone_end;



	if (unlikely(ptr == NULL))

		return;



	page = virt_to_page(ptr);

	redzone_start = round_up((unsigned long)(ptr + size),

				KASAN_SHADOW_SCALE_SIZE);

	redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));



	kasan_unpoison_shadow(ptr, size);

	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,

		KASAN_PAGE_REDZONE);

}



void kasan_krealloc(const void *object, size_t size)

{

	struct page *page;



	if (unlikely(object == ZERO_SIZE_PTR))

		return;



	page = virt_to_head_page(object);



	if (unlikely(!PageSlab(page)))

		kasan_kmalloc_large(object, size);

	else

		kasan_kmalloc(page->slab_cache, object, size);

}



void kasan_kfree_large(const void *ptr)

{

	struct page *page = virt_to_page(ptr);



	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),

			KASAN_FREE_PAGE);

}



#define DEFINE_ASAN_LOAD_STORE(size)				\

	void __asan_load##size(unsigned long addr)		\

	{							\