slab: Common Kmalloc cache determination (2c59dd65) · Commits · e / devices / android_kernel_fairphone_FP5

include/linux/slub_def.h

+10 −31

Original line number	Original line	Diff line number	Diff line
	@@ -115,29 +115,6 @@ struct kmem_cache {
	struct kmem_cache_node *node[MAX_NUMNODES];		struct kmem_cache_node *node[MAX_NUMNODES];
	};		};

	#ifdef CONFIG_ZONE_DMA
	#define SLUB_DMA __GFP_DMA
	#else
	/* Disable DMA functionality */
	#define SLUB_DMA (__force gfp_t)0
	#endif

	/*
	* Find the slab cache for a given combination of allocation flags and size.
	*
	* This ought to end up with a global pointer to the right cache
	* in kmalloc_caches.
	*/
	static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
	{
	int index = kmalloc_index(size);

	if (index == 0)
	return NULL;

	return kmalloc_caches[index];
	}

	void kmem_cache_alloc(struct kmem_cache , gfp_t);		void kmem_cache_alloc(struct kmem_cache , gfp_t);
	void *__kmalloc(size_t size, gfp_t flags);		void *__kmalloc(size_t size, gfp_t flags);

	@@ -195,13 +172,14 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags)
	if (size > KMALLOC_MAX_CACHE_SIZE)		if (size > KMALLOC_MAX_CACHE_SIZE)
	return kmalloc_large(size, flags);		return kmalloc_large(size, flags);

	if (!(flags & SLUB_DMA)) {		if (!(flags & GFP_DMA)) {
	struct kmem_cache *s = kmalloc_slab(size);		int index = kmalloc_index(size);

	if (!s)		if (!index)
	return ZERO_SIZE_PTR;		return ZERO_SIZE_PTR;

	return kmem_cache_alloc_trace(s, flags, size);		return kmem_cache_alloc_trace(kmalloc_caches[index],
			flags, size);
	}		}
	}		}
	return __kmalloc(size, flags);		return __kmalloc(size, flags);
	@@ -228,13 +206,14 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s,
	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)		static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
	{		{
	if (__builtin_constant_p(size) &&		if (__builtin_constant_p(size) &&
	size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) {		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
	struct kmem_cache *s = kmalloc_slab(size);		int index = kmalloc_index(size);

	if (!s)		if (!index)
	return ZERO_SIZE_PTR;		return ZERO_SIZE_PTR;

	return kmem_cache_alloc_node_trace(s, flags, node, size);		return kmem_cache_alloc_node_trace(kmalloc_caches[index],
			flags, node, size);
	}		}
	return __kmalloc_node(size, flags, node);		return __kmalloc_node(size, flags, node);
	}		}

mm/slab.c

+3 −37

Original line number	Original line	Diff line number	Diff line
	@@ -656,40 +656,6 @@ static inline struct array_cache cpu_cache_get(struct kmem_cache cachep)
	return cachep->array[smp_processor_id()];		return cachep->array[smp_processor_id()];
	}		}

	static inline struct kmem_cache *__find_general_cachep(size_t size,
	gfp_t gfpflags)
	{
	int i;

	#if DEBUG
	/* This happens if someone tries to call
	* kmem_cache_create(), or __kmalloc(), before
	* the generic caches are initialized.
	*/
	BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
	#endif
	if (!size)
	return ZERO_SIZE_PTR;

	i = kmalloc_index(size);

	/*
	* Really subtle: The last entry with cs->cs_size==ULONG_MAX
	* has cs_{dma,}cachep==NULL. Thus no special case
	* for large kmalloc calls required.
	*/
	#ifdef CONFIG_ZONE_DMA
	if (unlikely(gfpflags & GFP_DMA))
	return kmalloc_dma_caches[i];
	#endif
	return kmalloc_caches[i];
	}

	static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
	{
	return __find_general_cachep(size, gfpflags);
	}

	static size_t slab_mgmt_size(size_t nr_objs, size_t align)		static size_t slab_mgmt_size(size_t nr_objs, size_t align)
	{		{
	return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);		return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
	@@ -2426,7 +2392,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
	cachep->reciprocal_buffer_size = reciprocal_value(size);		cachep->reciprocal_buffer_size = reciprocal_value(size);

	if (flags & CFLGS_OFF_SLAB) {		if (flags & CFLGS_OFF_SLAB) {
	cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);		cachep->slabp_cache = kmalloc_slab(slab_size, 0u);
	/*		/*
	* This is a possibility for one of the malloc_sizes caches.		* This is a possibility for one of the malloc_sizes caches.
	* But since we go off slab only for object size greater than		* But since we go off slab only for object size greater than
	@@ -3729,7 +3695,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
	{		{
	struct kmem_cache *cachep;		struct kmem_cache *cachep;

	cachep = kmem_find_general_cachep(size, flags);		cachep = kmalloc_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))		if (unlikely(ZERO_OR_NULL_PTR(cachep)))
	return cachep;		return cachep;
	return kmem_cache_alloc_node_trace(cachep, flags, node, size);		return kmem_cache_alloc_node_trace(cachep, flags, node, size);
	@@ -3774,7 +3740,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
	* Then kmalloc uses the uninlined functions instead of the inline		* Then kmalloc uses the uninlined functions instead of the inline
	* functions.		* functions.
	*/		*/
	cachep = __find_general_cachep(size, flags);		cachep = kmalloc_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))		if (unlikely(ZERO_OR_NULL_PTR(cachep)))
	return cachep;		return cachep;
	ret = slab_alloc(cachep, flags, caller);		ret = slab_alloc(cachep, flags, caller);

mm/slab.h

+3 −0

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,9 @@ unsigned long calculate_alignment(unsigned long flags,
	#ifndef CONFIG_SLOB		#ifndef CONFIG_SLOB
	/* Kmalloc array related functions */		/* Kmalloc array related functions */
	void create_kmalloc_caches(unsigned long);		void create_kmalloc_caches(unsigned long);

			/* Find the kmalloc slab corresponding for a certain size */
			struct kmem_cache *kmalloc_slab(size_t, gfp_t);
	#endif		#endif

mm/slab_common.c

+103 −2

Original line number	Original line	Diff line number	Diff line
	@@ -327,6 +327,68 @@ struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
	EXPORT_SYMBOL(kmalloc_dma_caches);		EXPORT_SYMBOL(kmalloc_dma_caches);
	#endif		#endif

			/*
			* Conversion table for small slabs sizes / 8 to the index in the
			* kmalloc array. This is necessary for slabs < 192 since we have non power
			* of two cache sizes there. The size of larger slabs can be determined using
			* fls.
			*/
			static s8 size_index[24] = {
			3, /* 8 */
			4, /* 16 */
			5, /* 24 */
			5, /* 32 */
			6, /* 40 */
			6, /* 48 */
			6, /* 56 */
			6, /* 64 */
			1, /* 72 */
			1, /* 80 */
			1, /* 88 */
			1, /* 96 */
			7, /* 104 */
			7, /* 112 */
			7, /* 120 */
			7, /* 128 */
			2, /* 136 */
			2, /* 144 */
			2, /* 152 */
			2, /* 160 */
			2, /* 168 */
			2, /* 176 */
			2, /* 184 */
			2 /* 192 */
			};

			static inline int size_index_elem(size_t bytes)
			{
			return (bytes - 1) / 8;
			}

			/*
			* Find the kmem_cache structure that serves a given size of
			* allocation
			*/
			struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
			{
			int index;

			if (size <= 192) {
			if (!size)
			return ZERO_SIZE_PTR;

			index = size_index[size_index_elem(size)];
			} else
			index = fls(size - 1);

			#ifdef CONFIG_ZONE_DMA
			if (unlikely((flags & SLAB_CACHE_DMA)))
			return kmalloc_dma_caches[index];

			#endif
			return kmalloc_caches[index];
			}

	/*		/*
	* Create the kmalloc array. Some of the regular kmalloc arrays		* Create the kmalloc array. Some of the regular kmalloc arrays
	* may already have been created because they were needed to		* may already have been created because they were needed to
	@@ -336,6 +398,47 @@ void __init create_kmalloc_caches(unsigned long flags)
	{		{
	int i;		int i;

			/*
			* Patch up the size_index table if we have strange large alignment
			* requirements for the kmalloc array. This is only the case for
			* MIPS it seems. The standard arches will not generate any code here.
			*
			* Largest permitted alignment is 256 bytes due to the way we
			* handle the index determination for the smaller caches.
			*
			* Make sure that nothing crazy happens if someone starts tinkering
			* around with ARCH_KMALLOC_MINALIGN
			*/
			BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 \|\|
			(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

			for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
			int elem = size_index_elem(i);

			if (elem >= ARRAY_SIZE(size_index))
			break;
			size_index[elem] = KMALLOC_SHIFT_LOW;
			}

			if (KMALLOC_MIN_SIZE >= 64) {
			/*
			* The 96 byte size cache is not used if the alignment
			* is 64 byte.
			*/
			for (i = 64 + 8; i <= 96; i += 8)
			size_index[size_index_elem(i)] = 7;

			}

			if (KMALLOC_MIN_SIZE >= 128) {
			/*
			* The 192 byte sized cache is not used if the alignment
			* is 128 byte. Redirect kmalloc to use the 256 byte cache
			* instead.
			*/
			for (i = 128 + 8; i <= 192; i += 8)
			size_index[size_index_elem(i)] = 8;
			}
	/* Caches that are not of the two-to-the-power-of size */		/* Caches that are not of the two-to-the-power-of size */
	if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])		if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
	kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);		kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
	@@ -379,8 +482,6 @@ void __init create_kmalloc_caches(unsigned long flags)
	}		}
	#endif		#endif
	}		}


	#endif /* !CONFIG_SLOB */		#endif /* !CONFIG_SLOB */

mm/slub.c

+5 −103

Original line number	Original line	Diff line number	Diff line
	@@ -2982,7 +2982,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
	s->allocflags \|= __GFP_COMP;		s->allocflags \|= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)		if (s->flags & SLAB_CACHE_DMA)
	s->allocflags \|= SLUB_DMA;		s->allocflags \|= GFP_DMA;

	if (s->flags & SLAB_RECLAIM_ACCOUNT)		if (s->flags & SLAB_RECLAIM_ACCOUNT)
	s->allocflags \|= __GFP_RECLAIMABLE;		s->allocflags \|= __GFP_RECLAIMABLE;
	@@ -3210,64 +3210,6 @@ static int __init setup_slub_nomerge(char *str)

	__setup("slub_nomerge", setup_slub_nomerge);		__setup("slub_nomerge", setup_slub_nomerge);

	/*
	* Conversion table for small slabs sizes / 8 to the index in the
	* kmalloc array. This is necessary for slabs < 192 since we have non power
	* of two cache sizes there. The size of larger slabs can be determined using
	* fls.
	*/
	static s8 size_index[24] = {
	3, /* 8 */
	4, /* 16 */
	5, /* 24 */
	5, /* 32 */
	6, /* 40 */
	6, /* 48 */
	6, /* 56 */
	6, /* 64 */
	1, /* 72 */
	1, /* 80 */
	1, /* 88 */
	1, /* 96 */
	7, /* 104 */
	7, /* 112 */
	7, /* 120 */
	7, /* 128 */
	2, /* 136 */
	2, /* 144 */
	2, /* 152 */
	2, /* 160 */
	2, /* 168 */
	2, /* 176 */
	2, /* 184 */
	2 /* 192 */
	};

	static inline int size_index_elem(size_t bytes)
	{
	return (bytes - 1) / 8;
	}

	static struct kmem_cache *get_slab(size_t size, gfp_t flags)
	{
	int index;

	if (size <= 192) {
	if (!size)
	return ZERO_SIZE_PTR;

	index = size_index[size_index_elem(size)];
	} else
	index = fls(size - 1);

	#ifdef CONFIG_ZONE_DMA
	if (unlikely((flags & SLUB_DMA)))
	return kmalloc_dma_caches[index];

	#endif
	return kmalloc_caches[index];
	}

	void *__kmalloc(size_t size, gfp_t flags)		void *__kmalloc(size_t size, gfp_t flags)
	{		{
	struct kmem_cache *s;		struct kmem_cache *s;
	@@ -3276,7 +3218,7 @@ void *__kmalloc(size_t size, gfp_t flags)
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))		if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
	return kmalloc_large(size, flags);		return kmalloc_large(size, flags);

	s = get_slab(size, flags);		s = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))		if (unlikely(ZERO_OR_NULL_PTR(s)))
	return s;		return s;
	@@ -3319,7 +3261,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
	return ret;		return ret;
	}		}

	s = get_slab(size, flags);		s = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))		if (unlikely(ZERO_OR_NULL_PTR(s)))
	return s;		return s;
	@@ -3632,7 +3574,6 @@ void __init kmem_cache_init(void)
	{		{
	static __initdata struct kmem_cache boot_kmem_cache,		static __initdata struct kmem_cache boot_kmem_cache,
	boot_kmem_cache_node;		boot_kmem_cache_node;
	int i;

	if (debug_guardpage_minorder())		if (debug_guardpage_minorder())
	slub_max_order = 0;		slub_max_order = 0;
	@@ -3663,45 +3604,6 @@ void __init kmem_cache_init(void)
	kmem_cache_node = bootstrap(&boot_kmem_cache_node);		kmem_cache_node = bootstrap(&boot_kmem_cache_node);

	/* Now we can use the kmem_cache to allocate kmalloc slabs */		/* Now we can use the kmem_cache to allocate kmalloc slabs */

	/*
	* Patch up the size_index table if we have strange large alignment
	* requirements for the kmalloc array. This is only the case for
	* MIPS it seems. The standard arches will not generate any code here.
	*
	* Largest permitted alignment is 256 bytes due to the way we
	* handle the index determination for the smaller caches.
	*
	* Make sure that nothing crazy happens if someone starts tinkering
	* around with ARCH_KMALLOC_MINALIGN
	*/
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 \|\|
	(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

	for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
	int elem = size_index_elem(i);
	if (elem >= ARRAY_SIZE(size_index))
	break;
	size_index[elem] = KMALLOC_SHIFT_LOW;
	}

	if (KMALLOC_MIN_SIZE == 64) {
	/*
	* The 96 byte size cache is not used if the alignment
	* is 64 byte.
	*/
	for (i = 64 + 8; i <= 96; i += 8)
	size_index[size_index_elem(i)] = 7;
	} else if (KMALLOC_MIN_SIZE == 128) {
	/*
	* The 192 byte sized cache is not used if the alignment
	* is 128 byte. Redirect kmalloc to use the 256 byte cache
	* instead.
	*/
	for (i = 128 + 8; i <= 192; i += 8)
	size_index[size_index_elem(i)] = 8;
	}

	create_kmalloc_caches(0);		create_kmalloc_caches(0);

	#ifdef CONFIG_SMP		#ifdef CONFIG_SMP
	@@ -3877,7 +3779,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))		if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
	return kmalloc_large(size, gfpflags);		return kmalloc_large(size, gfpflags);

	s = get_slab(size, gfpflags);		s = kmalloc_slab(size, gfpflags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))		if (unlikely(ZERO_OR_NULL_PTR(s)))
	return s;		return s;
	@@ -3907,7 +3809,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
	return ret;		return ret;
	}		}

	s = get_slab(size, gfpflags);		s = kmalloc_slab(size, gfpflags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))		if (unlikely(ZERO_OR_NULL_PTR(s)))
	return s;		return s;