Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0c9aac08 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux 

Pull SLAB changes from Pekka Enberg:
 "There's the new kmalloc_array() API, minor fixes and performance
  improvements, but quite honestly, nothing terribly exciting."

* 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux:
  mm: SLAB Out-of-memory diagnostics
  slab: introduce kmalloc_array()
  slub: per cpu partial statistics change
  slub: include include for prefetch
  slub: Do not hold slub_lock when calling sysfs_slab_add()
  slub: prefetch next freelist pointer in slab_alloc()
  slab, cleanup: remove unneeded return
parents ed0bb8ea 8bdec192

include/linux/slab.h

+14 −3
Original line number Diff line number Diff line
@@ -190,7 +190,7 @@ size_t ksize(const void *); 
#endif



/**

 * kcalloc - allocate memory for an array. The memory is set to zero.

 * kmalloc_array - allocate memory for an array.

 * @n: number of elements.

 * @size: element size.

 * @flags: the type of memory to allocate.
@@ -240,11 +240,22 @@ size_t ksize(const void *); 
 * for general use, and so are not documented here. For a full list of

 * potential flags, always refer to linux/gfp.h.

 */

static inline void *kcalloc(size_t n, size_t size, gfp_t flags)

static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags)

{

	if (size != 0 && n > ULONG_MAX / size)

		return NULL;

	return __kmalloc(n * size, flags | __GFP_ZERO);

	return __kmalloc(n * size, flags);

}



/**

 * kcalloc - allocate memory for an array. The memory is set to zero.

 * @n: number of elements.

 * @size: element size.

 * @flags: the type of memory to allocate (see kmalloc).

 */

static inline void *kcalloc(size_t n, size_t size, gfp_t flags)

{

	return kmalloc_array(n, size, flags | __GFP_ZERO);

}



#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB)

include/linux/slub_def.h

+4 −2
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ enum stat_item { 
	FREE_FROZEN,		/* Freeing to frozen slab */

	FREE_ADD_PARTIAL,	/* Freeing moves slab to partial list */

	FREE_REMOVE_PARTIAL,	/* Freeing removes last object */

	ALLOC_FROM_PARTIAL,	/* Cpu slab acquired from partial list */

	ALLOC_FROM_PARTIAL,	/* Cpu slab acquired from node partial list */

	ALLOC_SLAB,		/* Cpu slab acquired from page allocator */

	ALLOC_REFILL,		/* Refill cpu slab from slab freelist */

	ALLOC_NODE_MISMATCH,	/* Switching cpu slab */
@@ -38,7 +38,9 @@ enum stat_item { 
	CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */

	CMPXCHG_DOUBLE_FAIL,	/* Number of times that cmpxchg double did not match */

	CPU_PARTIAL_ALLOC,	/* Used cpu partial on alloc */

	CPU_PARTIAL_FREE,	/* USed cpu partial on free */

	CPU_PARTIAL_FREE,	/* Refill cpu partial on free */

	CPU_PARTIAL_NODE,	/* Refill cpu partial from node partial */

	CPU_PARTIAL_DRAIN,	/* Drain cpu partial to node partial */

	NR_SLUB_STAT_ITEMS };



struct kmem_cache_cpu {

mm/slab.c

+52 −4
Original line number Diff line number Diff line
@@ -1731,6 +1731,52 @@ static int __init cpucache_init(void) 
}

__initcall(cpucache_init);



static noinline void

slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)

{

	struct kmem_list3 *l3;

	struct slab *slabp;

	unsigned long flags;

	int node;



	printk(KERN_WARNING

		"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",

		nodeid, gfpflags);

	printk(KERN_WARNING "  cache: %s, object size: %d, order: %d\n",

		cachep->name, cachep->buffer_size, cachep->gfporder);



	for_each_online_node(node) {

		unsigned long active_objs = 0, num_objs = 0, free_objects = 0;

		unsigned long active_slabs = 0, num_slabs = 0;



		l3 = cachep->nodelists[node];

		if (!l3)

			continue;



		spin_lock_irqsave(&l3->list_lock, flags);

		list_for_each_entry(slabp, &l3->slabs_full, list) {

			active_objs += cachep->num;

			active_slabs++;

		}

		list_for_each_entry(slabp, &l3->slabs_partial, list) {

			active_objs += slabp->inuse;

			active_slabs++;

		}

		list_for_each_entry(slabp, &l3->slabs_free, list)

			num_slabs++;



		free_objects += l3->free_objects;

		spin_unlock_irqrestore(&l3->list_lock, flags);



		num_slabs += active_slabs;

		num_objs = num_slabs * cachep->num;

		printk(KERN_WARNING

			"  node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",

			node, active_slabs, num_slabs, active_objs, num_objs,

			free_objects);

	}

}



/*

 * Interface to system's page allocator. No need to hold the cache-lock.

 *
@@ -1757,8 +1803,11 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) 
		flags |= __GFP_RECLAIMABLE;



	page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);

	if (!page)

	if (!page) {

		if (!(flags & __GFP_NOWARN) && printk_ratelimit())

			slab_out_of_memory(cachep, flags, nodeid);

		return NULL;

	}



	nr_pages = (1 << cachep->gfporder);

	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
@@ -3696,13 +3745,12 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, 


	if (likely(ac->avail < ac->limit)) {

		STATS_INC_FREEHIT(cachep);

		ac->entry[ac->avail++] = objp;

		return;

	} else {

		STATS_INC_FREEMISS(cachep);

		cache_flusharray(cachep, ac);

		ac->entry[ac->avail++] = objp;

	}



	ac->entry[ac->avail++] = objp;

}



/**

mm/slub.c

+21 −5
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <linux/math64.h>

#include <linux/fault-inject.h>

#include <linux/stacktrace.h>

#include <linux/prefetch.h>



#include <trace/events/kmem.h>
@@ -269,6 +270,11 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) 
	return *(void **)(object + s->offset);

}



static void prefetch_freepointer(const struct kmem_cache *s, void *object)

{

	prefetch(object + s->offset);

}



static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)

{

	void *p;
@@ -1560,6 +1566,7 @@ static void *get_partial_node(struct kmem_cache *s, 
		} else {

			page->freelist = t;

			available = put_cpu_partial(s, page, 0);

			stat(s, CPU_PARTIAL_NODE);

		}

		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)

			break;
@@ -1983,6 +1990,7 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) 
				local_irq_restore(flags);

				pobjects = 0;

				pages = 0;

				stat(s, CPU_PARTIAL_DRAIN);

			}

		}
@@ -1994,7 +2002,6 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) 
		page->next = oldpage;



	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);

	stat(s, CPU_PARTIAL_FREE);

	return pobjects;

}
@@ -2319,6 +2326,8 @@ redo: 
		object = __slab_alloc(s, gfpflags, node, addr, c);



	else {

		void *next_object = get_freepointer_safe(s, object);



		/*

		 * The cmpxchg will only match if there was no additional

		 * operation and if we are on the right processor.
@@ -2334,11 +2343,12 @@ redo: 
		if (unlikely(!this_cpu_cmpxchg_double(

				s->cpu_slab->freelist, s->cpu_slab->tid,

				object, tid,

				get_freepointer_safe(s, object), next_tid(tid)))) {

				next_object, next_tid(tid)))) {



			note_cmpxchg_failure("slab_alloc", s, tid);

			goto redo;

		}

		prefetch_freepointer(s, next_object);

		stat(s, ALLOC_FASTPATH);

	}
@@ -2475,9 +2485,10 @@ static void __slab_free(struct kmem_cache *s, struct page *page, 
		 * If we just froze the page then put it onto the

		 * per cpu partial list.

		 */

		if (new.frozen && !was_frozen)

		if (new.frozen && !was_frozen) {

			put_cpu_partial(s, page, 1);



			stat(s, CPU_PARTIAL_FREE);

		}

		/*

		 * The list lock was not taken therefore no list

		 * activity can be necessary.
@@ -3939,13 +3950,14 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, 
		if (kmem_cache_open(s, n,

				size, align, flags, ctor)) {

			list_add(&s->list, &slab_caches);

			up_write(&slub_lock);

			if (sysfs_slab_add(s)) {

				down_write(&slub_lock);

				list_del(&s->list);

				kfree(n);

				kfree(s);

				goto err;

			}

			up_write(&slub_lock);

			return s;

		}

		kfree(n);
@@ -5069,6 +5081,8 @@ STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);

STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);

STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);

STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);

STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);

#endif



static struct attribute *slab_attrs[] = {
@@ -5134,6 +5148,8 @@ static struct attribute *slab_attrs[] = { 
	&cmpxchg_double_cpu_fail_attr.attr,

	&cpu_partial_alloc_attr.attr,

	&cpu_partial_free_attr.attr,

	&cpu_partial_node_attr.attr,

	&cpu_partial_drain_attr.attr,

#endif

#ifdef CONFIG_FAILSLAB

	&failslab_attr.attr,