Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c11abbba authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'slub/lockless' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6 

* 'slub/lockless' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6: (21 commits)
  slub: When allocating a new slab also prep the first object
  slub: disable interrupts in cmpxchg_double_slab when falling back to pagelock
  Avoid duplicate _count variables in page_struct
  Revert "SLUB: Fix build breakage in linux/mm_types.h"
  SLUB: Fix build breakage in linux/mm_types.h
  slub: slabinfo update for cmpxchg handling
  slub: Not necessary to check for empty slab on load_freelist
  slub: fast release on full slab
  slub: Add statistics for the case that the current slab does not match the node
  slub: Get rid of the another_slab label
  slub: Avoid disabling interrupts in free slowpath
  slub: Disable interrupts in free_debug processing
  slub: Invert locking and avoid slab lock
  slub: Rework allocator fastpaths
  slub: Pass kmem_cache struct to lock and freeze slab
  slub: explicit list_lock taking
  slub: Add cmpxchg_double_slab()
  mm: Rearrange struct page
  slub: Move page->frozen handling near where the page->freelist handling occurs
  slub: Do not use frozen page flag but a bit in the page counters
  ...
parents 1d3fe4a7 9e577e8b

include/linux/mm_types.h

+61 −28
Original line number Diff line number Diff line
@@ -30,23 +30,61 @@ struct address_space; 
 * moment. Note that we have no way to track which tasks are using

 * a page, though if it is a pagecache page, rmap structures can tell us

 * who is mapping it.

 *

 * The objects in struct page are organized in double word blocks in

 * order to allows us to use atomic double word operations on portions

 * of struct page. That is currently only used by slub but the arrangement

 * allows the use of atomic double word operations on the flags/mapping

 * and lru list pointers also.

 */

struct page {

	/* First double word block */

	unsigned long flags;		/* Atomic flags, some possibly

					 * updated asynchronously */

	atomic_t _count;		/* Usage count, see below. */

	struct address_space *mapping;	/* If low bit clear, points to

					 * inode address_space, or NULL.

					 * If page mapped as anonymous

					 * memory, low bit is set, and

					 * it points to anon_vma object:

					 * see PAGE_MAPPING_ANON below.

					 */

	/* Second double word */

	struct {

		union {

			pgoff_t index;		/* Our offset within mapping. */

			void *freelist;		/* slub first free object */

		};



		union {

			/* Used for cmpxchg_double in slub */

			unsigned long counters;



			struct {



				union {

					atomic_t _mapcount;	/* Count of ptes mapped in mms,

							 * to show when page is mapped

							 * & limit reverse map searches.

							 */

		struct {		/* SLUB */

			u16 inuse;

			u16 objects;



					struct {

						unsigned inuse:16;

						unsigned objects:15;

						unsigned frozen:1;

					};

				};

				atomic_t _count;		/* Usage count, see below. */

			};

		};

	};



	/* Third double word block */

	struct list_head lru;		/* Pageout list, eg. active_list

					 * protected by zone->lru_lock !

					 */



	/* Remainder is not double word aligned */

	union {

	    struct {

		unsigned long private;		/* Mapping-private opaque data:

					 	 * usually used for buffer_heads

						 * if PagePrivate set; used for
@@ -54,27 +92,13 @@ struct page { 
						 * indicates order in the buddy

						 * system if PG_buddy is set.

						 */

		struct address_space *mapping;	/* If low bit clear, points to

						 * inode address_space, or NULL.

						 * If page mapped as anonymous

						 * memory, low bit is set, and

						 * it points to anon_vma object:

						 * see PAGE_MAPPING_ANON below.

						 */

	    };

#if USE_SPLIT_PTLOCKS

		spinlock_t ptl;

#endif

		struct kmem_cache *slab;	/* SLUB: Pointer to slab */

		struct page *first_page;	/* Compound tail pages */

	};

	union {

		pgoff_t index;		/* Our offset within mapping. */

		void *freelist;		/* SLUB: freelist req. slab lock */

	};

	struct list_head lru;		/* Pageout list, eg. active_list

					 * protected by zone->lru_lock !

					 */



	/*

	 * On machines where all RAM is mapped into kernel address space,

	 * we can simply calculate the virtual address. On machines with
@@ -100,7 +124,16 @@ struct page { 
	 */

	void *shadow;

#endif

};

}

/*

 * If another subsystem starts using the double word pairing for atomic

 * operations on struct page then it must change the #if to ensure

 * proper alignment of the page struct.

 */

#if defined(CONFIG_SLUB) && defined(CONFIG_CMPXCHG_LOCAL)

	__attribute__((__aligned__(2*sizeof(unsigned long))))

#endif

;



typedef unsigned long __nocast vm_flags_t;

include/linux/page-flags.h

+0 −5
Original line number Diff line number Diff line
@@ -124,9 +124,6 @@ enum pageflags { 


	/* SLOB */

	PG_slob_free = PG_private,



	/* SLUB */

	PG_slub_frozen = PG_active,

};



#ifndef __GENERATING_BOUNDS_H
@@ -212,8 +209,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) 


__PAGEFLAG(SlobFree, slob_free)



__PAGEFLAG(SlubFrozen, slub_frozen)



/*

 * Private page markings that may be used by the filesystem that owns the page

 * for its own purposes.

include/linux/slub_def.h

+3 −0
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ enum stat_item { 
	ALLOC_FROM_PARTIAL,	/* Cpu slab acquired from partial list */

	ALLOC_SLAB,		/* Cpu slab acquired from page allocator */

	ALLOC_REFILL,		/* Refill cpu slab from slab freelist */

	ALLOC_NODE_MISMATCH,	/* Switching cpu slab */

	FREE_SLAB,		/* Slab freed to the page allocator */

	CPUSLAB_FLUSH,		/* Abandoning of the cpu slab */

	DEACTIVATE_FULL,	/* Cpu slab was full when deactivated */
@@ -31,8 +32,10 @@ enum stat_item { 
	DEACTIVATE_TO_HEAD,	/* Cpu slab was moved to the head of partials */

	DEACTIVATE_TO_TAIL,	/* Cpu slab was moved to the tail of partials */

	DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */

	DEACTIVATE_BYPASS,	/* Implicit deactivation */

	ORDER_FALLBACK,		/* Number of times fallback was necessary */

	CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */

	CMPXCHG_DOUBLE_FAIL,	/* Number of times that cmpxchg double did not match */

	NR_SLUB_STAT_ITEMS };



struct kmem_cache_cpu {

mm/slub.c

+512 −252

File changed.

Preview size limit exceeded, changes collapsed.

tools/slub/slabinfo.c

+40 −19
Original line number Diff line number Diff line
@@ -2,8 +2,9 @@ 
 * Slabinfo: Tool to get reports about slabs

 *

 * (C) 2007 sgi, Christoph Lameter

 * (C) 2011 Linux Foundation, Christoph Lameter

 *

 * Compile by:

 * Compile with:

 *

 * gcc -o slabinfo slabinfo.c

 */
@@ -39,6 +40,8 @@ struct slabinfo { 
	unsigned long cpuslab_flush, deactivate_full, deactivate_empty;

	unsigned long deactivate_to_head, deactivate_to_tail;

	unsigned long deactivate_remote_frees, order_fallback;

	unsigned long cmpxchg_double_cpu_fail, cmpxchg_double_fail;

	unsigned long alloc_node_mismatch, deactivate_bypass;

	int numa[MAX_NODES];

	int numa_partial[MAX_NODES];

} slabinfo[MAX_SLABS];
@@ -99,7 +102,7 @@ static void fatal(const char *x, ...) 


static void usage(void)

{

	printf("slabinfo 5/7/2007. (c) 2007 sgi.\n\n"

	printf("slabinfo 4/15/2011. (c) 2007 sgi/(c) 2011 Linux Foundation.\n\n"

		"slabinfo [-ahnpvtsz] [-d debugopts] [slab-regexp]\n"

		"-a|--aliases           Show aliases\n"

		"-A|--activity          Most active slabs first\n"
@@ -293,7 +296,7 @@ int line = 0; 
static void first_line(void)

{

	if (show_activity)

		printf("Name                   Objects      Alloc       Free   %%Fast Fallb O\n");

		printf("Name                   Objects      Alloc       Free   %%Fast Fallb O CmpX   UL\n");

	else

		printf("Name                   Objects Objsize    Space "

			"Slabs/Part/Cpu  O/S O %%Fr %%Ef Flg\n");
@@ -379,14 +382,14 @@ static void show_tracking(struct slabinfo *s) 
	printf("\n%s: Kernel object allocation\n", s->name);

	printf("-----------------------------------------------------------------------\n");

	if (read_slab_obj(s, "alloc_calls"))

		printf(buffer);

		printf("%s", buffer);

	else

		printf("No Data\n");



	printf("\n%s: Kernel object freeing\n", s->name);

	printf("------------------------------------------------------------------------\n");

	if (read_slab_obj(s, "free_calls"))

		printf(buffer);

		printf("%s", buffer);

	else

		printf("No Data\n");
@@ -400,7 +403,7 @@ static void ops(struct slabinfo *s) 
	if (read_slab_obj(s, "ops")) {

		printf("\n%s: kmem_cache operations\n", s->name);

		printf("--------------------------------------------\n");

		printf(buffer);

		printf("%s", buffer);

	} else

		printf("\n%s has no kmem_cache operations\n", s->name);

}
@@ -462,19 +465,32 @@ static void slab_stats(struct slabinfo *s) 
	if (s->cpuslab_flush)

		printf("Flushes %8lu\n", s->cpuslab_flush);



	if (s->alloc_refill)

		printf("Refill %8lu\n", s->alloc_refill);



	total = s->deactivate_full + s->deactivate_empty +

			s->deactivate_to_head + s->deactivate_to_tail;



	if (total)

		printf("Deactivate Full=%lu(%lu%%) Empty=%lu(%lu%%) "

			"ToHead=%lu(%lu%%) ToTail=%lu(%lu%%)\n",

			s->deactivate_full, (s->deactivate_full * 100) / total,

			s->deactivate_empty, (s->deactivate_empty * 100) / total,

			s->deactivate_to_head, (s->deactivate_to_head * 100) / total,

			s->deactivate_to_head + s->deactivate_to_tail + s->deactivate_bypass;



	if (total) {

		printf("\nSlab Deactivation             Ocurrences  %%\n");

		printf("-------------------------------------------------\n");

		printf("Slab full                     %7lu  %3lu%%\n",

			s->deactivate_full, (s->deactivate_full * 100) / total);

		printf("Slab empty                    %7lu  %3lu%%\n",

			s->deactivate_empty, (s->deactivate_empty * 100) / total);

		printf("Moved to head of partial list %7lu  %3lu%%\n",

			s->deactivate_to_head, (s->deactivate_to_head * 100) / total);

		printf("Moved to tail of partial list %7lu  %3lu%%\n",

			s->deactivate_to_tail, (s->deactivate_to_tail * 100) / total);

		printf("Deactivation bypass           %7lu  %3lu%%\n",

			s->deactivate_bypass, (s->deactivate_bypass * 100) / total);

		printf("Refilled from foreign frees   %7lu  %3lu%%\n",

			s->alloc_refill, (s->alloc_refill * 100) / total);

		printf("Node mismatch                 %7lu  %3lu%%\n",

			s->alloc_node_mismatch, (s->alloc_node_mismatch * 100) / total);

	}



	if (s->cmpxchg_double_fail || s->cmpxchg_double_cpu_fail)

		printf("\nCmpxchg_double Looping\n------------------------\n");

		printf("Locked Cmpxchg Double redos   %lu\nUnlocked Cmpxchg Double redos %lu\n",

			s->cmpxchg_double_fail, s->cmpxchg_double_cpu_fail);

}



static void report(struct slabinfo *s)
@@ -573,12 +589,13 @@ static void slabcache(struct slabinfo *s) 
		total_alloc = s->alloc_fastpath + s->alloc_slowpath;

		total_free = s->free_fastpath + s->free_slowpath;



		printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d\n",

		printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d %4ld %4ld\n",

			s->name, s->objects,

			total_alloc, total_free,

			total_alloc ? (s->alloc_fastpath * 100 / total_alloc) : 0,

			total_free ? (s->free_fastpath * 100 / total_free) : 0,

			s->order_fallback, s->order);

			s->order_fallback, s->order, s->cmpxchg_double_fail,

			s->cmpxchg_double_cpu_fail);

	}

	else

		printf("%-21s %8ld %7d %8s %14s %4d %1d %3ld %3ld %s\n",
@@ -1190,6 +1207,10 @@ static void read_slab_dir(void) 
			slab->deactivate_to_tail = get_obj("deactivate_to_tail");

			slab->deactivate_remote_frees = get_obj("deactivate_remote_frees");

			slab->order_fallback = get_obj("order_fallback");

			slab->cmpxchg_double_cpu_fail = get_obj("cmpxchg_double_cpu_fail");

			slab->cmpxchg_double_fail = get_obj("cmpxchg_double_fail");

			slab->alloc_node_mismatch = get_obj("alloc_node_mismatch");

			slab->deactivate_bypass = get_obj("deactivate_bypass");

			chdir("..");

			if (slab->name[0] == ':')

				alias_targets++;