Merge tag 'perf-core-for-mingo' of...

Specify number of groups

-l::

--loop=::

--nr_loops=::

Specify number of loops

Example of *messaging*

Options of *memcpy*

^^^^^^^^^^^^^^^^^^^

-l::

--length::

Specify length of memory to copy (default: 1MB).

--size::

Specify size of memory to copy (default: 1MB).

Available units are B, KB, MB, GB and TB (case insensitive).

-r::

--routine::

Specify routine to copy (default: default).

Available routines are depend on the architecture.

-f::

--function::

Specify function to copy (default: default).

Available functions are depend on the architecture.

On x86-64, x86-64-unrolled, x86-64-movsq and x86-64-movsb are supported.

-i::

--iterations::

-l::

--nr_loops::

Repeat memcpy invocation this number of times.

-c::

--cycle::

--cycles::

Use perf's cpu-cycles event instead of gettimeofday syscall.

-o::

--only-prefault::

Show only the result with page faults before memcpy.

-n::

--no-prefault::

Show only the result without page faults before memcpy.

*memset*::

Suite for evaluating performance of simple memory set in various ways.

Options of *memset*

^^^^^^^^^^^^^^^^^^^

-l::

--length::

Specify length of memory to set (default: 1MB).

--size::

Specify size of memory to set (default: 1MB).

Available units are B, KB, MB, GB and TB (case insensitive).

-r::

--routine::

Specify routine to set (default: default).

Available routines are depend on the architecture.

-f::

--function::

Specify function to set (default: default).

Available functions are depend on the architecture.

On x86-64, x86-64-unrolled, x86-64-stosq and x86-64-stosb are supported.

-i::

--iterations::

-l::

--nr_loops::

Repeat memset invocation this number of times.

-c::

--cycle::

--cycles::

Use perf's cpu-cycles event instead of gettimeofday syscall.

-o::

--only-prefault::

Show only the result with page faults before memset.

-n::

--no-prefault::

Show only the result without page faults before memset.

SUITES FOR 'numa'

~~~~~~~~~~~~~~~~~

*mem*::

perf-y += sched-messaging.o

perf-y += sched-pipe.o

perf-y += mem-memcpy.o

perf-y += mem-functions.o

perf-y += futex-hash.o

perf-y += futex-wake.o

perf-y += futex-wake-parallel.o

/*

 * mem-memcpy.c

 *

 * Simple memcpy() and memset() benchmarks

 *

 * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp>

 */

#include "../perf.h"

#include "../util/util.h"

#include "../util/parse-options.h"

#include "../util/header.h"

#include "../util/cloexec.h"

#include "bench.h"

#include "mem-memcpy-arch.h"

#include "mem-memset-arch.h"

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <sys/time.h>

#include <errno.h>

#define K 1024

static const char	*size_str	= "1MB";

static const char	*function_str	= "all";

static int		nr_loops	= 1;

static bool		use_cycles;

static int		cycles_fd;

static const struct option options[] = {

	OPT_STRING('s', "size", &size_str, "1MB",

		    "Specify the size of the memory buffers. "

		    "Available units: B, KB, MB, GB and TB (case insensitive)"),

	OPT_STRING('f', "function", &function_str, "all",

		    "Specify the function to run, \"all\" runs all available functions, \"help\" lists them"),

	OPT_INTEGER('l', "nr_loops", &nr_loops,

		    "Specify the number of loops to run. (default: 1)"),

	OPT_BOOLEAN('c', "cycles", &use_cycles,

		    "Use a cycles event instead of gettimeofday() to measure performance"),

	OPT_END()

};

typedef void *(*memcpy_t)(void *, const void *, size_t);

typedef void *(*memset_t)(void *, int, size_t);

struct function {

	const char *name;

	const char *desc;

	union {

		memcpy_t memcpy;

		memset_t memset;

	} fn;

};

static struct perf_event_attr cycle_attr = {

	.type		= PERF_TYPE_HARDWARE,

	.config		= PERF_COUNT_HW_CPU_CYCLES

};

static void init_cycles(void)

{

	cycles_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1, perf_event_open_cloexec_flag());

	if (cycles_fd < 0 && errno == ENOSYS)

		die("No CONFIG_PERF_EVENTS=y kernel support configured?\n");

	else

		BUG_ON(cycles_fd < 0);

}

static u64 get_cycles(void)

{

	int ret;

	u64 clk;

	ret = read(cycles_fd, &clk, sizeof(u64));

	BUG_ON(ret != sizeof(u64));

	return clk;

}

static double timeval2double(struct timeval *ts)

{

	return (double)ts->tv_sec + (double)ts->tv_usec / (double)1000000;

}

#define print_bps(x) do {						\

		if (x < K)						\

			printf(" %14lf bytes/sec\n", x);		\

		else if (x < K * K)					\

			printf(" %14lfd KB/sec\n", x / K);		\

		else if (x < K * K * K)					\

			printf(" %14lf MB/sec\n", x / K / K);		\

		else							\

			printf(" %14lf GB/sec\n", x / K / K / K);	\

	} while (0)

struct bench_mem_info {

	const struct function *functions;

	u64 (*do_cycles)(const struct function *r, size_t size);

	double (*do_gettimeofday)(const struct function *r, size_t size);

	const char *const *usage;

};

static void __bench_mem_function(struct bench_mem_info *info, int r_idx, size_t size, double size_total)

{

	const struct function *r = &info->functions[r_idx];

	double result_bps = 0.0;

	u64 result_cycles = 0;

	printf("# function '%s' (%s)\n", r->name, r->desc);

	if (bench_format == BENCH_FORMAT_DEFAULT)

		printf("# Copying %s bytes ...\n\n", size_str);

	if (use_cycles) {

		result_cycles = info->do_cycles(r, size);

	} else {

		result_bps = info->do_gettimeofday(r, size);

	}

	switch (bench_format) {

	case BENCH_FORMAT_DEFAULT:

		if (use_cycles) {

			printf(" %14lf cycles/byte\n", (double)result_cycles/size_total);

		} else {

			print_bps(result_bps);

		}

		break;

	case BENCH_FORMAT_SIMPLE:

		if (use_cycles) {

			printf("%lf\n", (double)result_cycles/size_total);

		} else {

			printf("%lf\n", result_bps);

		}

		break;

	default:

		BUG_ON(1);

		break;

	}

}

static int bench_mem_common(int argc, const char **argv, struct bench_mem_info *info)

{

	int i;

	size_t size;

	double size_total;

	argc = parse_options(argc, argv, options, info->usage, 0);

	if (use_cycles)

		init_cycles();

	size = (size_t)perf_atoll((char *)size_str);

	size_total = (double)size * nr_loops;

	if ((s64)size <= 0) {

		fprintf(stderr, "Invalid size:%s\n", size_str);

		return 1;

	}

	if (!strncmp(function_str, "all", 3)) {

		for (i = 0; info->functions[i].name; i++)

			__bench_mem_function(info, i, size, size_total);

		return 0;

	}

	for (i = 0; info->functions[i].name; i++) {

		if (!strcmp(info->functions[i].name, function_str))

			break;

	}

	if (!info->functions[i].name) {

		if (strcmp(function_str, "help") && strcmp(function_str, "h"))

			printf("Unknown function: %s\n", function_str);

		printf("Available functions:\n");

		for (i = 0; info->functions[i].name; i++) {

			printf("\t%s ... %s\n",

			       info->functions[i].name, info->functions[i].desc);

		}

		return 1;

	}

	__bench_mem_function(info, i, size, size_total);

	return 0;

}

static void memcpy_alloc_mem(void **dst, void **src, size_t size)

{

	*dst = zalloc(size);

	if (!*dst)

		die("memory allocation failed - maybe size is too large?\n");

	*src = zalloc(size);

	if (!*src)

		die("memory allocation failed - maybe size is too large?\n");

	/* Make sure to always prefault zero pages even if MMAP_THRESH is crossed: */

	memset(*src, 0, size);

}

static u64 do_memcpy_cycles(const struct function *r, size_t size)

{

	u64 cycle_start = 0ULL, cycle_end = 0ULL;

	void *src = NULL, *dst = NULL;

	memcpy_t fn = r->fn.memcpy;

	int i;

	memcpy_alloc_mem(&dst, &src, size);

	/*

	 * We prefault the freshly allocated memory range here,

	 * to not measure page fault overhead:

	 */

	fn(dst, src, size);

	cycle_start = get_cycles();

	for (i = 0; i < nr_loops; ++i)

		fn(dst, src, size);

	cycle_end = get_cycles();

	free(src);

	free(dst);

	return cycle_end - cycle_start;

}

static double do_memcpy_gettimeofday(const struct function *r, size_t size)

{

	struct timeval tv_start, tv_end, tv_diff;

	memcpy_t fn = r->fn.memcpy;

	void *src = NULL, *dst = NULL;

	int i;

	memcpy_alloc_mem(&dst, &src, size);

	/*

	 * We prefault the freshly allocated memory range here,

	 * to not measure page fault overhead:

	 */

	fn(dst, src, size);

	BUG_ON(gettimeofday(&tv_start, NULL));

	for (i = 0; i < nr_loops; ++i)

		fn(dst, src, size);

	BUG_ON(gettimeofday(&tv_end, NULL));

	timersub(&tv_end, &tv_start, &tv_diff);

	free(src);

	free(dst);

	return (double)(((double)size * nr_loops) / timeval2double(&tv_diff));

}

struct function memcpy_functions[] = {

	{ .name		= "default",

	  .desc		= "Default memcpy() provided by glibc",

	  .fn.memcpy	= memcpy },

#ifdef HAVE_ARCH_X86_64_SUPPORT

# define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},

# include "mem-memcpy-x86-64-asm-def.h"

# undef MEMCPY_FN

#endif

	{ .name = NULL, }

};

static const char * const bench_mem_memcpy_usage[] = {

	"perf bench mem memcpy <options>",

	NULL

};

int bench_mem_memcpy(int argc, const char **argv, const char *prefix __maybe_unused)

{

	struct bench_mem_info info = {

		.functions		= memcpy_functions,

		.do_cycles		= do_memcpy_cycles,

		.do_gettimeofday	= do_memcpy_gettimeofday,

		.usage			= bench_mem_memcpy_usage,

	};

	return bench_mem_common(argc, argv, &info);

}

static void memset_alloc_mem(void **dst, size_t size)

{

	*dst = zalloc(size);

	if (!*dst)

		die("memory allocation failed - maybe size is too large?\n");

}

static u64 do_memset_cycles(const struct function *r, size_t size)

{

	u64 cycle_start = 0ULL, cycle_end = 0ULL;

	memset_t fn = r->fn.memset;

	void *dst = NULL;

	int i;

	memset_alloc_mem(&dst, size);

	/*

	 * We prefault the freshly allocated memory range here,

	 * to not measure page fault overhead:

	 */

	fn(dst, -1, size);

	cycle_start = get_cycles();

	for (i = 0; i < nr_loops; ++i)

		fn(dst, i, size);

	cycle_end = get_cycles();

	free(dst);

	return cycle_end - cycle_start;

}

static double do_memset_gettimeofday(const struct function *r, size_t size)

{

	struct timeval tv_start, tv_end, tv_diff;

	memset_t fn = r->fn.memset;

	void *dst = NULL;

	int i;

	memset_alloc_mem(&dst, size);

	/*

	 * We prefault the freshly allocated memory range here,

	 * to not measure page fault overhead:

	 */

	fn(dst, -1, size);

	BUG_ON(gettimeofday(&tv_start, NULL));

	for (i = 0; i < nr_loops; ++i)

		fn(dst, i, size);

	BUG_ON(gettimeofday(&tv_end, NULL));

	timersub(&tv_end, &tv_start, &tv_diff);

	free(dst);

	return (double)(((double)size * nr_loops) / timeval2double(&tv_diff));

}

static const char * const bench_mem_memset_usage[] = {

	"perf bench mem memset <options>",

	NULL

};

static const struct function memset_functions[] = {

	{ .name		= "default",

	  .desc		= "Default memset() provided by glibc",

	  .fn.memset	= memset },

#ifdef HAVE_ARCH_X86_64_SUPPORT

# define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn },

# include "mem-memset-x86-64-asm-def.h"

# undef MEMSET_FN

#endif

	{ .name = NULL, }

};

int bench_mem_memset(int argc, const char **argv, const char *prefix __maybe_unused)

{

	struct bench_mem_info info = {

		.functions		= memset_functions,

		.do_cycles		= do_memset_cycles,

		.do_gettimeofday	= do_memset_gettimeofday,

		.usage			= bench_mem_memset_usage,

	};

	return bench_mem_common(argc, argv, &info);

}

/*

 * mem-memcpy.c

 *

 * memcpy: Simple memory copy in various ways

 *

 * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp>

 */

#include "../perf.h"

#include "../util/util.h"

#include "../util/parse-options.h"

#include "../util/header.h"

#include "../util/cloexec.h"

#include "bench.h"

#include "mem-memcpy-arch.h"

#include "mem-memset-arch.h"

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <sys/time.h>

#include <errno.h>

#define K 1024

static const char	*length_str	= "1MB";

static const char	*routine	= "default";

static int		iterations	= 1;

static bool		use_cycle;

static int		cycle_fd;

static bool		only_prefault;

static bool		no_prefault;

static const struct option options[] = {

	OPT_STRING('l', "length", &length_str, "1MB",

		    "Specify length of memory to copy. "

		    "Available units: B, KB, MB, GB and TB (upper and lower)"),

	OPT_STRING('r', "routine", &routine, "default",

		    "Specify routine to copy, \"all\" runs all available routines"),

	OPT_INTEGER('i', "iterations", &iterations,

		    "repeat memcpy() invocation this number of times"),

	OPT_BOOLEAN('c', "cycle", &use_cycle,

		    "Use cycles event instead of gettimeofday() for measuring"),

	OPT_BOOLEAN('o', "only-prefault", &only_prefault,

		    "Show only the result with page faults before memcpy()"),

	OPT_BOOLEAN('n', "no-prefault", &no_prefault,

		    "Show only the result without page faults before memcpy()"),

	OPT_END()

};

typedef void *(*memcpy_t)(void *, const void *, size_t);

typedef void *(*memset_t)(void *, int, size_t);

struct routine {

	const char *name;

	const char *desc;

	union {

		memcpy_t memcpy;

		memset_t memset;

	} fn;

};

struct routine memcpy_routines[] = {

	{ .name = "default",

	  .desc = "Default memcpy() provided by glibc",

	  .fn.memcpy = memcpy },

#ifdef HAVE_ARCH_X86_64_SUPPORT

#define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},

#include "mem-memcpy-x86-64-asm-def.h"

#undef MEMCPY_FN

#endif

	{ NULL,

	  NULL,

	  {NULL}   }

};

static const char * const bench_mem_memcpy_usage[] = {

	"perf bench mem memcpy <options>",

	NULL

};

static struct perf_event_attr cycle_attr = {

	.type		= PERF_TYPE_HARDWARE,

	.config		= PERF_COUNT_HW_CPU_CYCLES

};

static void init_cycle(void)

{

	cycle_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1,

				       perf_event_open_cloexec_flag());

	if (cycle_fd < 0 && errno == ENOSYS)

		die("No CONFIG_PERF_EVENTS=y kernel support configured?\n");

	else

		BUG_ON(cycle_fd < 0);

}

static u64 get_cycle(void)

{

	int ret;

	u64 clk;

	ret = read(cycle_fd, &clk, sizeof(u64));

	BUG_ON(ret != sizeof(u64));

	return clk;

}

static double timeval2double(struct timeval *ts)

{

	return (double)ts->tv_sec +

		(double)ts->tv_usec / (double)1000000;

}

#define pf (no_prefault ? 0 : 1)

#define print_bps(x) do {					\

		if (x < K)					\

			printf(" %14lf B/Sec", x);		\

		else if (x < K * K)				\

			printf(" %14lfd KB/Sec", x / K);	\

		else if (x < K * K * K)				\

			printf(" %14lf MB/Sec", x / K / K);	\

		else						\

			printf(" %14lf GB/Sec", x / K / K / K); \

	} while (0)

struct bench_mem_info {

	const struct routine *routines;

	u64 (*do_cycle)(const struct routine *r, size_t len, bool prefault);

	double (*do_gettimeofday)(const struct routine *r, size_t len, bool prefault);

	const char *const *usage;

};

static void __bench_mem_routine(struct bench_mem_info *info, int r_idx, size_t len, double totallen)

{

	const struct routine *r = &info->routines[r_idx];

	double result_bps[2];

	u64 result_cycle[2];

	result_cycle[0] = result_cycle[1] = 0ULL;

	result_bps[0] = result_bps[1] = 0.0;

	printf("Routine %s (%s)\n", r->name, r->desc);

	if (bench_format == BENCH_FORMAT_DEFAULT)

		printf("# Copying %s Bytes ...\n\n", length_str);

	if (!only_prefault && !no_prefault) {

		/* show both of results */

		if (use_cycle) {

			result_cycle[0] = info->do_cycle(r, len, false);

			result_cycle[1] = info->do_cycle(r, len, true);

		} else {

			result_bps[0]   = info->do_gettimeofday(r, len, false);

			result_bps[1]   = info->do_gettimeofday(r, len, true);

		}

	} else {

		if (use_cycle)

			result_cycle[pf] = info->do_cycle(r, len, only_prefault);

		else

			result_bps[pf] = info->do_gettimeofday(r, len, only_prefault);

	}

	switch (bench_format) {

	case BENCH_FORMAT_DEFAULT:

		if (!only_prefault && !no_prefault) {

			if (use_cycle) {

				printf(" %14lf Cycle/Byte\n",

					(double)result_cycle[0]

					/ totallen);

				printf(" %14lf Cycle/Byte (with prefault)\n",

					(double)result_cycle[1]

					/ totallen);

			} else {

				print_bps(result_bps[0]);

				printf("\n");

				print_bps(result_bps[1]);

				printf(" (with prefault)\n");

			}

		} else {

			if (use_cycle) {

				printf(" %14lf Cycle/Byte",

					(double)result_cycle[pf]

					/ totallen);

			} else

				print_bps(result_bps[pf]);

			printf("%s\n", only_prefault ? " (with prefault)" : "");

		}

		break;

	case BENCH_FORMAT_SIMPLE:

		if (!only_prefault && !no_prefault) {

			if (use_cycle) {

				printf("%lf %lf\n",

					(double)result_cycle[0] / totallen,

					(double)result_cycle[1] / totallen);

			} else {

				printf("%lf %lf\n",

					result_bps[0], result_bps[1]);

			}

		} else {

			if (use_cycle) {

				printf("%lf\n", (double)result_cycle[pf]

					/ totallen);

			} else

				printf("%lf\n", result_bps[pf]);

		}

		break;

	default:

		/* reaching this means there's some disaster: */

		die("unknown format: %d\n", bench_format);

		break;

	}

}

static int bench_mem_common(int argc, const char **argv,

		     const char *prefix __maybe_unused,

		     struct bench_mem_info *info)

{

	int i;

	size_t len;

	double totallen;

	argc = parse_options(argc, argv, options,

			     info->usage, 0);

	if (no_prefault && only_prefault) {

		fprintf(stderr, "Invalid options: -o and -n are mutually exclusive\n");