Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cafb168a authored by Steven Rostedt's avatar Steven Rostedt
Browse files

tracing: make the function profiler per cpu



Impact: speed enhancement

By making the function profiler record in per cpu data we not only
get better readings, avoid races, we also do not have to take any
locks.

Signed-off-by: default avatarSteven Rostedt <srostedt@redhat.com>
parent 0706f1c4
Loading
Loading
Loading
Loading
+130 −69
Original line number Original line Diff line number Diff line
@@ -257,28 +257,28 @@ struct ftrace_profile_page {
	struct ftrace_profile		records[];
	struct ftrace_profile		records[];
};
};


struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

#define PROFILE_RECORDS_SIZE						\
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))


#define PROFILES_PER_PAGE					\
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))


/* TODO: make these percpu, to prevent cache line bouncing */
static struct ftrace_profile_page *profile_pages_start;
static struct ftrace_profile_page *profile_pages;

static struct hlist_head *ftrace_profile_hash;
static int ftrace_profile_bits;
static int ftrace_profile_bits;
static int ftrace_profile_enabled;
static int ftrace_profile_enabled;
static DEFINE_MUTEX(ftrace_profile_lock);
static DEFINE_MUTEX(ftrace_profile_lock);


static DEFINE_PER_CPU(atomic_t, ftrace_profile_disable);
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);


#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */


static raw_spinlock_t ftrace_profile_rec_lock =
	(raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;

static void *
static void *
function_stat_next(void *v, int idx)
function_stat_next(void *v, int idx)
{
{
@@ -303,7 +303,13 @@ function_stat_next(void *v, int idx)


static void *function_stat_start(struct tracer_stat *trace)
static void *function_stat_start(struct tracer_stat *trace)
{
{
	return function_stat_next(&profile_pages_start->records[0], 0);
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
}
}


#ifdef CONFIG_FUNCTION_GRAPH_TRACER
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -374,20 +380,11 @@ static int function_stat_show(struct seq_file *m, void *v)
	return 0;
	return 0;
}
}


static struct tracer_stat function_stats = {
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
	.name = "functions",
	.stat_start = function_stat_start,
	.stat_next = function_stat_next,
	.stat_cmp = function_stat_cmp,
	.stat_headers = function_stat_headers,
	.stat_show = function_stat_show
};

static void ftrace_profile_reset(void)
{
{
	struct ftrace_profile_page *pg;
	struct ftrace_profile_page *pg;


	pg = profile_pages = profile_pages_start;
	pg = stat->pages = stat->start;


	while (pg) {
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
@@ -395,24 +392,24 @@ static void ftrace_profile_reset(void)
		pg = pg->next;
		pg = pg->next;
	}
	}


	memset(ftrace_profile_hash, 0,
	memset(stat->hash, 0,
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
}


int ftrace_profile_pages_init(void)
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
{
{
	struct ftrace_profile_page *pg;
	struct ftrace_profile_page *pg;
	int i;
	int i;


	/* If we already allocated, do nothing */
	/* If we already allocated, do nothing */
	if (profile_pages)
	if (stat->pages)
		return 0;
		return 0;


	profile_pages = (void *)get_zeroed_page(GFP_KERNEL);
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!profile_pages)
	if (!stat->pages)
		return -ENOMEM;
		return -ENOMEM;


	pg = profile_pages_start = profile_pages;
	pg = stat->start = stat->pages;


	/* allocate 10 more pages to start */
	/* allocate 10 more pages to start */
	for (i = 0; i < 10; i++) {
	for (i = 0; i < 10; i++) {
@@ -430,13 +427,16 @@ int ftrace_profile_pages_init(void)
	return 0;
	return 0;
}
}


static int ftrace_profile_init(void)
static int ftrace_profile_init_cpu(int cpu)
{
{
	struct ftrace_profile_stat *stat;
	int size;
	int size;


	if (ftrace_profile_hash) {
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
		/* If the profile is already created, simply reset it */
		/* If the profile is already created, simply reset it */
		ftrace_profile_reset();
		ftrace_profile_reset(stat);
		return 0;
		return 0;
	}
	}


@@ -446,29 +446,45 @@ static int ftrace_profile_init(void)
	 */
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
	size = FTRACE_PROFILE_HASH_SIZE;


	ftrace_profile_hash =
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
		kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);


	if (!ftrace_profile_hash)
	if (!stat->hash)
		return -ENOMEM;
		return -ENOMEM;


	if (!ftrace_profile_bits) {
		size--;
		size--;


		for (; size; size >>= 1)
		for (; size; size >>= 1)
			ftrace_profile_bits++;
			ftrace_profile_bits++;
	}


	/* Preallocate a few pages */
	/* Preallocate a few pages */
	if (ftrace_profile_pages_init() < 0) {
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(ftrace_profile_hash);
		kfree(stat->hash);
		ftrace_profile_hash = NULL;
		stat->hash = NULL;
		return -ENOMEM;
		return -ENOMEM;
	}
	}


	return 0;
	return 0;
}
}


static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

	for_each_online_cpu(cpu) {
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

/* interrupts must be disabled */
/* interrupts must be disabled */
static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
{
{
	struct ftrace_profile *rec;
	struct ftrace_profile *rec;
	struct hlist_head *hhd;
	struct hlist_head *hhd;
@@ -476,7 +492,7 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
	unsigned long key;
	unsigned long key;


	key = hash_long(ip, ftrace_profile_bits);
	key = hash_long(ip, ftrace_profile_bits);
	hhd = &ftrace_profile_hash[key];
	hhd = &stat->hash[key];


	if (hlist_empty(hhd))
	if (hlist_empty(hhd))
		return NULL;
		return NULL;
@@ -489,52 +505,50 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
	return NULL;
	return NULL;
}
}


static void ftrace_add_profile(struct ftrace_profile *rec)
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
{
{
	unsigned long key;
	unsigned long key;


	key = hash_long(rec->ip, ftrace_profile_bits);
	key = hash_long(rec->ip, ftrace_profile_bits);
	hlist_add_head_rcu(&rec->node, &ftrace_profile_hash[key]);
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
}
}


/* Interrupts must be disabled calling this */
/* Interrupts must be disabled calling this */
static struct ftrace_profile *
static struct ftrace_profile *
ftrace_profile_alloc(unsigned long ip, bool alloc_safe)
ftrace_profile_alloc(struct ftrace_profile_stat *stat,
		     unsigned long ip, bool alloc_safe)
{
{
	struct ftrace_profile *rec = NULL;
	struct ftrace_profile *rec = NULL;


	/* prevent recursion */
	/* prevent recursion */
	if (atomic_inc_return(&__get_cpu_var(ftrace_profile_disable)) != 1)
	if (atomic_inc_return(&stat->disabled) != 1)
		goto out;
		goto out;


	__raw_spin_lock(&ftrace_profile_rec_lock);

	/* Try to always keep another page available */
	/* Try to always keep another page available */
	if (!profile_pages->next && alloc_safe)
	if (!stat->pages->next && alloc_safe)
		profile_pages->next = (void *)get_zeroed_page(GFP_ATOMIC);
		stat->pages->next = (void *)get_zeroed_page(GFP_ATOMIC);


	/*
	/*
	 * Try to find the function again since another
	 * Try to find the function again since another
	 * task on another CPU could have added it
	 * task on another CPU could have added it
	 */
	 */
	rec = ftrace_find_profiled_func(ip);
	rec = ftrace_find_profiled_func(stat, ip);
	if (rec)
	if (rec)
		goto out_unlock;
		goto out;


	if (profile_pages->index == PROFILES_PER_PAGE) {
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!profile_pages->next)
		if (!stat->pages->next)
			goto out_unlock;
			goto out;
		profile_pages = profile_pages->next;
		stat->pages = stat->pages->next;
	}
	}


	rec = &profile_pages->records[profile_pages->index++];
	rec = &stat->pages->records[stat->pages->index++];
	rec->ip = ip;
	rec->ip = ip;
	ftrace_add_profile(rec);
	ftrace_add_profile(stat, rec);


 out_unlock:
	__raw_spin_unlock(&ftrace_profile_rec_lock);
 out:
 out:
	atomic_dec(&__get_cpu_var(ftrace_profile_disable));
	atomic_dec(&stat->disabled);


	return rec;
	return rec;
}
}
@@ -552,6 +566,7 @@ static bool ftrace_safe_to_allocate(void)
static void
static void
function_profile_call(unsigned long ip, unsigned long parent_ip)
function_profile_call(unsigned long ip, unsigned long parent_ip)
{
{
	struct ftrace_profile_stat *stat;
	struct ftrace_profile *rec;
	struct ftrace_profile *rec;
	unsigned long flags;
	unsigned long flags;
	bool alloc_safe;
	bool alloc_safe;
@@ -562,9 +577,14 @@ function_profile_call(unsigned long ip, unsigned long parent_ip)
	alloc_safe = ftrace_safe_to_allocate();
	alloc_safe = ftrace_safe_to_allocate();


	local_irq_save(flags);
	local_irq_save(flags);
	rec = ftrace_find_profiled_func(ip);

	stat = &__get_cpu_var(ftrace_profile_stats);
	if (!stat->hash)
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
	if (!rec) {
	if (!rec) {
		rec = ftrace_profile_alloc(ip, alloc_safe);
		rec = ftrace_profile_alloc(stat, ip, alloc_safe);
		if (!rec)
		if (!rec)
			goto out;
			goto out;
	}
	}
@@ -583,13 +603,19 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace)


static void profile_graph_return(struct ftrace_graph_ret *trace)
static void profile_graph_return(struct ftrace_graph_ret *trace)
{
{
	unsigned long flags;
	struct ftrace_profile_stat *stat;
	struct ftrace_profile *rec;
	struct ftrace_profile *rec;
	unsigned long flags;


	local_irq_save(flags);
	local_irq_save(flags);
	rec = ftrace_find_profiled_func(trace->func);
	stat = &__get_cpu_var(ftrace_profile_stats);
	if (!stat->hash)
		goto out;

	rec = ftrace_find_profiled_func(stat, trace->func);
	if (rec)
	if (rec)
		rec->time += trace->rettime - trace->calltime;
		rec->time += trace->rettime - trace->calltime;
 out:
	local_irq_restore(flags);
	local_irq_restore(flags);
}
}


@@ -687,17 +713,52 @@ static const struct file_operations ftrace_profile_fops = {
	.write		= ftrace_profile_write,
	.write		= ftrace_profile_write,
};
};


/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
	.name = "functions",
	.stat_start = function_stat_start,
	.stat_next = function_stat_next,
	.stat_cmp = function_stat_cmp,
	.stat_headers = function_stat_headers,
	.stat_show = function_stat_show
};

static void ftrace_profile_debugfs(struct dentry *d_tracer)
static void ftrace_profile_debugfs(struct dentry *d_tracer)
{
{
	struct ftrace_profile_stat *stat;
	struct dentry *entry;
	struct dentry *entry;
	char *name;
	int ret;
	int ret;
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);


	ret = register_stat_tracer(&function_stats);
		/* allocate enough for function name + cpu number */
		name = kmalloc(32, GFP_KERNEL);
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			kfree(stat);
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		snprintf(name, 32, "function%d", cpu);
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
		if (ret) {
		pr_warning("Warning: could not register "
			WARN(1,
			   "function stats\n");
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
			return;
		}
		}
	}


	entry = debugfs_create_file("function_profile_enabled", 0644,
	entry = debugfs_create_file("function_profile_enabled", 0644,
				    d_tracer, NULL, &ftrace_profile_fops);
				    d_tracer, NULL, &ftrace_profile_fops);