tracing: make the function profiler per cpu (cafb168a) · Commits · e / devices / android_kernel_xiaomi_nabu

kernel/trace/ftrace.c

+130 −69

Original line number	Original line	Diff line number	Diff line
	@@ -257,28 +257,28 @@ struct ftrace_profile_page {
	struct ftrace_profile records[];		struct ftrace_profile records[];
	};		};

			struct ftrace_profile_stat {
			atomic_t disabled;
			struct hlist_head *hash;
			struct ftrace_profile_page *pages;
			struct ftrace_profile_page *start;
			struct tracer_stat stat;
			};

	#define PROFILE_RECORDS_SIZE \		#define PROFILE_RECORDS_SIZE \
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))		(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))

	#define PROFILES_PER_PAGE \		#define PROFILES_PER_PAGE \
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))		(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))

	/* TODO: make these percpu, to prevent cache line bouncing */
	static struct ftrace_profile_page *profile_pages_start;
	static struct ftrace_profile_page *profile_pages;

	static struct hlist_head *ftrace_profile_hash;
	static int ftrace_profile_bits;		static int ftrace_profile_bits;
	static int ftrace_profile_enabled;		static int ftrace_profile_enabled;
	static DEFINE_MUTEX(ftrace_profile_lock);		static DEFINE_MUTEX(ftrace_profile_lock);

	static DEFINE_PER_CPU(atomic_t, ftrace_profile_disable);		static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);

	#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */		#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */

	static raw_spinlock_t ftrace_profile_rec_lock =
	(raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;

	static void *		static void *
	function_stat_next(void *v, int idx)		function_stat_next(void *v, int idx)
	{		{
	@@ -303,7 +303,13 @@ function_stat_next(void *v, int idx)

	static void function_stat_start(struct tracer_stat trace)		static void function_stat_start(struct tracer_stat trace)
	{		{
	return function_stat_next(&profile_pages_start->records[0], 0);		struct ftrace_profile_stat *stat =
			container_of(trace, struct ftrace_profile_stat, stat);

			if (!stat \|\| !stat->start)
			return NULL;

			return function_stat_next(&stat->start->records[0], 0);
	}		}

	#ifdef CONFIG_FUNCTION_GRAPH_TRACER		#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	@@ -374,20 +380,11 @@ static int function_stat_show(struct seq_file m, void v)
	return 0;		return 0;
	}		}

	static struct tracer_stat function_stats = {		static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
	.name = "functions",
	.stat_start = function_stat_start,
	.stat_next = function_stat_next,
	.stat_cmp = function_stat_cmp,
	.stat_headers = function_stat_headers,
	.stat_show = function_stat_show
	};

	static void ftrace_profile_reset(void)
	{		{
	struct ftrace_profile_page *pg;		struct ftrace_profile_page *pg;

	pg = profile_pages = profile_pages_start;		pg = stat->pages = stat->start;

	while (pg) {		while (pg) {
	memset(pg->records, 0, PROFILE_RECORDS_SIZE);		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
	@@ -395,24 +392,24 @@ static void ftrace_profile_reset(void)
	pg = pg->next;		pg = pg->next;
	}		}

	memset(ftrace_profile_hash, 0,		memset(stat->hash, 0,
	FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));		FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
	}		}

	int ftrace_profile_pages_init(void)		int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
	{		{
	struct ftrace_profile_page *pg;		struct ftrace_profile_page *pg;
	int i;		int i;

	/* If we already allocated, do nothing */		/* If we already allocated, do nothing */
	if (profile_pages)		if (stat->pages)
	return 0;		return 0;

	profile_pages = (void *)get_zeroed_page(GFP_KERNEL);		stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!profile_pages)		if (!stat->pages)
	return -ENOMEM;		return -ENOMEM;

	pg = profile_pages_start = profile_pages;		pg = stat->start = stat->pages;

	/* allocate 10 more pages to start */		/* allocate 10 more pages to start */
	for (i = 0; i < 10; i++) {		for (i = 0; i < 10; i++) {
	@@ -430,13 +427,16 @@ int ftrace_profile_pages_init(void)
	return 0;		return 0;
	}		}

	static int ftrace_profile_init(void)		static int ftrace_profile_init_cpu(int cpu)
	{		{
			struct ftrace_profile_stat *stat;
	int size;		int size;

	if (ftrace_profile_hash) {		stat = &per_cpu(ftrace_profile_stats, cpu);

			if (stat->hash) {
	/* If the profile is already created, simply reset it */		/* If the profile is already created, simply reset it */
	ftrace_profile_reset();		ftrace_profile_reset(stat);
	return 0;		return 0;
	}		}

	@@ -446,29 +446,45 @@ static int ftrace_profile_init(void)
	*/		*/
	size = FTRACE_PROFILE_HASH_SIZE;		size = FTRACE_PROFILE_HASH_SIZE;

	ftrace_profile_hash =		stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
	kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);

	if (!ftrace_profile_hash)		if (!stat->hash)
	return -ENOMEM;		return -ENOMEM;

			if (!ftrace_profile_bits) {
	size--;		size--;

	for (; size; size >>= 1)		for (; size; size >>= 1)
	ftrace_profile_bits++;		ftrace_profile_bits++;
			}

	/* Preallocate a few pages */		/* Preallocate a few pages */
	if (ftrace_profile_pages_init() < 0) {		if (ftrace_profile_pages_init(stat) < 0) {
	kfree(ftrace_profile_hash);		kfree(stat->hash);
	ftrace_profile_hash = NULL;		stat->hash = NULL;
	return -ENOMEM;		return -ENOMEM;
	}		}

	return 0;		return 0;
	}		}

			static int ftrace_profile_init(void)
			{
			int cpu;
			int ret = 0;

			for_each_online_cpu(cpu) {
			ret = ftrace_profile_init_cpu(cpu);
			if (ret)
			break;
			}

			return ret;
			}

	/* interrupts must be disabled */		/* interrupts must be disabled */
	static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)		static struct ftrace_profile *
			ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
	{		{
	struct ftrace_profile *rec;		struct ftrace_profile *rec;
	struct hlist_head *hhd;		struct hlist_head *hhd;
	@@ -476,7 +492,7 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
	unsigned long key;		unsigned long key;

	key = hash_long(ip, ftrace_profile_bits);		key = hash_long(ip, ftrace_profile_bits);
	hhd = &ftrace_profile_hash[key];		hhd = &stat->hash[key];

	if (hlist_empty(hhd))		if (hlist_empty(hhd))
	return NULL;		return NULL;
	@@ -489,52 +505,50 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
	return NULL;		return NULL;
	}		}

	static void ftrace_add_profile(struct ftrace_profile *rec)		static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			struct ftrace_profile *rec)
	{		{
	unsigned long key;		unsigned long key;

	key = hash_long(rec->ip, ftrace_profile_bits);		key = hash_long(rec->ip, ftrace_profile_bits);
	hlist_add_head_rcu(&rec->node, &ftrace_profile_hash[key]);		hlist_add_head_rcu(&rec->node, &stat->hash[key]);
	}		}

	/* Interrupts must be disabled calling this */		/* Interrupts must be disabled calling this */
	static struct ftrace_profile *		static struct ftrace_profile *
	ftrace_profile_alloc(unsigned long ip, bool alloc_safe)		ftrace_profile_alloc(struct ftrace_profile_stat *stat,
			unsigned long ip, bool alloc_safe)
	{		{
	struct ftrace_profile *rec = NULL;		struct ftrace_profile *rec = NULL;

	/* prevent recursion */		/* prevent recursion */
	if (atomic_inc_return(&__get_cpu_var(ftrace_profile_disable)) != 1)		if (atomic_inc_return(&stat->disabled) != 1)
	goto out;		goto out;

	__raw_spin_lock(&ftrace_profile_rec_lock);

	/* Try to always keep another page available */		/* Try to always keep another page available */
	if (!profile_pages->next && alloc_safe)		if (!stat->pages->next && alloc_safe)
	profile_pages->next = (void *)get_zeroed_page(GFP_ATOMIC);		stat->pages->next = (void *)get_zeroed_page(GFP_ATOMIC);

	/*		/*
	* Try to find the function again since another		* Try to find the function again since another
	* task on another CPU could have added it		* task on another CPU could have added it
	*/		*/
	rec = ftrace_find_profiled_func(ip);		rec = ftrace_find_profiled_func(stat, ip);
	if (rec)		if (rec)
	goto out_unlock;		goto out;

	if (profile_pages->index == PROFILES_PER_PAGE) {		if (stat->pages->index == PROFILES_PER_PAGE) {
	if (!profile_pages->next)		if (!stat->pages->next)
	goto out_unlock;		goto out;
	profile_pages = profile_pages->next;		stat->pages = stat->pages->next;
	}		}

	rec = &profile_pages->records[profile_pages->index++];		rec = &stat->pages->records[stat->pages->index++];
	rec->ip = ip;		rec->ip = ip;
	ftrace_add_profile(rec);		ftrace_add_profile(stat, rec);

	out_unlock:
	__raw_spin_unlock(&ftrace_profile_rec_lock);
	out:		out:
	atomic_dec(&__get_cpu_var(ftrace_profile_disable));		atomic_dec(&stat->disabled);

	return rec;		return rec;
	}		}
	@@ -552,6 +566,7 @@ static bool ftrace_safe_to_allocate(void)
	static void		static void
	function_profile_call(unsigned long ip, unsigned long parent_ip)		function_profile_call(unsigned long ip, unsigned long parent_ip)
	{		{
			struct ftrace_profile_stat *stat;
	struct ftrace_profile *rec;		struct ftrace_profile *rec;
	unsigned long flags;		unsigned long flags;
	bool alloc_safe;		bool alloc_safe;
	@@ -562,9 +577,14 @@ function_profile_call(unsigned long ip, unsigned long parent_ip)
	alloc_safe = ftrace_safe_to_allocate();		alloc_safe = ftrace_safe_to_allocate();

	local_irq_save(flags);		local_irq_save(flags);
	rec = ftrace_find_profiled_func(ip);
			stat = &__get_cpu_var(ftrace_profile_stats);
			if (!stat->hash)
			goto out;

			rec = ftrace_find_profiled_func(stat, ip);
	if (!rec) {		if (!rec) {
	rec = ftrace_profile_alloc(ip, alloc_safe);		rec = ftrace_profile_alloc(stat, ip, alloc_safe);
	if (!rec)		if (!rec)
	goto out;		goto out;
	}		}
	@@ -583,13 +603,19 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace)

	static void profile_graph_return(struct ftrace_graph_ret *trace)		static void profile_graph_return(struct ftrace_graph_ret *trace)
	{		{
	unsigned long flags;		struct ftrace_profile_stat *stat;
	struct ftrace_profile *rec;		struct ftrace_profile *rec;
			unsigned long flags;

	local_irq_save(flags);		local_irq_save(flags);
	rec = ftrace_find_profiled_func(trace->func);		stat = &__get_cpu_var(ftrace_profile_stats);
			if (!stat->hash)
			goto out;

			rec = ftrace_find_profiled_func(stat, trace->func);
	if (rec)		if (rec)
	rec->time += trace->rettime - trace->calltime;		rec->time += trace->rettime - trace->calltime;
			out:
	local_irq_restore(flags);		local_irq_restore(flags);
	}		}

	@@ -687,17 +713,52 @@ static const struct file_operations ftrace_profile_fops = {
	.write = ftrace_profile_write,		.write = ftrace_profile_write,
	};		};

			/* used to initialize the real stat files */
			static struct tracer_stat function_stats __initdata = {
			.name = "functions",
			.stat_start = function_stat_start,
			.stat_next = function_stat_next,
			.stat_cmp = function_stat_cmp,
			.stat_headers = function_stat_headers,
			.stat_show = function_stat_show
			};

	static void ftrace_profile_debugfs(struct dentry *d_tracer)		static void ftrace_profile_debugfs(struct dentry *d_tracer)
	{		{
			struct ftrace_profile_stat *stat;
	struct dentry *entry;		struct dentry *entry;
			char *name;
	int ret;		int ret;
			int cpu;

			for_each_possible_cpu(cpu) {
			stat = &per_cpu(ftrace_profile_stats, cpu);

	ret = register_stat_tracer(&function_stats);		/* allocate enough for function name + cpu number */
			name = kmalloc(32, GFP_KERNEL);
			if (!name) {
			/*
			* The files created are permanent, if something happens
			* we still do not free memory.
			*/
			kfree(stat);
			WARN(1,
			"Could not allocate stat file for cpu %d\n",
			cpu);
			return;
			}
			stat->stat = function_stats;
			snprintf(name, 32, "function%d", cpu);
			stat->stat.name = name;
			ret = register_stat_tracer(&stat->stat);
	if (ret) {		if (ret) {
	pr_warning("Warning: could not register "		WARN(1,
	"function stats\n");		"Could not register function stat for cpu %d\n",
			cpu);
			kfree(name);
	return;		return;
	}		}
			}

	entry = debugfs_create_file("function_profile_enabled", 0644,		entry = debugfs_create_file("function_profile_enabled", 0644,
	d_tracer, NULL, &ftrace_profile_fops);		d_tracer, NULL, &ftrace_profile_fops);