Merge branch 'tracing/ftrace' into tracing/function-return-tracer (98580960) · Commits · e / devices / android_kernel_oneplus_sm8150

arch/x86/include/asm/ftrace.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,14 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
		*/
		return addr - 1;
		}

		#ifdef CONFIG_DYNAMIC_FTRACE

		struct dyn_arch_ftrace {
		/* No extra data needed for x86 */
		};

		#endif /* CONFIG_DYNAMIC_FTRACE */
		#endif /* __ASSEMBLY__ */
		#endif /* CONFIG_FUNCTION_TRACER */

arch/x86/kernel/ftrace.c

+26 −3

Original line number	Diff line number	Diff line
		@@ -166,7 +166,7 @@ static int ftrace_calc_offset(long ip, long addr)
		return (int)(addr - ip);
		}

		unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
		static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
		{
		static union ftrace_code_union calc;

		@@ -311,12 +311,12 @@ do_ftrace_mod_code(unsigned long ip, void *new_code)

		static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];

		unsigned char *ftrace_nop_replace(void)
		static unsigned char *ftrace_nop_replace(void)
		{
		return ftrace_nop;
		}

		int
		static int
		ftrace_modify_code(unsigned long ip, unsigned char *old_code,
		unsigned char *new_code)
		{
		@@ -349,6 +349,29 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
		return 0;
		}

		int ftrace_make_nop(struct module *mod,
		struct dyn_ftrace *rec, unsigned long addr)
		{
		unsigned char new, old;
		unsigned long ip = rec->ip;

		old = ftrace_call_replace(ip, addr);
		new = ftrace_nop_replace();

		return ftrace_modify_code(rec->ip, old, new);
		}

		int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
		{
		unsigned char new, old;
		unsigned long ip = rec->ip;

		old = ftrace_nop_replace();
		new = ftrace_call_replace(ip, addr);

		return ftrace_modify_code(rec->ip, old, new);
		}

		int ftrace_update_ftrace_func(ftrace_func_t func)
		{
		unsigned long ip = (unsigned long)(&ftrace_call);

include/linux/ftrace.h

+41 −12

Original line number	Diff line number	Diff line
		@@ -74,6 +74,9 @@ static inline void ftrace_start(void) { }
		#endif /* CONFIG_FUNCTION_TRACER */

		#ifdef CONFIG_DYNAMIC_FTRACE
		/* asm/ftrace.h must be defined for archs supporting dynamic ftrace */
		#include <asm/ftrace.h>

		enum {
		FTRACE_FL_FREE = (1 << 0),
		FTRACE_FL_FAILED = (1 << 1),
		@@ -88,6 +91,7 @@ struct dyn_ftrace {
		struct list_head list;
		unsigned long ip; /* address of mcount call-site */
		unsigned long flags;
		struct dyn_arch_ftrace arch;
		};

		int ftrace_force_update(void);
		@@ -95,22 +99,40 @@ void ftrace_set_filter(unsigned char *buf, int len, int reset);

		/* defined in arch */
		extern int ftrace_ip_converted(unsigned long ip);
		extern unsigned char *ftrace_nop_replace(void);
		extern unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr);
		extern int ftrace_dyn_arch_init(void *data);
		extern int ftrace_update_ftrace_func(ftrace_func_t func);
		extern void ftrace_caller(void);
		extern void ftrace_call(void);
		extern void mcount_call(void);

		/* May be defined in arch */
		extern int ftrace_arch_read_dyn_info(char *buf, int size);
		/**
		* ftrace_make_nop - convert code into top
		* @mod: module structure if called by module load initialization
		* @rec: the mcount call site record
		* @addr: the address that the call site should be calling
		*
		* This is a very sensitive operation and great care needs
		* to be taken by the arch. The operation should carefully
		* read the location, check to see if what is read is indeed
		* what we expect it to be, and then on success of the compare,
		* it should write to the location.
		*
		* The code segment at @rec->ip should be a caller to @addr
		*
		* Return must be:
		* 0 on success
		* -EFAULT on error reading the location
		* -EINVAL on a failed compare of the contents
		* -EPERM on error writing to the location
		* Any other value will be considered a failure.
		*/
		extern int ftrace_make_nop(struct module *mod,
		struct dyn_ftrace *rec, unsigned long addr);

		/**
		* ftrace_modify_code - modify code segment
		* @ip: the address of the code segment
		* @old_code: the contents of what is expected to be there
		* @new_code: the code to patch in
		* ftrace_make_call - convert a nop call site into a call to addr
		* @rec: the mcount call site record
		* @addr: the address that the call site should call
		*
		* This is a very sensitive operation and great care needs
		* to be taken by the arch. The operation should carefully
		@@ -118,6 +140,8 @@ extern int ftrace_arch_read_dyn_info(char *buf, int size);
		* what we expect it to be, and then on success of the compare,
		* it should write to the location.
		*
		* The code segment at @rec->ip should be a nop
		*
		* Return must be:
		* 0 on success
		* -EFAULT on error reading the location
		@@ -125,8 +149,11 @@ extern int ftrace_arch_read_dyn_info(char *buf, int size);
		* -EPERM on error writing to the location
		* Any other value will be considered a failure.
		*/
		extern int ftrace_modify_code(unsigned long ip, unsigned char *old_code,
		unsigned char *new_code);
		extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);


		/* May be defined in arch */
		extern int ftrace_arch_read_dyn_info(char *buf, int size);

		extern int skip_trace(unsigned long ip);

		@@ -259,11 +286,13 @@ static inline void ftrace_dump(void) { }

		#ifdef CONFIG_FTRACE_MCOUNT_RECORD
		extern void ftrace_init(void);
		extern void ftrace_init_module(unsigned long start, unsigned long end);
		extern void ftrace_init_module(struct module *mod,
		unsigned long start, unsigned long end);
		#else
		static inline void ftrace_init(void) { }
		static inline void
		ftrace_init_module(unsigned long start, unsigned long end) { }
		ftrace_init_module(struct module *mod,
		unsigned long start, unsigned long end) { }
		#endif

kernel/module.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -2201,7 +2201,7 @@ static noinline struct module load_module(void __user umod,
		/* sechdrs[0].sh_size is always zero */
		mseg = section_objs(hdr, sechdrs, secstrings, "__mcount_loc",
		sizeof(*mseg), &num_mcount);
		ftrace_init_module(mseg, mseg + num_mcount);
		ftrace_init_module(mod, mseg, mseg + num_mcount);

		err = module_finalize(hdr, sechdrs, mod);
		if (err < 0)

kernel/trace/ftrace.c

+109 −115

Original line number	Diff line number	Diff line
		@@ -334,7 +334,7 @@ ftrace_record_ip(unsigned long ip)
		{
		struct dyn_ftrace *rec;

		if (!ftrace_enabled \|\| ftrace_disabled)
		if (ftrace_disabled)
		return NULL;

		rec = ftrace_alloc_dyn_node(ip);
		@@ -348,107 +348,129 @@ ftrace_record_ip(unsigned long ip)
		return rec;
		}

		static void print_ip_ins(const char fmt, unsigned char p)
		{
		int i;

		printk(KERN_CONT "%s", fmt);

		for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
		}

		static void ftrace_bug(int failed, unsigned long ip)
		{
		switch (failed) {
		case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
		case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		printk(KERN_CONT "\n");
		break;
		case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
		default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
		}
		}

		#define FTRACE_ADDR ((long)(ftrace_caller))

		static int
		__ftrace_replace_code(struct dyn_ftrace *rec,
		unsigned char old, unsigned char new, int enable)
		__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
		{
		unsigned long ip, fl;

		ip = rec->ip;

		if (ftrace_filtered && enable) {
		/*
		* If filtering is on:
		* If this record is not to be traced and
		* it is not enabled then do nothing.
		*
		* If this record is set to be filtered and
		* is enabled then do nothing.
		* If this record is not to be traced and
		* it is enabled then disabled it.
		*
		* If this record is set to be filtered and
		* it is not enabled, enable it.
		*
		* If this record is not set to be filtered
		* and it is not enabled do nothing.
		*
		* If this record is set not to trace then
		* do nothing.
		*
		* If this record is set not to trace and
		* it is enabled then disable it.
		*
		* If this record is not set to be filtered and
		* it is enabled, disable it.
		*/

		fl = rec->flags & (FTRACE_FL_FILTER \| FTRACE_FL_NOTRACE \|
		FTRACE_FL_ENABLED);

		if ((fl == (FTRACE_FL_FILTER \| FTRACE_FL_ENABLED)) \|\|
		(fl == (FTRACE_FL_FILTER \| FTRACE_FL_NOTRACE)) \|\|
		!fl \|\| (fl == FTRACE_FL_NOTRACE))
		if (rec->flags & FTRACE_FL_NOTRACE) {
		if (rec->flags & FTRACE_FL_ENABLED)
		rec->flags &= ~FTRACE_FL_ENABLED;
		else
		return 0;

		} else if (ftrace_filtered && enable) {
		/*
		* If it is enabled disable it,
		* otherwise enable it!
		* Filtering is on:
		*/
		if (fl & FTRACE_FL_ENABLED) {
		/* swap new and old */
		new = old;
		old = ftrace_call_replace(ip, FTRACE_ADDR);
		rec->flags &= ~FTRACE_FL_ENABLED;
		} else {
		new = ftrace_call_replace(ip, FTRACE_ADDR);
		rec->flags \|= FTRACE_FL_ENABLED;
		}
		} else {

		if (enable) {
		/*
		* If this record is set not to trace and is
		* not enabled, do nothing.
		*/
		fl = rec->flags & (FTRACE_FL_NOTRACE \| FTRACE_FL_ENABLED);
		if (fl == FTRACE_FL_NOTRACE)
		fl = rec->flags & (FTRACE_FL_FILTER \| FTRACE_FL_ENABLED);

		/* Record is filtered and enabled, do nothing */
		if (fl == (FTRACE_FL_FILTER \| FTRACE_FL_ENABLED))
		return 0;

		new = ftrace_call_replace(ip, FTRACE_ADDR);
		} else
		old = ftrace_call_replace(ip, FTRACE_ADDR);
		/* Record is not filtered and is not enabled do nothing */
		if (!fl)
		return 0;

		/* Record is not filtered but enabled, disable it */
		if (fl == FTRACE_FL_ENABLED)
		rec->flags &= ~FTRACE_FL_ENABLED;
		else
		/* Otherwise record is filtered but not enabled, enable it */
		rec->flags \|= FTRACE_FL_ENABLED;
		} else {
		/* Disable or not filtered */

		if (enable) {
		/* if record is enabled, do nothing */
		if (rec->flags & FTRACE_FL_ENABLED)
		return 0;

		rec->flags \|= FTRACE_FL_ENABLED;

		} else {

		/* if record is not enabled do nothing */
		if (!(rec->flags & FTRACE_FL_ENABLED))
		return 0;

		rec->flags &= ~FTRACE_FL_ENABLED;
		}
		}

		return ftrace_modify_code(ip, old, new);
		if (rec->flags & FTRACE_FL_ENABLED)
		return ftrace_make_call(rec, FTRACE_ADDR);
		else
		return ftrace_make_nop(NULL, rec, FTRACE_ADDR);
		}

		static void ftrace_replace_code(int enable)
		{
		int i, failed;
		unsigned char new = NULL, old = NULL;
		struct dyn_ftrace *rec;
		struct ftrace_page *pg;

		if (enable)
		old = ftrace_nop_replace();
		else
		new = ftrace_nop_replace();

		for (pg = ftrace_pages_start; pg; pg = pg->next) {
		for (i = 0; i < pg->index; i++) {
		rec = &pg->records[i];

		/* don't modify code that has already faulted */
		if (rec->flags & FTRACE_FL_FAILED)
		/*
		* Skip over free records and records that have
		* failed.
		*/
		if (rec->flags & FTRACE_FL_FREE \|\|
		rec->flags & FTRACE_FL_FAILED)
		continue;

		/* ignore updates to this record's mcount site */
		@@ -459,68 +481,30 @@ static void ftrace_replace_code(int enable)
		unfreeze_record(rec);
		}

		failed = __ftrace_replace_code(rec, old, new, enable);
		failed = __ftrace_replace_code(rec, enable);
		if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
		rec->flags \|= FTRACE_FL_FAILED;
		if ((system_state == SYSTEM_BOOTING) \|\|
		!core_kernel_text(rec->ip)) {
		ftrace_free_rec(rec);
		} else
		ftrace_bug(failed, rec->ip);
		}
		}
		}
		}
		}

		static void print_ip_ins(const char fmt, unsigned char p)
		{
		int i;

		printk(KERN_CONT "%s", fmt);

		for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
		}

		static int
		ftrace_code_disable(struct dyn_ftrace *rec)
		ftrace_code_disable(struct module mod, struct dyn_ftrace rec)
		{
		unsigned long ip;
		unsigned char nop, call;
		int ret;

		ip = rec->ip;

		nop = ftrace_nop_replace();
		call = ftrace_call_replace(ip, mcount_addr);

		ret = ftrace_modify_code(ip, call, nop);
		ret = ftrace_make_nop(mod, rec, mcount_addr);
		if (ret) {
		switch (ret) {
		case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
		case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" expected: ", call);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		print_ip_ins(" replace: ", nop);
		printk(KERN_CONT "\n");
		break;
		case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
		default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
		}

		ftrace_bug(ret, ip);
		rec->flags \|= FTRACE_FL_FAILED;
		return 0;
		}
		@@ -560,7 +544,6 @@ static void ftrace_startup(void)

		mutex_lock(&ftrace_start_lock);
		ftrace_start_up++;
		if (ftrace_start_up == 1)
		command \|= FTRACE_ENABLE_CALLS;

		if (saved_ftrace_func != ftrace_trace_function) {
		@@ -639,7 +622,7 @@ static cycle_t ftrace_update_time;
		static unsigned long ftrace_update_cnt;
		unsigned long ftrace_update_tot_cnt;

		static int ftrace_update_code(void)
		static int ftrace_update_code(struct module *mod)
		{
		struct dyn_ftrace p, t;
		cycle_t start, stop;
		@@ -656,7 +639,7 @@ static int ftrace_update_code(void)
		list_del_init(&p->list);

		/* convert record (i.e, patch mcount-call with NOP) */
		if (ftrace_code_disable(p)) {
		if (ftrace_code_disable(mod, p)) {
		p->flags \|= FTRACE_FL_CONVERTED;
		ftrace_update_cnt++;
		} else
		@@ -1211,7 +1194,7 @@ ftrace_regex_release(struct inode inode, struct file file, int enable)

		mutex_lock(&ftrace_sysctl_lock);
		mutex_lock(&ftrace_start_lock);
		if (iter->filtered && ftrace_start_up && ftrace_enabled)
		if (ftrace_start_up && ftrace_enabled)
		ftrace_run_update_code(FTRACE_ENABLE_CALLS);
		mutex_unlock(&ftrace_start_lock);
		mutex_unlock(&ftrace_sysctl_lock);
		@@ -1298,7 +1281,8 @@ static __init int ftrace_init_debugfs(void)

		fs_initcall(ftrace_init_debugfs);

		static int ftrace_convert_nops(unsigned long *start,
		static int ftrace_convert_nops(struct module *mod,
		unsigned long *start,
		unsigned long *end)
		{
		unsigned long *p;
		@@ -1309,23 +1293,32 @@ static int ftrace_convert_nops(unsigned long *start,
		p = start;
		while (p < end) {
		addr = ftrace_call_adjust(*p++);
		/*
		* Some architecture linkers will pad between
		* the different mcount_loc sections of different
		* object files to satisfy alignments.
		* Skip any NULL pointers.
		*/
		if (!addr)
		continue;
		ftrace_record_ip(addr);
		}

		/* disable interrupts to prevent kstop machine */
		local_irq_save(flags);
		ftrace_update_code();
		ftrace_update_code(mod);
		local_irq_restore(flags);
		mutex_unlock(&ftrace_start_lock);

		return 0;
		}

		void ftrace_init_module(unsigned long start, unsigned long end)
		void ftrace_init_module(struct module *mod,
		unsigned long start, unsigned long end)
		{
		if (ftrace_disabled \|\| start == end)
		return;
		ftrace_convert_nops(start, end);
		ftrace_convert_nops(mod, start, end);
		}

		extern unsigned long __start_mcount_loc[];
		@@ -1355,7 +1348,8 @@ void __init ftrace_init(void)

		last_ftrace_enabled = ftrace_enabled = 1;

		ret = ftrace_convert_nops(__start_mcount_loc,
		ret = ftrace_convert_nops(NULL,
		__start_mcount_loc,
		__stop_mcount_loc);

		return;