Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5b485629 authored by Masami Hiramatsu's avatar Masami Hiramatsu Committed by Ingo Molnar
Browse files

kprobes, extable: Identify kprobes trampolines as kernel text area 



Improve __kernel_text_address()/kernel_text_address() to return
true if the given address is on a kprobe's instruction slot
trampoline.

This can help stacktraces to determine the address is on a
text area or not.

To implement this atomically in is_kprobe_*_slot(), also change
the insn_cache page list to an RCU list.

This changes timings a bit (it delays page freeing to the RCU garbage
collection phase), but none of that is in the hot path.

Note: this change can add small overhead to stack unwinders because
it adds 2 additional checks to __kernel_text_address(). However, the
impact should be very small, because kprobe_insn_pages list has 1 entry
per 256 probes(on x86, on arm/arm64 it will be 1024 probes),
and kprobe_optinsn_pages has 1 entry per 32 probes(on x86).
In most use cases, the number of kprobe events may be less
than 20, which means that is_kprobe_*_slot() will check just one entry.

Tested-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarMasami Hiramatsu <mhiramat@kernel.org>
Acked-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Cc: Ananth N Mavinakayanahalli <ananth@linux.vnet.ibm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/148388747896.6869.6354262871751682264.stgit@devbox


[ Improved the changelog and coding style. ]
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent f913f3a6

include/linux/kprobes.h

+29 −1
Original line number Original line Diff line number Diff line
@@ -278,9 +278,13 @@ struct kprobe_insn_cache { 
	int nr_garbage;

	int nr_garbage;

};

};





#ifdef __ARCH_WANT_KPROBES_INSN_SLOT

extern kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c);

extern kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c);

extern void __free_insn_slot(struct kprobe_insn_cache *c,

extern void __free_insn_slot(struct kprobe_insn_cache *c,

			     kprobe_opcode_t *slot, int dirty);

			     kprobe_opcode_t *slot, int dirty);

/* sleep-less address checking routine  */

extern bool __is_insn_slot_addr(struct kprobe_insn_cache *c,

				unsigned long addr);





#define DEFINE_INSN_CACHE_OPS(__name)					\

#define DEFINE_INSN_CACHE_OPS(__name)					\

extern struct kprobe_insn_cache kprobe_##__name##_slots;		\

extern struct kprobe_insn_cache kprobe_##__name##_slots;		\
@@ -294,6 +298,18 @@ static inline void free_##__name##_slot(kprobe_opcode_t *slot, int dirty)\ 
{									\

{									\

	__free_insn_slot(&kprobe_##__name##_slots, slot, dirty);	\

	__free_insn_slot(&kprobe_##__name##_slots, slot, dirty);	\

}									\

}									\

									\

static inline bool is_kprobe_##__name##_slot(unsigned long addr)	\

{									\

	return __is_insn_slot_addr(&kprobe_##__name##_slots, addr);	\

}

#else /* __ARCH_WANT_KPROBES_INSN_SLOT */

#define DEFINE_INSN_CACHE_OPS(__name)					\

static inline bool is_kprobe_##__name##_slot(unsigned long addr)	\

{									\

	return 0;							\

}

#endif





DEFINE_INSN_CACHE_OPS(insn);

DEFINE_INSN_CACHE_OPS(insn);
@@ -330,7 +346,6 @@ extern int proc_kprobes_optimization_handler(struct ctl_table *table, 
					     int write, void __user *buffer,

					     int write, void __user *buffer,

					     size_t *length, loff_t *ppos);

					     size_t *length, loff_t *ppos);

#endif

#endif



#endif /* CONFIG_OPTPROBES */

#endif /* CONFIG_OPTPROBES */

#ifdef CONFIG_KPROBES_ON_FTRACE

#ifdef CONFIG_KPROBES_ON_FTRACE

extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,

extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
@@ -481,6 +496,19 @@ static inline int enable_jprobe(struct jprobe *jp) 
	return enable_kprobe(&jp->kp);

	return enable_kprobe(&jp->kp);

}

}





#ifndef CONFIG_KPROBES

static inline bool is_kprobe_insn_slot(unsigned long addr)

{

	return false;

}

#endif

#ifndef CONFIG_OPTPROBES

static inline bool is_kprobe_optinsn_slot(unsigned long addr)

{

	return false;

}

#endif



#ifdef CONFIG_KPROBES

#ifdef CONFIG_KPROBES

/*

/*

 * Blacklist ganerating macro. Specify functions which is not probed

 * Blacklist ganerating macro. Specify functions which is not probed

kernel/extable.c

+8 −1
Original line number Original line Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <linux/module.h>

#include <linux/module.h>

#include <linux/mutex.h>

#include <linux/mutex.h>

#include <linux/init.h>

#include <linux/init.h>

#include <linux/kprobes.h>





#include <asm/sections.h>

#include <asm/sections.h>

#include <linux/uaccess.h>

#include <linux/uaccess.h>
@@ -104,6 +105,8 @@ int __kernel_text_address(unsigned long addr) 
		return 1;

		return 1;

	if (is_ftrace_trampoline(addr))

	if (is_ftrace_trampoline(addr))

		return 1;

		return 1;

	if (is_kprobe_optinsn_slot(addr) || is_kprobe_insn_slot(addr))

		return 1;

	/*

	/*

	 * There might be init symbols in saved stacktraces.

	 * There might be init symbols in saved stacktraces.

	 * Give those symbols a chance to be printed in

	 * Give those symbols a chance to be printed in
@@ -123,7 +126,11 @@ int kernel_text_address(unsigned long addr) 
		return 1;

		return 1;

	if (is_module_text_address(addr))

	if (is_module_text_address(addr))

		return 1;

		return 1;

	return is_ftrace_trampoline(addr);

	if (is_ftrace_trampoline(addr))

		return 1;

	if (is_kprobe_optinsn_slot(addr) || is_kprobe_insn_slot(addr))

		return 1;

	return 0;

}

}





/*

/*

kernel/kprobes.c

+54 −19
Original line number Original line Diff line number Diff line
@@ -149,9 +149,11 @@ kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 
	struct kprobe_insn_page *kip;

	struct kprobe_insn_page *kip;

	kprobe_opcode_t *slot = NULL;

	kprobe_opcode_t *slot = NULL;





	/* Since the slot array is not protected by rcu, we need a mutex */

	mutex_lock(&c->mutex);

	mutex_lock(&c->mutex);

 retry:

 retry:

	list_for_each_entry(kip, &c->pages, list) {

	rcu_read_lock();

	list_for_each_entry_rcu(kip, &c->pages, list) {

		if (kip->nused < slots_per_page(c)) {

		if (kip->nused < slots_per_page(c)) {

			int i;

			int i;

			for (i = 0; i < slots_per_page(c); i++) {

			for (i = 0; i < slots_per_page(c); i++) {
@@ -159,6 +161,7 @@ kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 
					kip->slot_used[i] = SLOT_USED;

					kip->slot_used[i] = SLOT_USED;

					kip->nused++;

					kip->nused++;

					slot = kip->insns + (i * c->insn_size);

					slot = kip->insns + (i * c->insn_size);

					rcu_read_unlock();

					goto out;

					goto out;

				}

				}

			}

			}
@@ -167,6 +170,7 @@ kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 
			WARN_ON(1);

			WARN_ON(1);

		}

		}

	}

	}

	rcu_read_unlock();





	/* If there are any garbage slots, collect it and try again. */

	/* If there are any garbage slots, collect it and try again. */

	if (c->nr_garbage && collect_garbage_slots(c) == 0)

	if (c->nr_garbage && collect_garbage_slots(c) == 0)
@@ -193,7 +197,7 @@ kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 
	kip->nused = 1;

	kip->nused = 1;

	kip->ngarbage = 0;

	kip->ngarbage = 0;

	kip->cache = c;

	kip->cache = c;

	list_add(&kip->list, &c->pages);

	list_add_rcu(&kip->list, &c->pages);

	slot = kip->insns;

	slot = kip->insns;

out:

out:

	mutex_unlock(&c->mutex);

	mutex_unlock(&c->mutex);
@@ -213,7 +217,8 @@ static int collect_one_slot(struct kprobe_insn_page *kip, int idx) 
		 * next time somebody inserts a probe.

		 * next time somebody inserts a probe.

		 */

		 */

		if (!list_is_singular(&kip->list)) {

		if (!list_is_singular(&kip->list)) {

			list_del(&kip->list);

			list_del_rcu(&kip->list);

			synchronize_rcu();

			kip->cache->free(kip->insns);

			kip->cache->free(kip->insns);

			kfree(kip);

			kfree(kip);

		}

		}
@@ -235,8 +240,7 @@ static int collect_garbage_slots(struct kprobe_insn_cache *c) 
			continue;

			continue;

		kip->ngarbage = 0;	/* we will collect all garbages */

		kip->ngarbage = 0;	/* we will collect all garbages */

		for (i = 0; i < slots_per_page(c); i++) {

		for (i = 0; i < slots_per_page(c); i++) {

			if (kip->slot_used[i] == SLOT_DIRTY &&

			if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))

			    collect_one_slot(kip, i))

				break;

				break;

		}

		}

	}

	}
@@ -248,29 +252,60 @@ void __free_insn_slot(struct kprobe_insn_cache *c, 
		      kprobe_opcode_t *slot, int dirty)

		      kprobe_opcode_t *slot, int dirty)

{

{

	struct kprobe_insn_page *kip;

	struct kprobe_insn_page *kip;

	long idx;





	mutex_lock(&c->mutex);

	mutex_lock(&c->mutex);

	list_for_each_entry(kip, &c->pages, list) {

	rcu_read_lock();

		long idx = ((long)slot - (long)kip->insns) /

	list_for_each_entry_rcu(kip, &c->pages, list) {

		idx = ((long)slot - (long)kip->insns) /

			(c->insn_size * sizeof(kprobe_opcode_t));

			(c->insn_size * sizeof(kprobe_opcode_t));

		if (idx >= 0 && idx < slots_per_page(c)) {

		if (idx >= 0 && idx < slots_per_page(c))

			goto out;

	}

	/* Could not find this slot. */

	WARN_ON(1);

	kip = NULL;

out:

	rcu_read_unlock();

	/* Mark and sweep: this may sleep */

	if (kip) {

		/* Check double free */

		WARN_ON(kip->slot_used[idx] != SLOT_USED);

		WARN_ON(kip->slot_used[idx] != SLOT_USED);

		if (dirty) {

		if (dirty) {

			kip->slot_used[idx] = SLOT_DIRTY;

			kip->slot_used[idx] = SLOT_DIRTY;

			kip->ngarbage++;

			kip->ngarbage++;

			if (++c->nr_garbage > slots_per_page(c))

			if (++c->nr_garbage > slots_per_page(c))

				collect_garbage_slots(c);

				collect_garbage_slots(c);

			} else

		} else {

			collect_one_slot(kip, idx);

			collect_one_slot(kip, idx);

			goto out;

		}

		}

	}

	}

	/* Could not free this slot. */

	WARN_ON(1);

out:

	mutex_unlock(&c->mutex);

	mutex_unlock(&c->mutex);

}

}





/*

 * Check given address is on the page of kprobe instruction slots.

 * This will be used for checking whether the address on a stack

 * is on a text area or not.

 */

bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)

{

	struct kprobe_insn_page *kip;

	bool ret = false;



	rcu_read_lock();

	list_for_each_entry_rcu(kip, &c->pages, list) {

		if (addr >= (unsigned long)kip->insns &&

		    addr < (unsigned long)kip->insns + PAGE_SIZE) {

			ret = true;

			break;

		}

	}

	rcu_read_unlock();



	return ret;

}



#ifdef CONFIG_OPTPROBES

#ifdef CONFIG_OPTPROBES

/* For optimized_kprobe buffer */

/* For optimized_kprobe buffer */

struct kprobe_insn_cache kprobe_optinsn_slots = {

struct kprobe_insn_cache kprobe_optinsn_slots = {