Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d6be29b8 authored by Masami Hiramatsu's avatar Masami Hiramatsu Committed by Ingo Molnar
Browse files

x86: kprobes code for x86 unification 



This patch unifies kprobes code.

- Unify kprobes_*.h to kprobes.h
- Unify kprobes_*.c to kprobes.c
  (Differences are separated by ifdefs)
 - Most differences are related to REX prefix and rip relatives.
 - Two inline assembly code are different.
 - One difference in kprobe_handlre()
 - One fixup exception code is different, but it will be unified
   if mm/extable_*.c are unified.
- Merge history logs into arch/x86/kernel/kprobes.c.

Signed-off-by: default avatarMasami Hiramatsu <mhiramat@redhat.com>
Signed-off-by: default avatarJim Keniston <jkenisto@us.ibm.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 8533bbe9

arch/x86/kernel/Makefile_32

+1 −1
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ obj-$(CONFIG_KEXEC) += machine_kexec_32.o relocate_kernel_32.o crash.o 
obj-$(CONFIG_CRASH_DUMP)	+= crash_dump_32.o

obj-$(CONFIG_X86_NUMAQ)		+= numaq_32.o

obj-$(CONFIG_X86_SUMMIT_NUMA)	+= summit_32.o

obj-$(CONFIG_KPROBES)		+= kprobes_32.o

obj-$(CONFIG_KPROBES)		+= kprobes.o

obj-$(CONFIG_MODULES)		+= module_32.o

obj-$(CONFIG_ACPI_SRAT) 	+= srat_32.o

obj-$(CONFIG_EFI) 		+= efi.o efi_32.o efi_stub_32.o

arch/x86/kernel/Makefile_64

+1 −1
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o 
obj-$(CONFIG_GART_IOMMU)	+= pci-gart_64.o aperture_64.o

obj-$(CONFIG_CALGARY_IOMMU)	+= pci-calgary_64.o tce_64.o

obj-$(CONFIG_SWIOTLB)		+= pci-swiotlb_64.o

obj-$(CONFIG_KPROBES)		+= kprobes_64.o

obj-$(CONFIG_KPROBES)		+= kprobes.o

obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer_64.o

obj-$(CONFIG_X86_VSMP)		+= vsmp_64.o

obj-$(CONFIG_K8_NB)		+= k8.o

arch/x86/kernel/kprobes_64.c→arch/x86/kernel/kprobes.c

+128 −36
Original line number Diff line number Diff line
@@ -22,14 +22,22 @@ 
 *		Rusty Russell).

 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes

 *		interface to access function arguments.

 * 2004-Oct	Jim Keniston <kenistoj@us.ibm.com> and Prasanna S Panchamukhi

 *		<prasanna@in.ibm.com> adapted for x86_64

 * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi

 *		<prasanna@in.ibm.com> adapted for x86_64 from i386.

 * 2005-Mar	Roland McGrath <roland@redhat.com>

 *		Fixed to handle %rip-relative addressing mode correctly.

 * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston

 *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi

 *		<prasanna@in.ibm.com> added function-return probes.

 * 2005-May	Rusty Lynch <rusty.lynch@intel.com>

 * 		Added function return probes functionality

 * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added

 * 		kprobe-booster and kretprobe-booster for i386.

 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster

 * 		and kretprobe-booster for x86-64

 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven

 * 		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>

 * 		unified x86 kprobes code.

 */



#include <linux/kprobes.h>
@@ -51,7 +59,19 @@ void jprobe_return_end(void); 
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;

DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);



#ifdef CONFIG_X86_64

#define stack_addr(regs) ((unsigned long *)regs->sp)

#else

/*

 * "&regs->sp" looks wrong, but it's correct for x86_32.  x86_32 CPUs

 * don't save the ss and esp registers if the CPU is already in kernel

 * mode when it traps.  So for kprobes, regs->sp and regs->ss are not

 * the [nonexistent] saved stack pointer and ss register, but rather

 * the top 8 bytes of the pre-int3 stack.  So &regs->sp happens to

 * point to the top of the pre-int3 stack.

 */

#define stack_addr(regs) ((unsigned long *)&regs->sp)

#endif



#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\

	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
@@ -151,8 +171,8 @@ static __always_inline void set_jmp_op(void *from, void *to) 
}



/*

 * returns non-zero if opcode is boostable.

 * RIP relative instructions are adjusted at copying time

 * Returns non-zero if opcode is boostable.

 * RIP relative instructions are adjusted at copying time in 64 bits mode

 */

static __always_inline int can_boost(kprobe_opcode_t *opcodes)

{
@@ -173,8 +193,10 @@ retry: 
	}



	switch (opcode & 0xf0) {

#ifdef CONFIG_X86_64

	case 0x40:

		goto retry; /* REX prefix is boostable */

#endif

	case 0x60:

		if (0x63 < opcode && opcode < 0x67)

			goto retry; /* prefixes */
@@ -206,7 +228,7 @@ retry: 
}



/*

 * returns non-zero if opcode modifies the interrupt flag.

 * Returns non-zero if opcode modifies the interrupt flag.

 */

static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)

{
@@ -217,16 +239,18 @@ static int __kprobes is_IF_modifier(kprobe_opcode_t *insn) 
	case 0x9d:		/* popf/popfd */

		return 1;

	}



#ifdef CONFIG_X86_64

	/*

	 * on 64 bit x86, 0x40-0x4f are prefixes so we need to look

	 * at the next byte instead.. but of course not recurse infinitely

	 */

	if (*insn  >= 0x40 && *insn <= 0x4f)

		return is_IF_modifier(++insn);

#endif

	return 0;

}



#ifdef CONFIG_X86_64

/*

 * Adjust the displacement if the instruction uses the %rip-relative

 * addressing mode.
@@ -263,17 +287,20 @@ static void __kprobes fix_riprel(struct kprobe *p) 
	if ((*insn & 0xf0) == 0x40)

		++insn;



	if (*insn == 0x0f) {	/* Two-byte opcode.  */

	if (*insn == 0x0f) {

		/* Two-byte opcode.  */

		++insn;

		need_modrm = test_bit(*insn,

				      (unsigned long *)twobyte_has_modrm);

	} else			/* One-byte opcode.  */

	} else

		/* One-byte opcode.  */

		need_modrm = test_bit(*insn,

				      (unsigned long *)onebyte_has_modrm);



	if (need_modrm) {

		u8 modrm = *++insn;

		if ((modrm & 0xc7) == 0x05) { /* %rip+disp32 addressing mode */

		if ((modrm & 0xc7) == 0x05) {

			/* %rip+disp32 addressing mode */

			/* Displacement follows ModRM byte.  */

			++insn;

			/*
@@ -296,11 +323,14 @@ static void __kprobes fix_riprel(struct kprobe *p) 
		}

	}

}

#endif



static void __kprobes arch_copy_kprobe(struct kprobe *p)

{

	memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));

#ifdef CONFIG_X86_64

	fix_riprel(p);

#endif

	if (can_boost(p->addr))

		p->ainsn.boostable = 0;

	else
@@ -365,13 +395,13 @@ static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 
static __always_inline void clear_btf(void)

{

	if (test_thread_flag(TIF_DEBUGCTLMSR))

		wrmsrl(MSR_IA32_DEBUGCTLMSR, 0);

		wrmsr(MSR_IA32_DEBUGCTLMSR, 0, 0);

}



static __always_inline void restore_btf(void)

{

	if (test_thread_flag(TIF_DEBUGCTLMSR))

		wrmsrl(MSR_IA32_DEBUGCTLMSR, current->thread.debugctlmsr);

		wrmsr(MSR_IA32_DEBUGCTLMSR, current->thread.debugctlmsr, 0);

}



static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
@@ -427,6 +457,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) 
				regs->flags &= ~TF_MASK;

				regs->flags |= kcb->kprobe_saved_flags;

				goto no_kprobe;

#ifdef CONFIG_X86_64

			} else if (kcb->kprobe_status == KPROBE_HIT_SSDONE) {

				/* TODO: Provide re-entrancy from

				 * post_kprobes_handler() and avoid exception
@@ -437,6 +468,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) 
				regs->ip = (unsigned long)p->addr;

				reset_current_kprobe();

				return 1;

#endif

			}

			/* We have reentered the kprobe_handler(), since

			 * another probe was hit while within the handler.
@@ -461,10 +493,9 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) 
				goto no_kprobe;

			}

			p = __get_cpu_var(current_kprobe);

			if (p->break_handler && p->break_handler(p, regs)) {

			if (p->break_handler && p->break_handler(p, regs))

				goto ss_probe;

		}

		}

		goto no_kprobe;

	}
@@ -519,8 +550,10 @@ no_kprobe: 
 */

 void __kprobes kretprobe_trampoline_holder(void)

 {

 	asm volatile (  ".global kretprobe_trampoline\n"

	asm volatile (

			".global kretprobe_trampoline\n"

			"kretprobe_trampoline: \n"

#ifdef CONFIG_X86_64

			/* We don't bother saving the ss register */

			"	pushq %rsp\n"

			"	pushfq\n"
@@ -566,13 +599,48 @@ no_kprobe: 
			/* Skip orig_ax, ip, cs */

			"	addq $24, %rsp\n"

			"	popfq\n"

#else

			"	pushf\n"

			/*

			 * Skip cs, ip, orig_ax.

			 * trampoline_handler() will plug in these values

			 */

			"	subl $12, %esp\n"

			"	pushl %fs\n"

			"	pushl %ds\n"

			"	pushl %es\n"

			"	pushl %eax\n"

			"	pushl %ebp\n"

			"	pushl %edi\n"

			"	pushl %esi\n"

			"	pushl %edx\n"

			"	pushl %ecx\n"

			"	pushl %ebx\n"

			"	movl %esp, %eax\n"

			"	call trampoline_handler\n"

			/* Move flags to cs */

			"	movl 52(%esp), %edx\n"

			"	movl %edx, 48(%esp)\n"

			/* Replace saved flags with true return address. */

			"	movl %eax, 52(%esp)\n"

			"	popl %ebx\n"

			"	popl %ecx\n"

			"	popl %edx\n"

			"	popl %esi\n"

			"	popl %edi\n"

			"	popl %ebp\n"

			"	popl %eax\n"

			/* Skip ip, orig_ax, es, ds, fs */

			"	addl $20, %esp\n"

			"	popf\n"

#endif

			"	ret\n");

 }



/*

 * Called from kretprobe_trampoline

 */

fastcall void * __kprobes trampoline_handler(struct pt_regs *regs)

void * __kprobes trampoline_handler(struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;
@@ -584,7 +652,11 @@ fastcall void * __kprobes trampoline_handler(struct pt_regs *regs) 
	spin_lock_irqsave(&kretprobe_lock, flags);

	head = kretprobe_inst_table_head(current);

	/* fixup registers */

#ifdef CONFIG_X86_64

	regs->cs = __KERNEL_CS;

#else

	regs->cs = __KERNEL_CS | get_kernel_rpl();

#endif

	regs->ip = trampoline_address;

	regs->orig_ax = ~0UL;
@@ -671,9 +743,11 @@ static void __kprobes resume_execution(struct kprobe *p, 
	unsigned long orig_ip = (unsigned long)p->addr;

	kprobe_opcode_t *insn = p->ainsn.insn;



#ifdef CONFIG_X86_64

	/*skip the REX prefix*/

	if (*insn >= 0x40 && *insn <= 0x4f)

		insn++;

#endif



	regs->flags &= ~TF_MASK;

	switch (*insn) {
@@ -693,6 +767,11 @@ static void __kprobes resume_execution(struct kprobe *p, 
	case 0xe8:	/* call relative - Fix return addr */

		*tos = orig_ip + (*tos - copy_ip);

		break;

#ifndef CONFIG_X86_64

	case 0x9a:	/* call absolute -- same as call absolute, indirect */

		*tos = orig_ip + (*tos - copy_ip);

		goto no_change;

#endif

	case 0xff:

		if ((insn[1] & 0x30) == 0x10) {

			/*
@@ -783,7 +862,6 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 
{

	struct kprobe *cur = kprobe_running();

	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

	const struct exception_table_entry *fixup;



	switch (kcb->kprobe_status) {

	case KPROBE_HIT_SS:
@@ -826,12 +904,19 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 
		 * In case the user-specified fault handler returned

		 * zero, try to fix up.

		 */

#ifdef CONFIG_X86_64

		{

			const struct exception_table_entry *fixup;

			fixup = search_exception_tables(regs->ip);

			if (fixup) {

				regs->ip = fixup->fixup;

				return 1;

			}



		}

#else

		if (fixup_exception(regs))

			return 1;

#endif

		/*

		 * fixup routine could not handle it,

		 * Let do_page_fault() fix it.
@@ -907,7 +992,12 @@ void __kprobes jprobe_return(void) 
{

	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();



	asm volatile ("       xchg   %%rbx,%%rsp     \n"

	asm volatile (

#ifdef CONFIG_X86_64

			"       xchg   %%rbx,%%rsp	\n"

#else

			"       xchgl   %%ebx,%%esp	\n"

#endif

			"       int3			\n"

			"       .globl jprobe_return_end\n"

			"       jprobe_return_end:	\n"
@@ -921,14 +1011,16 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 
	u8 *addr = (u8 *) (regs->ip - 1);

	struct jprobe *jp = container_of(p, struct jprobe, kp);



	if ((addr > (u8 *) jprobe_return) && (addr < (u8 *) jprobe_return_end)) {

	if ((addr > (u8 *) jprobe_return) &&

	    (addr < (u8 *) jprobe_return_end)) {

		if (stack_addr(regs) != kcb->jprobe_saved_sp) {

			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;

			printk("current sp %p does not match saved sp %p\n",

			printk(KERN_ERR

			       "current sp %p does not match saved sp %p\n",

			       stack_addr(regs), kcb->jprobe_saved_sp);

			printk("Saved registers for jprobe %p\n", jp);

			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);

			show_registers(saved_regs);

			printk("Current registers\n");

			printk(KERN_ERR "Current registers\n");

			show_registers(regs);

			BUG();

		}

arch/x86/kernel/kprobes_32.c

deleted100644 → 0
+0 −848

File deleted.

Preview size limit exceeded, changes collapsed.

include/asm-x86/kprobes.h

+98 −5
Original line number Diff line number Diff line 
#ifdef CONFIG_X86_32

# include "kprobes_32.h"

#else

# include "kprobes_64.h"

#endif
 
#ifndef _ASM_KPROBES_H

#define _ASM_KPROBES_H

/*

 *  Kernel Probes (KProbes)

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

 *

 * Copyright (C) IBM Corporation, 2002, 2004

 *

 * See arch/x86/kernel/kprobes.c for x86 kprobes history.

 */

#include <linux/types.h>

#include <linux/ptrace.h>

#include <linux/percpu.h>



#define  __ARCH_WANT_KPROBES_INSN_SLOT



struct pt_regs;

struct kprobe;



typedef u8 kprobe_opcode_t;

#define BREAKPOINT_INSTRUCTION	0xcc

#define RELATIVEJUMP_INSTRUCTION 0xe9

#define MAX_INSN_SIZE 16

#define MAX_STACK_SIZE 64

#define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \

	(((unsigned long)current_thread_info()) + THREAD_SIZE \

	 - (unsigned long)(ADDR))) \

	? (MAX_STACK_SIZE) \

	: (((unsigned long)current_thread_info()) + THREAD_SIZE \

	   - (unsigned long)(ADDR)))



#define ARCH_SUPPORTS_KRETPROBES

#define flush_insn_slot(p)	do { } while (0)



extern const int kretprobe_blacklist_size;



void arch_remove_kprobe(struct kprobe *p);

void kretprobe_trampoline(void);



/* Architecture specific copy of original instruction*/

struct arch_specific_insn {

	/* copy of the original instruction */

	kprobe_opcode_t *insn;

	/*

	 * boostable = -1: This instruction type is not boostable.

	 * boostable = 0: This instruction type is boostable.

	 * boostable = 1: This instruction has been boosted: we have

	 * added a relative jump after the instruction copy in insn,

	 * so no single-step and fixup are needed (unless there's

	 * a post_handler or break_handler).

	 */

	int boostable;

};



struct prev_kprobe {

	struct kprobe *kp;

	unsigned long status;

	unsigned long old_flags;

	unsigned long saved_flags;

};



/* per-cpu kprobe control block */

struct kprobe_ctlblk {

	unsigned long kprobe_status;

	unsigned long kprobe_old_flags;

	unsigned long kprobe_saved_flags;

	unsigned long *jprobe_saved_sp;

	struct pt_regs jprobe_saved_regs;

	kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE];

	struct prev_kprobe prev_kprobe;

};



/* trap3/1 are intr gates for kprobes.  So, restore the status of IF,

 * if necessary, before executing the original int3/1 (trap) handler.

 */

static inline void restore_interrupts(struct pt_regs *regs)

{

	if (regs->flags & IF_MASK)

		local_irq_enable();

}



extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);

extern int kprobe_exceptions_notify(struct notifier_block *self,

				    unsigned long val, void *data);

#endif				/* _ASM_KPROBES_H */