Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6ba3c97a authored by Frederic Weisbecker's avatar Frederic Weisbecker
Browse files

x86: Exception hooks for userspace RCU extended QS 



Add necessary hooks to x86 exception for userspace
RCU extended quiescent state support.

This includes traps, page fault, debug exceptions, etc...

Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Alessio Igor Bogani <abogani@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Avi Kivity <avi@redhat.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Geoff Levand <geoff@infradead.org>
Cc: Gilad Ben Yossef <gilad@benyossef.com>
Cc: Hakan Akkan <hakanakkan@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Kevin Hilman <khilman@ti.com>
Cc: Max Krasnyansky <maxk@qualcomm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Hemminger <shemminger@vyatta.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sven-Thorsten Dietrich <thebigcorporation@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent ef3f6288

arch/x86/include/asm/rcu.h

0 → 100644
+20 −0
Original line number Diff line number Diff line 
#ifndef _ASM_X86_RCU_H

#define _ASM_X86_RCU_H



#include <linux/rcupdate.h>

#include <asm/ptrace.h>



static inline void exception_enter(struct pt_regs *regs)

{

	rcu_user_exit();

}



static inline void exception_exit(struct pt_regs *regs)

{

#ifdef CONFIG_RCU_USER_QS

	if (user_mode(regs))

		rcu_user_enter();

#endif

}



#endif

arch/x86/kernel/traps.c

+55 −26
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ 
#include <asm/i387.h>

#include <asm/fpu-internal.h>

#include <asm/mce.h>

#include <asm/rcu.h>



#include <asm/mach_traps.h>
@@ -180,11 +181,15 @@ vm86_trap: 
#define DO_ERROR(trapnr, signr, str, name)				\

dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	\

{									\

	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)	\

							== NOTIFY_STOP)	\

	exception_enter(regs);						\

	if (notify_die(DIE_TRAP, str, regs, error_code,			\

			trapnr, signr) == NOTIFY_STOP) {		\

		exception_exit(regs);					\

		return;							\

	}								\

	conditional_sti(regs);						\

	do_trap(trapnr, signr, str, regs, error_code, NULL);		\

	exception_exit(regs);						\

}



#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr)		\
@@ -195,11 +200,15 @@ dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ 
	info.si_errno = 0;						\

	info.si_code = sicode;						\

	info.si_addr = (void __user *)siaddr;				\

	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)	\

							== NOTIFY_STOP)	\

	exception_enter(regs);						\

	if (notify_die(DIE_TRAP, str, regs, error_code,			\

			trapnr, signr) == NOTIFY_STOP) {		\

		exception_exit(regs);					\

		return;							\

	}								\

	conditional_sti(regs);						\

	do_trap(trapnr, signr, str, regs, error_code, &info);		\

	exception_exit(regs);						\

}



DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
@@ -222,19 +231,22 @@ DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check, 
/* Runs on IST stack */

dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)

{

	exception_enter(regs);

	if (notify_die(DIE_TRAP, "stack segment", regs, error_code,

			X86_TRAP_SS, SIGBUS) == NOTIFY_STOP)

		return;

		       X86_TRAP_SS, SIGBUS) != NOTIFY_STOP) {

		preempt_conditional_sti(regs);

		do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);

		preempt_conditional_cli(regs);

	}

	exception_exit(regs);

}



dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)

{

	static const char str[] = "double fault";

	struct task_struct *tsk = current;



	exception_enter(regs);

	/* Return not checked because double check cannot be ignored */

	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
@@ -255,27 +267,28 @@ do_general_protection(struct pt_regs *regs, long error_code) 
{

	struct task_struct *tsk;



	exception_enter(regs);

	conditional_sti(regs);



#ifdef CONFIG_X86_32

	if (regs->flags & X86_VM_MASK) {

		local_irq_enable();

		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);

		return;

		goto exit;

	}

#endif



	tsk = current;

	if (!user_mode(regs)) {

		if (fixup_exception(regs))

			return;

			goto exit;



		tsk->thread.error_code = error_code;

		tsk->thread.trap_nr = X86_TRAP_GP;

		if (!notify_die(DIE_GPF, "general protection fault", regs, error_code,

			       X86_TRAP_GP, SIGSEGV) == NOTIFY_STOP)

		if (notify_die(DIE_GPF, "general protection fault", regs, error_code,

			       X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)

			die("general protection fault", regs, error_code);

		return;

		goto exit;

	}



	tsk->thread.error_code = error_code;
@@ -291,7 +304,8 @@ do_general_protection(struct pt_regs *regs, long error_code) 
	}



	force_sig(SIGSEGV, tsk);

	return;

exit:

	exception_exit(regs);

}



/* May run on IST stack. */
@@ -306,15 +320,16 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co 
	    ftrace_int3_handler(regs))

		return;

#endif

	exception_enter(regs);

#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP

	if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,

				SIGTRAP) == NOTIFY_STOP)

		return;

		goto exit;

#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */



	if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,

			SIGTRAP) == NOTIFY_STOP)

		return;

		goto exit;



	/*

	 * Let others (NMI) know that the debug stack is in use
@@ -325,6 +340,8 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co 
	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);

	preempt_conditional_cli(regs);

	debug_stack_usage_dec();

exit:

	exception_exit(regs);

}



#ifdef CONFIG_X86_64
@@ -385,6 +402,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 
	unsigned long dr6;

	int si_code;



	exception_enter(regs);



	get_debugreg(dr6, 6);



	/* Filter out all the reserved bits which are preset to 1 */
@@ -400,7 +419,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 


	/* Catch kmemcheck conditions first of all! */

	if ((dr6 & DR_STEP) && kmemcheck_trap(regs))

		return;

		goto exit;



	/* DR6 may or may not be cleared by the CPU */

	set_debugreg(0, 6);
@@ -415,7 +434,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 


	if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code,

							SIGTRAP) == NOTIFY_STOP)

		return;

		goto exit;



	/*

	 * Let others (NMI) know that the debug stack is in use
@@ -431,7 +450,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 
					X86_TRAP_DB);

		preempt_conditional_cli(regs);

		debug_stack_usage_dec();

		return;

		goto exit;

	}



	/*
@@ -452,7 +471,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 
	preempt_conditional_cli(regs);

	debug_stack_usage_dec();



	return;

exit:

	exception_exit(regs);

}



/*
@@ -549,14 +569,17 @@ dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code) 
#ifdef CONFIG_X86_32

	ignore_fpu_irq = 1;

#endif



	exception_enter(regs);

	math_error(regs, error_code, X86_TRAP_MF);

	exception_exit(regs);

}



dotraplinkage void

do_simd_coprocessor_error(struct pt_regs *regs, long error_code)

{

	exception_enter(regs);

	math_error(regs, error_code, X86_TRAP_XF);

	exception_exit(regs);

}



dotraplinkage void
@@ -623,6 +646,7 @@ EXPORT_SYMBOL_GPL(math_state_restore); 
dotraplinkage void __kprobes

do_device_not_available(struct pt_regs *regs, long error_code)

{

	exception_enter(regs);

#ifdef CONFIG_MATH_EMULATION

	if (read_cr0() & X86_CR0_EM) {

		struct math_emu_info info = { };
@@ -631,6 +655,7 @@ do_device_not_available(struct pt_regs *regs, long error_code) 


		info.regs = regs;

		math_emulate(&info);

		exception_exit(regs);

		return;

	}

#endif
@@ -638,12 +663,15 @@ do_device_not_available(struct pt_regs *regs, long error_code) 
#ifdef CONFIG_X86_32

	conditional_sti(regs);

#endif

	exception_exit(regs);

}



#ifdef CONFIG_X86_32

dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)

{

	siginfo_t info;



	exception_enter(regs);

	local_irq_enable();



	info.si_signo = SIGILL;
@@ -651,11 +679,12 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) 
	info.si_code = ILL_BADSTK;

	info.si_addr = NULL;

	if (notify_die(DIE_TRAP, "iret exception", regs, error_code,

			X86_TRAP_IRET, SIGILL) == NOTIFY_STOP)

		return;

			X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {

		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,

			&info);

	}

	exception_exit(regs);

}

#endif



/* Set of traps needed for early debugging. */

arch/x86/mm/fault.c

+11 −2
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include <asm/pgalloc.h>		/* pgd_*(), ...			*/

#include <asm/kmemcheck.h>		/* kmemcheck_*(), ...		*/

#include <asm/fixmap.h>			/* VSYSCALL_START		*/

#include <asm/rcu.h>			/* exception_enter(), ...	*/



/*

 * Page fault error code bits:
@@ -1000,8 +1001,8 @@ static int fault_in_kernel_space(unsigned long address) 
 * and the problem, and then passes it off to one of the appropriate

 * routines.

 */

dotraplinkage void __kprobes

do_page_fault(struct pt_regs *regs, unsigned long error_code)

static void __kprobes

__do_page_fault(struct pt_regs *regs, unsigned long error_code)

{

	struct vm_area_struct *vma;

	struct task_struct *tsk;
@@ -1209,3 +1210,11 @@ good_area: 


	up_read(&mm->mmap_sem);

}



dotraplinkage void __kprobes

do_page_fault(struct pt_regs *regs, unsigned long error_code)

{

	exception_enter(regs);

	__do_page_fault(regs, error_code);

	exception_exit(regs);

}