Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c002a1e6 authored by Alexander van Heukelum's avatar Alexander van Heukelum Committed by Ingo Molnar
Browse files

x86: introduce save_rest and restructure the PTREGSCALL macro in entry_64.S 



Impact: cleanup

The save_rest function completes a partial stack frame for use
by the PTREGSCALL macro. This also avoids the indirect call in
PTREGSCALLs.

This adds the macro movq_cfi_restore to hide the CFI_RESTORE
annotation when restoring a register from the stack frame.

Signed-off-by: default avatarAlexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 14ae22ba

arch/x86/kernel/entry_64.S

+53 −38
Original line number Original line Diff line number Diff line
@@ -78,6 +78,11 @@ 
	CFI_REL_OFFSET \reg, \offset

	CFI_REL_OFFSET \reg, \offset

	.endm

	.endm





	.macro movq_cfi_restore offset reg

	movq \offset(%rsp), %\reg

	CFI_RESTORE \reg

	.endm



#ifdef CONFIG_FUNCTION_TRACER

#ifdef CONFIG_FUNCTION_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE

#ifdef CONFIG_DYNAMIC_FTRACE

ENTRY(mcount)

ENTRY(mcount)
@@ -186,21 +191,21 @@ ENTRY(native_usergs_sysret64) 
 */

 */





	/* %rsp:at FRAMEEND */

	/* %rsp:at FRAMEEND */

	.macro FIXUP_TOP_OF_STACK tmp

	.macro FIXUP_TOP_OF_STACK tmp offset=0

	movq %gs:pda_oldrsp,\tmp

	movq %gs:pda_oldrsp,\tmp

	movq  	\tmp,RSP(%rsp)

	movq \tmp,RSP+\offset(%rsp)

	movq    $__USER_DS,SS(%rsp)

	movq $__USER_DS,SS+\offset(%rsp)

	movq    $__USER_CS,CS(%rsp)

	movq $__USER_CS,CS+\offset(%rsp)

	movq 	$-1,RCX(%rsp)

	movq $-1,RCX+\offset(%rsp)

	movq	R11(%rsp),\tmp  /* get eflags */

	movq R11+\offset(%rsp),\tmp  /* get eflags */

	movq	\tmp,EFLAGS(%rsp)

	movq \tmp,EFLAGS+\offset(%rsp)

	.endm

	.endm





	.macro RESTORE_TOP_OF_STACK tmp,offset=0

	.macro RESTORE_TOP_OF_STACK tmp offset=0

	movq   RSP-\offset(%rsp),\tmp

	movq RSP+\offset(%rsp),\tmp

	movq \tmp,%gs:pda_oldrsp

	movq \tmp,%gs:pda_oldrsp

	movq   EFLAGS-\offset(%rsp),\tmp

	movq EFLAGS+\offset(%rsp),\tmp

	movq   \tmp,R11-\offset(%rsp)

	movq \tmp,R11+\offset(%rsp)

	.endm

	.endm





	.macro FAKE_STACK_FRAME child_rip

	.macro FAKE_STACK_FRAME child_rip
@@ -333,6 +338,21 @@ ENTRY(save_args) 
	CFI_ENDPROC

	CFI_ENDPROC

END(save_args)

END(save_args)





ENTRY(save_rest)

	PARTIAL_FRAME 1 REST_SKIP+8

	movq 5*8+16(%rsp), %r11	/* save return address */

	movq_cfi rbx, RBX+16

	movq_cfi rbp, RBP+16

	movq_cfi r12, R12+16

	movq_cfi r13, R13+16

	movq_cfi r14, R14+16

	movq_cfi r15, R15+16

	movq %r11, 8(%rsp)	/* return address */

	FIXUP_TOP_OF_STACK %r11, 16

	ret

	CFI_ENDPROC

END(save_rest)



/*

/*

 * A newly forked process directly context switches into this.

 * A newly forked process directly context switches into this.

 */

 */
@@ -353,7 +373,7 @@ rff_action: 
	je   int_ret_from_sys_call

	je   int_ret_from_sys_call

	testl $_TIF_IA32,TI_flags(%rcx)

	testl $_TIF_IA32,TI_flags(%rcx)

	jnz  int_ret_from_sys_call

	jnz  int_ret_from_sys_call

	RESTORE_TOP_OF_STACK %rdi,ARGOFFSET

	RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET

	jmp ret_from_sys_call

	jmp ret_from_sys_call

rff_trace:

rff_trace:

	movq %rsp,%rdi

	movq %rsp,%rdi
@@ -626,18 +646,20 @@ END(system_call) 
/*

/*

 * Certain special system calls that need to save a complete full stack frame.

 * Certain special system calls that need to save a complete full stack frame.

 */

 */



	.macro PTREGSCALL label,func,arg

	.macro PTREGSCALL label,func,arg

	.globl \label

ENTRY(\label)

\label:

	PARTIAL_FRAME 1 8		/* offset 8: return address */

	leaq	\func(%rip),%rax

	subq $REST_SKIP, %rsp

	leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */

	CFI_ADJUST_CFA_OFFSET REST_SKIP

	call save_rest

	DEFAULT_FRAME 0 8		/* offset 8: return address */

	leaq 8(%rsp), \arg	/* pt_regs pointer */

	call \func

	jmp ptregscall_common

	jmp ptregscall_common

	CFI_ENDPROC

END(\label)

END(\label)

	.endm

	.endm





	CFI_STARTPROC



	PTREGSCALL stub_clone, sys_clone, %r8

	PTREGSCALL stub_clone, sys_clone, %r8

	PTREGSCALL stub_fork, sys_fork, %rdi

	PTREGSCALL stub_fork, sys_fork, %rdi

	PTREGSCALL stub_vfork, sys_vfork, %rdi

	PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -645,22 +667,15 @@ END(\label) 
	PTREGSCALL stub_iopl, sys_iopl, %rsi

	PTREGSCALL stub_iopl, sys_iopl, %rsi





ENTRY(ptregscall_common)

ENTRY(ptregscall_common)

	popq %r11

	DEFAULT_FRAME 1 8	/* offset 8: return address */

	CFI_ADJUST_CFA_OFFSET -8

	RESTORE_TOP_OF_STACK %r11, 8

	CFI_REGISTER rip, r11

	movq_cfi_restore R15+8, r15

	SAVE_REST

	movq_cfi_restore R14+8, r14

	movq %r11, %r15

	movq_cfi_restore R13+8, r13

	CFI_REGISTER rip, r15

	movq_cfi_restore R12+8, r12

	FIXUP_TOP_OF_STACK %r11

	movq_cfi_restore RBP+8, rbp

	call *%rax

	movq_cfi_restore RBX+8, rbx

	RESTORE_TOP_OF_STACK %r11

	ret $REST_SKIP		/* pop extended registers */

	movq %r15, %r11

	CFI_REGISTER rip, r11

	RESTORE_REST

	pushq %r11

	CFI_ADJUST_CFA_OFFSET 8

	CFI_REL_OFFSET rip, 0

	ret

	CFI_ENDPROC

	CFI_ENDPROC

END(ptregscall_common)

END(ptregscall_common)