Merge "arm64: Change cpu_resume() to enable mmu early then access sleep_sp by va" (4fba233d) · Commits · e / devices / android_kernel_sony_msm8998

arch/arm64/include/asm/suspend.h

+3 −6

Original line number	Diff line number	Diff line
		#ifndef __ASM_SUSPEND_H
		#define __ASM_SUSPEND_H

		#define NR_CTX_REGS 11
		#define NR_CTX_REGS 10
		#define NR_CALLEE_SAVED_REGS 12

		/*
		@@ -17,11 +17,6 @@ struct cpu_suspend_ctx {
		u64 sp;
		} __aligned(16);

		struct sleep_save_sp {
		phys_addr_t *save_ptr_stash;
		phys_addr_t save_ptr_stash_phys;
		};

		/*
		* Memory to save the cpu state is allocated on the stack by
		* __cpu_suspend_enter()'s caller, and populated by __cpu_suspend_enter().
		@@ -39,6 +34,8 @@ struct sleep_stack_data {
		unsigned long callee_saved_regs[NR_CALLEE_SAVED_REGS];
		};

		extern unsigned long *sleep_save_stash;

		extern int cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
		extern void cpu_resume(void);
		int __cpu_suspend_enter(struct sleep_stack_data *state);

arch/arm64/kernel/asm-offsets.c

+0 −3

Original line number	Diff line number	Diff line
		@@ -140,9 +140,6 @@ int main(void)
		DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
		DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
		DEFINE(MPIDR_HASH_SHIFTS, offsetof(struct mpidr_hash, shift_aff));
		DEFINE(SLEEP_SAVE_SP_SZ, sizeof(struct sleep_save_sp));
		DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys));
		DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash));
		DEFINE(SLEEP_STACK_DATA_SYSTEM_REGS, offsetof(struct sleep_stack_data, system_regs));
		DEFINE(SLEEP_STACK_DATA_CALLEE_REGS, offsetof(struct sleep_stack_data, callee_saved_regs));
		#endif

arch/arm64/kernel/head.S

+1 −1

Original line number	Diff line number	Diff line
		@@ -663,7 +663,7 @@ ENDPROC(__secondary_switched)
		* If it isn't, park the CPU
		*/
		.section ".idmap.text", "ax"
		__enable_mmu:
		ENTRY(__enable_mmu)
		mrs x22, sctlr_el1 // preserve old SCTLR_EL1 value
		mrs x1, ID_AA64MMFR0_EL1
		ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4

arch/arm64/kernel/setup.c

+0 −1

Original line number	Diff line number	Diff line
		@@ -183,7 +183,6 @@ static void __init smp_build_mpidr_hash(void)
		*/
		if (mpidr_hash_size() > 4 * num_possible_cpus())
		pr_warn("Large number of MPIDR hash buckets detected\n");
		__flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
		}

		static void __init setup_machine_fdt(phys_addr_t dt_phys)

arch/arm64/kernel/sleep.S

+26 −40

Original line number	Diff line number	Diff line
		@@ -73,8 +73,8 @@ ENTRY(__cpu_suspend_enter)
		str x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]

		/* find the mpidr_hash */
		ldr x1, =sleep_save_sp
		ldr x1, [x1, #SLEEP_SAVE_SP_VIRT]
		ldr x1, =sleep_save_stash
		ldr x1, [x1]
		mrs x7, mpidr_el1
		ldr x9, =mpidr_hash
		ldr x10, [x9, #MPIDR_HASH_MASK]
		@@ -87,44 +87,27 @@ ENTRY(__cpu_suspend_enter)
		compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
		add x1, x1, x8, lsl #3

		str x0, [x1]
		add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
		stp x29, lr, [sp, #-16]!
		bl __cpu_suspend_save
		bl cpu_do_suspend
		ldp x29, lr, [sp], #16
		mov x0, #1
		ret
		ENDPROC(__cpu_suspend_enter)
		.ltorg

		/*
		* x0 must contain the sctlr value retrieved from restored context
		*/
		.pushsection ".idmap.text", "ax"
		ENTRY(cpu_resume_mmu)
		ldr x3, =cpu_resume_after_mmu
		msr sctlr_el1, x0 // restore sctlr_el1
		isb
		/*
		* Invalidate the local I-cache so that any instructions fetched
		* speculatively from the PoC are discarded, since they may have
		* been dynamically patched at the PoU.
		*/
		ic iallu
		dsb nsh
		isb
		br x3 // global jump to virtual address
		ENDPROC(cpu_resume_mmu)
		.popsection
		cpu_resume_after_mmu:
		#ifdef CONFIG_KASAN
		mov x0, sp
		bl kasan_unpoison_remaining_stack
		#endif
		mov x0, #0 // return zero on success
		ret
		ENDPROC(cpu_resume_after_mmu)

		ENTRY(cpu_resume)
		bl el2_setup // if in EL2 drop to EL1 cleanly
		/* enable the MMU early - so we can access sleep_save_stash by va */
		adr_l lr, __enable_mmu /* __cpu_setup will return here */
		ldr x27, =_cpu_resume /* __enable_mmu will branch here */
		adrp x25, idmap_pg_dir
		adrp x26, swapper_pg_dir
		b __cpu_setup
		ENDPROC(cpu_resume)

		ENTRY(_cpu_resume)
		mrs x1, mpidr_el1
		adrp x8, mpidr_hash
		add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address
		@@ -134,29 +117,32 @@ ENTRY(cpu_resume)
		ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
		compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
		/* x7 contains hash index, let's use it to grab context pointer */
		ldr_l x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS
		ldr_l x0, sleep_save_stash
		ldr x0, [x0, x7, lsl #3]
		add x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS
		add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
		/* load sp from context */
		ldr x2, [x0, #CPU_CTX_SP]
		/* load physical address of identity map page table in x1 */
		adrp x1, idmap_pg_dir
		mov sp, x2
		/* save thread_info */
		and x2, x2, #~(THREAD_SIZE - 1)
		msr sp_el0, x2
		/*
		* cpu_do_resume expects x0 to contain context physical address
		* pointer and x1 to contain physical address of 1:1 page tables
		* cpu_do_resume expects x0 to contain context address pointer
		*/
		bl cpu_do_resume // PC relative jump, MMU off
		/* Can't access these by physical address once the MMU is on */
		bl cpu_do_resume

		#ifdef CONFIG_KASAN
		mov x0, sp
		bl kasan_unpoison_remaining_stack
		#endif

		ldp x19, x20, [x29, #16]
		ldp x21, x22, [x29, #32]
		ldp x23, x24, [x29, #48]
		ldp x25, x26, [x29, #64]
		ldp x27, x28, [x29, #80]
		ldp x29, lr, [x29]
		b cpu_resume_mmu // Resume MMU, never returns
		ENDPROC(cpu_resume)
		mov x0, #0
		ret
		ENDPROC(_cpu_resume)