Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 71586276 authored by Will Deacon's avatar Will Deacon Committed by Catalin Marinas
Browse files

arm64: percpu: implement optimised pcpu access using tpidr_el1 



This patch implements optimised percpu variable accesses using the
el1 r/w thread register (tpidr_el1) along the same lines as arch/arm/.

Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 66aa8d6a

arch/arm64/include/asm/Kbuild

+0 −1
Original line number Diff line number Diff line
@@ -26,7 +26,6 @@ generic-y += mman.h 
generic-y += msgbuf.h

generic-y += mutex.h

generic-y += pci.h

generic-y += percpu.h

generic-y += poll.h

generic-y += posix_types.h

generic-y += resource.h

arch/arm64/include/asm/percpu.h

0 → 100644
+41 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2013 ARM Ltd.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

#ifndef __ASM_PERCPU_H

#define __ASM_PERCPU_H



static inline void set_my_cpu_offset(unsigned long off)

{

	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");

}



static inline unsigned long __my_cpu_offset(void)

{

	unsigned long off;

	register unsigned long *sp asm ("sp");



	/*

	 * We want to allow caching the value, so avoid using volatile and

	 * instead use a fake stack read to hazard against barrier().

	 */

	asm("mrs %0, tpidr_el1" : "=r" (off) : "Q" (*sp));



	return off;

}

#define __my_cpu_offset __my_cpu_offset()



#include <asm-generic/percpu.h>



#endif /* __ASM_PERCPU_H */

arch/arm64/kernel/setup.c

+10 −0
Original line number Diff line number Diff line
@@ -108,6 +108,16 @@ void __init early_print(const char *str, ...) 
	printk("%s", buf);

}



void __init smp_setup_processor_id(void)

{

	/*

	 * clear __my_cpu_offset on boot CPU to avoid hang caused by

	 * using percpu variable early, for example, lockdep will

	 * access percpu variable inside lock_release

	 */

	set_my_cpu_offset(0);

}



bool arch_match_cpu_phys_id(int cpu, u64 phys_id)

{

	return phys_id == cpu_logical_map(cpu);

arch/arm64/kernel/smp.c

+4 −2
Original line number Diff line number Diff line
@@ -122,8 +122,6 @@ asmlinkage void secondary_start_kernel(void) 
	struct mm_struct *mm = &init_mm;

	unsigned int cpu = smp_processor_id();



	printk("CPU%u: Booted secondary processor\n", cpu);



	/*

	 * All kernel threads share the same mm context; grab a

	 * reference and switch to it.
@@ -132,6 +130,9 @@ asmlinkage void secondary_start_kernel(void) 
	current->active_mm = mm;

	cpumask_set_cpu(cpu, mm_cpumask(mm));



	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

	printk("CPU%u: Booted secondary processor\n", cpu);



	/*

	 * TTBR0 is only used for the identity mapping at this stage. Make it

	 * point to zero page to avoid speculatively fetching new entries.
@@ -271,6 +272,7 @@ void __init smp_cpus_done(unsigned int max_cpus) 


void __init smp_prepare_boot_cpu(void)

{

	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

}



static void (*smp_cross_call)(const struct cpumask *, unsigned int);