Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0816b0f0 authored by Vlad Zolotarov's avatar Vlad Zolotarov Committed by Ingo Molnar
Browse files

x86: Add read_mostly declaration/definition to variables from smp.h 



Add "read-mostly" qualifier to the following variables in
smp.h:

 - cpu_sibling_map
 - cpu_core_map
 - cpu_llc_shared_map
 - cpu_llc_id
 - cpu_number
 - x86_cpu_to_apicid
 - x86_bios_cpu_apicid
 - x86_cpu_to_logical_apicid

As long as all the variables above are only written during the
initialization, this change is meant to prevent the false
sharing. More specifically, on vSMP Foundation platform
x86_cpu_to_apicid shared the same internode_cache_line with
frequently written lapic_events.

From the analysis of the first 33 per_cpu variables out of 219
(memories they describe, to be more specific) the 8 have read_mostly
nature (tlb_vector_offset, cpu_loops_per_jiffy, xen_debug_irq, etc.)
and 25 are frequently written (irq_stack_union, gdt_page,
exception_stacks, idt_desc, etc.).

Assuming that the spread of the rest of the per_cpu variables is
similar, identifying the read mostly memories will make more sense
in terms of long-term code maintenance comparing to identifying
frequently written memories.

Signed-off-by: default avatarVlad Zolotarov <vlad@scalemp.com>
Acked-by: default avatarShai Fultheim <shai@scalemp.com>
Cc: Shai Fultheim (Shai@ScaleMP.com) <Shai@scalemp.com>
Cc: ido@wizery.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/1719258.EYKzE4Zbq5@vlad


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent c35f7741

arch/x86/include/asm/apic.h

+1 −1
Original line number Diff line number Diff line
@@ -537,7 +537,7 @@ static inline const struct cpumask *default_target_cpus(void) 
#endif

}



DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);

DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);





static inline unsigned int read_apic_id(void)

arch/x86/include/asm/smp.h

+8 −8
Original line number Diff line number Diff line
@@ -31,12 +31,12 @@ static inline bool cpu_has_ht_siblings(void) 
	return has_siblings;

}



DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map);

DECLARE_PER_CPU(cpumask_var_t, cpu_core_map);

DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);

DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);

/* cpus sharing the last level cache: */

DECLARE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);

DECLARE_PER_CPU(u16, cpu_llc_id);

DECLARE_PER_CPU(int, cpu_number);

DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);

DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);

DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);



static inline struct cpumask *cpu_sibling_mask(int cpu)

{
@@ -53,10 +53,10 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu) 
	return per_cpu(cpu_llc_shared_map, cpu);

}



DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid);

DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);

DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);

DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);

#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)

DECLARE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid);

DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid);

#endif



/* Static state in head.S used to set up a CPU */

arch/x86/kernel/apic/apic.c

+3 −3
Original line number Diff line number Diff line
@@ -75,8 +75,8 @@ physid_mask_t phys_cpu_present_map; 
/*

 * Map cpu index to physical APIC ID

 */

DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);

DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);

DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);

DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);

EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);

EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
@@ -88,7 +88,7 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); 
 * used for the mapping.  This is where the behaviors of x86_64 and 32

 * actually diverge.  Let's keep it ugly for now.

 */

DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);

DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);



/*

 * Knob to control our willingness to enable the local APIC.

arch/x86/kernel/setup_percpu.c

+1 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ 
#include <asm/cpu.h>

#include <asm/stackprotector.h>



DEFINE_PER_CPU(int, cpu_number);

DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);

EXPORT_PER_CPU_SYMBOL(cpu_number);



#ifdef CONFIG_X86_64

arch/x86/kernel/smpboot.c

+4 −4
Original line number Diff line number Diff line
@@ -104,17 +104,17 @@ int smp_num_siblings = 1; 
EXPORT_SYMBOL(smp_num_siblings);



/* Last level cache ID of each logical CPU */

DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;

DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;



/* representing HT siblings of each logical CPU */

DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);

DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);

EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);



/* representing HT and core siblings of each logical CPU */

DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);

DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);

EXPORT_PER_CPU_SYMBOL(cpu_core_map);



DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);

DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);



/* Per CPU bogomips and other parameters */

DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);