Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a0776ec8 authored by Kenneth W Chen's avatar Kenneth W Chen Committed by Tony Luck
Browse files

[IA64] remove per-cpu ia64_phys_stacked_size_p8 



It's not efficient to use a per-cpu variable just to store
how many physical stack register a cpu has.  Ever since the
incarnation of ia64 up till upcoming Montecito processor, that
variable has "glued" to 96. Having a variable in memory means
that the kernel is burning an extra cacheline access on every
syscall and kernel exit path.  Such "static" value is better
served with the instruction patching utility exists today.
Convert ia64_phys_stacked_size_p8 into dynamic insn patching.

This also has a pleasant side effect of eliminating access to
per-cpu area while psr.ic=0 in the kernel exit path. (fixable
for per-cpu DTC work, but why bother?)

There are some concerns with the default value that the instruc-
tion encoded in the kernel image.  It shouldn't be concerned.
The reasons are:

(1) cpu_init() is called at CPU initialization.  In there, we
    find out physical stack register size from PAL and patch
    two instructions in kernel exit code.  The code in question
    can not be executed before the patching is done.

(2) current implementation stores zero in ia64_phys_stacked_size_p8,
    and that's what the current kernel exit path loads the value with.
    With the new code, it is equivalent that we store reg size 96
    in ia64_phys_stacked_size_p8, thus creating a better safety net.
    Given (1) above can never fail, having (2) is just a bonus.

All in all, this patch allow one less memory reference in the kernel
exit path, thus reducing syscall and interrupt return latency; and
avoid polluting potential useful data in the CPU cache.

Signed-off-by: default avatarKen Chen <kenneth.w.chen@intel.com>
Signed-off-by: default avatarTony Luck <tony.luck@intel.com>
parent 62d0cfcb

arch/ia64/kernel/entry.S

+2 −5
Original line number Diff line number Diff line
@@ -767,7 +767,7 @@ ENTRY(ia64_leave_syscall) 
	ld8.fill r15=[r3]			// M0|1 restore r15

	mov b6=r18				// I0   restore b6



	addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0 // A

	LOAD_PHYS_STACK_REG_SIZE(r17)

	mov f9=f0					// F    clear f9

(pKStk) br.cond.dpnt.many skip_rbs_switch		// B
@@ -775,7 +775,6 @@ ENTRY(ia64_leave_syscall) 
	shr.u r18=r19,16		// I0|1 get byte size of existing "dirty" partition

	cover				// B    add current frame into dirty partition & set cr.ifs

	;;

(pUStk) ld4 r17=[r17]			// M0|1 r17 = cpu_data->phys_stacked_size_p8

	mov r19=ar.bsp			// M2   get new backing store pointer

	mov f10=f0			// F    clear f10
@@ -953,9 +952,7 @@ GLOBAL_ENTRY(ia64_leave_kernel) 
	shr.u r18=r19,16	// get byte size of existing "dirty" partition

	;;

	mov r16=ar.bsp		// get existing backing store pointer

	addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0

	;;

	ld4 r17=[r17]		// r17 = cpu_data->phys_stacked_size_p8

	LOAD_PHYS_STACK_REG_SIZE(r17)

(pKStk)	br.cond.dpnt skip_rbs_switch



	/*

arch/ia64/kernel/patch.c

+20 −0
Original line number Diff line number Diff line
@@ -195,3 +195,23 @@ ia64_patch_gate (void) 
	ia64_patch_vtop(START(vtop), END(vtop));

	ia64_patch_mckinley_e9(START(mckinley_e9), END(mckinley_e9));

}



void ia64_patch_phys_stack_reg(unsigned long val)

{

	s32 * offp = (s32 *) __start___phys_stack_reg_patchlist;

	s32 * end = (s32 *) __end___phys_stack_reg_patchlist;

	u64 ip, mask, imm;



	/* see instruction format A4: adds r1 = imm13, r3 */

	mask = (0x3fUL << 27) | (0x7f << 13);

	imm = (((val >> 7) & 0x3f) << 27) | (val & 0x7f) << 13;



	while (offp < end) {

		ip = (u64) offp + *offp;

		ia64_patch(ip, mask, imm);

		ia64_fc(ip);

		++offp;

	}

	ia64_sync_i();

	ia64_srlz_i();

}

arch/ia64/kernel/setup.c

+5 −2
Original line number Diff line number Diff line
@@ -75,7 +75,6 @@ extern void ia64_setup_printk_clock(void); 


DEFINE_PER_CPU(struct cpuinfo_ia64, cpu_info);

DEFINE_PER_CPU(unsigned long, local_per_cpu_offset);

DEFINE_PER_CPU(unsigned long, ia64_phys_stacked_size_p8);

unsigned long ia64_cycles_per_usec;

struct ia64_boot_param *ia64_boot_param;

struct screen_info screen_info;
@@ -836,6 +835,7 @@ void __cpuinit 
cpu_init (void)

{

	extern void __cpuinit ia64_mmu_init (void *);

	static unsigned long max_num_phys_stacked = IA64_NUM_PHYS_STACK_REG;

	unsigned long num_phys_stacked;

	pal_vm_info_2_u_t vmi;

	unsigned int max_ctx;
@@ -949,7 +949,10 @@ cpu_init (void) 
		num_phys_stacked = 96;

	}

	/* size of physical stacked register partition plus 8 bytes: */

	__get_cpu_var(ia64_phys_stacked_size_p8) = num_phys_stacked*8 + 8;

	if (num_phys_stacked > max_num_phys_stacked) {

		ia64_patch_phys_stack_reg(num_phys_stacked*8 + 8);

		max_num_phys_stacked = num_phys_stacked;

	}

	platform_cpu_init();

	pm_idle = default_idle;

}

arch/ia64/kernel/vmlinux.lds.S

+7 −0
Original line number Diff line number Diff line
@@ -78,6 +78,13 @@ SECTIONS 
	  __stop___mca_table = .;

	}



  .data.patch.phys_stack_reg : AT(ADDR(.data.patch.phys_stack_reg) - LOAD_OFFSET)

	{

	  __start___phys_stack_reg_patchlist = .;

	  *(.data.patch.phys_stack_reg)

	  __end___phys_stack_reg_patchlist = .;

	}



  /* Global data */

  _data = .;

include/asm-ia64/asmmacro.h

+10 −0
Original line number Diff line number Diff line
@@ -103,6 +103,16 @@ 
# define FSYS_RETURN	br.ret.sptk.many b6

#endif



/*

 * If physical stack register size is different from DEF_NUM_STACK_REG,

 * dynamically patch the kernel for correct size.

 */

	.section ".data.patch.phys_stack_reg", "a"

	.previous

#define LOAD_PHYS_STACK_REG_SIZE(reg)			\

[1:]	adds reg=IA64_NUM_PHYS_STACK_REG*8+8,r0;	\

	.xdata4 ".data.patch.phys_stack_reg", 1b-.



/*

 * Up until early 2004, use of .align within a function caused bad unwind info.

 * TEXT_ALIGN(n) expands into ".align n" if a fixed GAS is available or into nothing