i386: Make patching more robust, fix paravirt issue (ab144f5e) · Commits · e / devices / android_kernel_xiaomi_nabu

arch/i386/kernel/alternative.c

+21 −12

Original line number	Original line	Diff line number	Diff line
	@@ -11,6 +11,8 @@
	#include <asm/mce.h>		#include <asm/mce.h>
	#include <asm/nmi.h>		#include <asm/nmi.h>

			#define MAX_PATCH_LEN (255-1)

	#ifdef CONFIG_HOTPLUG_CPU		#ifdef CONFIG_HOTPLUG_CPU
	static int smp_alt_once;		static int smp_alt_once;

	@@ -148,7 +150,8 @@ static unsigned char** find_nop_table(void)

	#endif /* CONFIG_X86_64 */		#endif /* CONFIG_X86_64 */

	static void nop_out(void *insns, unsigned int len)		/* Use this to add nops to a buffer, then text_poke the whole buffer. */
			static void add_nops(void *insns, unsigned int len)
	{		{
	unsigned char **noptable = find_nop_table();		unsigned char **noptable = find_nop_table();

	@@ -156,7 +159,7 @@ static void nop_out(void *insns, unsigned int len)
	unsigned int noplen = len;		unsigned int noplen = len;
	if (noplen > ASM_NOP_MAX)		if (noplen > ASM_NOP_MAX)
	noplen = ASM_NOP_MAX;		noplen = ASM_NOP_MAX;
	text_poke(insns, noptable[noplen], noplen);		memcpy(insns, noptable[noplen], noplen);
	insns += noplen;		insns += noplen;
	len -= noplen;		len -= noplen;
	}		}
	@@ -174,15 +177,15 @@ extern u8 __smp_locks[], __smp_locks_end[];
	void apply_alternatives(struct alt_instr start, struct alt_instr end)		void apply_alternatives(struct alt_instr start, struct alt_instr end)
	{		{
	struct alt_instr *a;		struct alt_instr *a;
	u8 *instr;		char insnbuf[MAX_PATCH_LEN];
	int diff;

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);		DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {		for (a = start; a < end; a++) {
			u8 *instr = a->instr;
	BUG_ON(a->replacementlen > a->instrlen);		BUG_ON(a->replacementlen > a->instrlen);
			BUG_ON(a->instrlen > sizeof(insnbuf));
	if (!boot_cpu_has(a->cpuid))		if (!boot_cpu_has(a->cpuid))
	continue;		continue;
	instr = a->instr;
	#ifdef CONFIG_X86_64		#ifdef CONFIG_X86_64
	/* vsyscall code is not mapped yet. resolve it manually. */		/* vsyscall code is not mapped yet. resolve it manually. */
	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {		if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	@@ -191,9 +194,10 @@ void apply_alternatives(struct alt_instr start, struct alt_instr end)
	__FUNCTION__, a->instr, instr);		__FUNCTION__, a->instr, instr);
	}		}
	#endif		#endif
	memcpy(instr, a->replacement, a->replacementlen);		memcpy(insnbuf, a->replacement, a->replacementlen);
	diff = a->instrlen - a->replacementlen;		add_nops(insnbuf + a->replacementlen,
	nop_out(instr + a->replacementlen, diff);		a->instrlen - a->replacementlen);
			text_poke(instr, insnbuf, a->instrlen);
	}		}
	}		}

	@@ -215,16 +219,18 @@ static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)		static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	{		{
	u8 **ptr;		u8 **ptr;
			char insn[1];

	if (noreplace_smp)		if (noreplace_smp)
	return;		return;

			add_nops(insn, 1);
	for (ptr = start; ptr < end; ptr++) {		for (ptr = start; ptr < end; ptr++) {
	if (*ptr < text)		if (*ptr < text)
	continue;		continue;
	if (*ptr > text_end)		if (*ptr > text_end)
	continue;		continue;
	nop_out(*ptr, 1);		text_poke(*ptr, insn, 1);
	};		};
	}		}

	@@ -351,6 +357,7 @@ void apply_paravirt(struct paravirt_patch_site *start,
	struct paravirt_patch_site *end)		struct paravirt_patch_site *end)
	{		{
	struct paravirt_patch_site *p;		struct paravirt_patch_site *p;
			char insnbuf[MAX_PATCH_LEN];

	if (noreplace_paravirt)		if (noreplace_paravirt)
	return;		return;
	@@ -358,13 +365,15 @@ void apply_paravirt(struct paravirt_patch_site *start,
	for (p = start; p < end; p++) {		for (p = start; p < end; p++) {
	unsigned int used;		unsigned int used;

	used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,		BUG_ON(p->len > MAX_PATCH_LEN);
	p->len);		used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
			(unsigned long)p->instr, p->len);

	BUG_ON(used > p->len);		BUG_ON(used > p->len);

	/* Pad the rest with nops */		/* Pad the rest with nops */
	nop_out(p->instr + used, p->len - used);		add_nops(insnbuf + used, p->len - used);
			text_poke(p->instr, insnbuf, p->len);
	}		}
	}		}
	extern struct paravirt_patch_site __start_parainstructions[],		extern struct paravirt_patch_site __start_parainstructions[],

arch/i386/kernel/paravirt.c

+26 −26

Original line number	Original line	Diff line number	Diff line
	@@ -69,7 +69,8 @@ DEF_NATIVE(read_tsc, "rdtsc");

	DEF_NATIVE(ud2a, "ud2a");		DEF_NATIVE(ud2a, "ud2a");

	static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)		static unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
			unsigned long addr, unsigned len)
	{		{
	const unsigned char start, end;		const unsigned char start, end;
	unsigned ret;		unsigned ret;
	@@ -90,7 +91,7 @@ static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)
	#undef SITE		#undef SITE

	patch_site:		patch_site:
	ret = paravirt_patch_insns(insns, len, start, end);		ret = paravirt_patch_insns(ibuf, len, start, end);
	break;		break;

	case PARAVIRT_PATCH(make_pgd):		case PARAVIRT_PATCH(make_pgd):
	@@ -107,7 +108,7 @@ static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)
	break;		break;

	default:		default:
	ret = paravirt_patch_default(type, clobbers, insns, len);		ret = paravirt_patch_default(type, clobbers, ibuf, addr, len);
	break;		break;
	}		}

	@@ -129,68 +130,67 @@ struct branch {
	u32 delta;		u32 delta;
	} __attribute__((packed));		} __attribute__((packed));

	unsigned paravirt_patch_call(void *target, u16 tgt_clobbers,		unsigned paravirt_patch_call(void *insnbuf,
	void *site, u16 site_clobbers,		const void *target, u16 tgt_clobbers,
			unsigned long addr, u16 site_clobbers,
	unsigned len)		unsigned len)
	{		{
	unsigned char *call = site;		struct branch *b = insnbuf;
	unsigned long delta = (unsigned long)target - (unsigned long)(call+5);		unsigned long delta = (unsigned long)target - (addr+5);
	struct branch b;

	if (tgt_clobbers & ~site_clobbers)		if (tgt_clobbers & ~site_clobbers)
	return len; /* target would clobber too much for this site */		return len; /* target would clobber too much for this site */
	if (len < 5)		if (len < 5)
	return len; /* call too long for patch site */		return len; /* call too long for patch site */

	b.opcode = 0xe8; /* call */		b->opcode = 0xe8; /* call */
	b.delta = delta;		b->delta = delta;
	BUILD_BUG_ON(sizeof(b) != 5);		BUILD_BUG_ON(sizeof(*b) != 5);
	text_poke(call, (unsigned char *)&b, 5);

	return 5;		return 5;
	}		}

	unsigned paravirt_patch_jmp(void target, void site, unsigned len)		unsigned paravirt_patch_jmp(const void target, void insnbuf,
			unsigned long addr, unsigned len)
	{		{
	unsigned char *jmp = site;		struct branch *b = insnbuf;
	unsigned long delta = (unsigned long)target - (unsigned long)(jmp+5);		unsigned long delta = (unsigned long)target - (addr+5);
	struct branch b;

	if (len < 5)		if (len < 5)
	return len; /* call too long for patch site */		return len; /* call too long for patch site */

	b.opcode = 0xe9; /* jmp */		b->opcode = 0xe9; /* jmp */
	b.delta = delta;		b->delta = delta;
	text_poke(jmp, (unsigned char *)&b, 5);

	return 5;		return 5;
	}		}

	unsigned paravirt_patch_default(u8 type, u16 clobbers, void *site, unsigned len)		unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
			unsigned long addr, unsigned len)
	{		{
	void opfunc = ((void **)&paravirt_ops + type);		void opfunc = ((void **)&paravirt_ops + type);
	unsigned ret;		unsigned ret;

	if (opfunc == NULL)		if (opfunc == NULL)
	/* If there's no function, patch it with a ud2a (BUG) */		/* If there's no function, patch it with a ud2a (BUG) */
	ret = paravirt_patch_insns(site, len, start_ud2a, end_ud2a);		ret = paravirt_patch_insns(insnbuf, len, start_ud2a, end_ud2a);
	else if (opfunc == paravirt_nop)		else if (opfunc == paravirt_nop)
	/* If the operation is a nop, then nop the callsite */		/* If the operation is a nop, then nop the callsite */
	ret = paravirt_patch_nop();		ret = paravirt_patch_nop();
	else if (type == PARAVIRT_PATCH(iret) \|\|		else if (type == PARAVIRT_PATCH(iret) \|\|
	type == PARAVIRT_PATCH(irq_enable_sysexit))		type == PARAVIRT_PATCH(irq_enable_sysexit))
	/* If operation requires a jmp, then jmp */		/* If operation requires a jmp, then jmp */
	ret = paravirt_patch_jmp(opfunc, site, len);		ret = paravirt_patch_jmp(opfunc, insnbuf, addr, len);
	else		else
	/* Otherwise call the function; assume target could		/* Otherwise call the function; assume target could
	clobber any caller-save reg */		clobber any caller-save reg */
	ret = paravirt_patch_call(opfunc, CLBR_ANY,		ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
	site, clobbers, len);		addr, clobbers, len);

	return ret;		return ret;
	}		}

	unsigned paravirt_patch_insns(void *site, unsigned len,		unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
	const char start, const char end)		const char start, const char end)
	{		{
	unsigned insn_len = end - start;		unsigned insn_len = end - start;
	@@ -198,7 +198,7 @@ unsigned paravirt_patch_insns(void *site, unsigned len,
	if (insn_len > len \|\| start == NULL)		if (insn_len > len \|\| start == NULL)
	insn_len = len;		insn_len = len;
	else		else
	memcpy(site, start, insn_len);		memcpy(insnbuf, start, insn_len);

	return insn_len;		return insn_len;
	}		}

arch/i386/kernel/vmi.c

+21 −14

Original line number	Original line	Diff line number	Diff line
	@@ -87,12 +87,14 @@ struct vmi_timer_ops vmi_timer_ops;
	#define IRQ_PATCH_INT_MASK 0		#define IRQ_PATCH_INT_MASK 0
	#define IRQ_PATCH_DISABLE 5		#define IRQ_PATCH_DISABLE 5

	static inline void patch_offset(unsigned char eip, unsigned char dest)		static inline void patch_offset(void *insnbuf,
			unsigned long eip, unsigned long dest)
	{		{
	(unsigned long )(eip+1) = dest-eip-5;		(unsigned long )(insnbuf+1) = dest-eip-5;
	}		}

	static unsigned patch_internal(int call, unsigned len, void *insns)		static unsigned patch_internal(int call, unsigned len, void *insnbuf,
			unsigned long eip)
	{		{
	u64 reloc;		u64 reloc;
	struct vmi_relocation_info const rel = (struct vmi_relocation_info )&reloc;		struct vmi_relocation_info const rel = (struct vmi_relocation_info )&reloc;
	@@ -100,14 +102,14 @@ static unsigned patch_internal(int call, unsigned len, void *insns)
	switch(rel->type) {		switch(rel->type) {
	case VMI_RELOCATION_CALL_REL:		case VMI_RELOCATION_CALL_REL:
	BUG_ON(len < 5);		BUG_ON(len < 5);
	(char )insns = MNEM_CALL;		(char )insnbuf = MNEM_CALL;
	patch_offset(insns, rel->eip);		patch_offset(insnbuf, eip, (unsigned long)rel->eip);
	return 5;		return 5;

	case VMI_RELOCATION_JUMP_REL:		case VMI_RELOCATION_JUMP_REL:
	BUG_ON(len < 5);		BUG_ON(len < 5);
	(char )insns = MNEM_JMP;		(char )insnbuf = MNEM_JMP;
	patch_offset(insns, rel->eip);		patch_offset(insnbuf, eip, (unsigned long)rel->eip);
	return 5;		return 5;

	case VMI_RELOCATION_NOP:		case VMI_RELOCATION_NOP:
	@@ -128,21 +130,26 @@ static unsigned patch_internal(int call, unsigned len, void *insns)
	* Apply patch if appropriate, return length of new instruction		* Apply patch if appropriate, return length of new instruction
	* sequence. The callee does nop padding for us.		* sequence. The callee does nop padding for us.
	*/		*/
	static unsigned vmi_patch(u8 type, u16 clobbers, void *insns, unsigned len)		static unsigned vmi_patch(u8 type, u16 clobbers, void *insns,
			unsigned long eip, unsigned len)
	{		{
	switch (type) {		switch (type) {
	case PARAVIRT_PATCH(irq_disable):		case PARAVIRT_PATCH(irq_disable):
	return patch_internal(VMI_CALL_DisableInterrupts, len, insns);		return patch_internal(VMI_CALL_DisableInterrupts, len,
			insns, eip);
	case PARAVIRT_PATCH(irq_enable):		case PARAVIRT_PATCH(irq_enable):
	return patch_internal(VMI_CALL_EnableInterrupts, len, insns);		return patch_internal(VMI_CALL_EnableInterrupts, len,
			insns, eip);
	case PARAVIRT_PATCH(restore_fl):		case PARAVIRT_PATCH(restore_fl):
	return patch_internal(VMI_CALL_SetInterruptMask, len, insns);		return patch_internal(VMI_CALL_SetInterruptMask, len,
			insns, eip);
	case PARAVIRT_PATCH(save_fl):		case PARAVIRT_PATCH(save_fl):
	return patch_internal(VMI_CALL_GetInterruptMask, len, insns);		return patch_internal(VMI_CALL_GetInterruptMask, len,
			insns, eip);
	case PARAVIRT_PATCH(iret):		case PARAVIRT_PATCH(iret):
	return patch_internal(VMI_CALL_IRET, len, insns);		return patch_internal(VMI_CALL_IRET, len, insns, eip);
	case PARAVIRT_PATCH(irq_enable_sysexit):		case PARAVIRT_PATCH(irq_enable_sysexit):
	return patch_internal(VMI_CALL_SYSEXIT, len, insns);		return patch_internal(VMI_CALL_SYSEXIT, len, insns, eip);
	default:		default:
	break;		break;
	}		}

arch/i386/xen/enlighten.c

+7 −5

Original line number	Original line	Diff line number	Diff line
	@@ -842,7 +842,8 @@ void __init xen_setup_vcpu_info_placement(void)
	}		}
	}		}

	static unsigned xen_patch(u8 type, u16 clobbers, void *insns, unsigned len)		static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
			unsigned long addr, unsigned len)
	{		{
	char start, end, *reloc;		char start, end, *reloc;
	unsigned ret;		unsigned ret;
	@@ -869,7 +870,7 @@ static unsigned xen_patch(u8 type, u16 clobbers, void *insns, unsigned len)
	if (start == NULL \|\| (end-start) > len)		if (start == NULL \|\| (end-start) > len)
	goto default_patch;		goto default_patch;

	ret = paravirt_patch_insns(insns, len, start, end);		ret = paravirt_patch_insns(insnbuf, len, start, end);

	/* Note: because reloc is assigned from something that		/* Note: because reloc is assigned from something that
	appears to be an array, gcc assumes it's non-null,		appears to be an array, gcc assumes it's non-null,
	@@ -877,8 +878,8 @@ static unsigned xen_patch(u8 type, u16 clobbers, void *insns, unsigned len)
	end. */		end. */
	if (reloc > start && reloc < end) {		if (reloc > start && reloc < end) {
	int reloc_off = reloc - start;		int reloc_off = reloc - start;
	long relocp = (long )(insns + reloc_off);		long relocp = (long )(insnbuf + reloc_off);
	long delta = start - (char *)insns;		long delta = start - (char *)addr;

	*relocp += delta;		*relocp += delta;
	}		}
	@@ -886,7 +887,8 @@ static unsigned xen_patch(u8 type, u16 clobbers, void *insns, unsigned len)

	default_patch:		default_patch:
	default:		default:
	ret = paravirt_patch_default(type, clobbers, insns, len);		ret = paravirt_patch_default(type, clobbers, insnbuf,
			addr, len);
	break;		break;
	}		}

drivers/lguest/lguest.c

+5 −4

Original line number	Original line	Diff line number	Diff line
	@@ -936,23 +936,24 @@ static const struct lguest_insns
	/* Now our patch routine is fairly simple (based on the native one in		/* Now our patch routine is fairly simple (based on the native one in
	* paravirt.c). If we have a replacement, we copy it in and return how much of		* paravirt.c). If we have a replacement, we copy it in and return how much of
	* the available space we used. */		* the available space we used. */
	static unsigned lguest_patch(u8 type, u16 clobber, void *insns, unsigned len)		static unsigned lguest_patch(u8 type, u16 clobber, void *ibuf,
			unsigned long addr, unsigned len)
	{		{
	unsigned int insn_len;		unsigned int insn_len;

	/* Don't do anything special if we don't have a replacement */		/* Don't do anything special if we don't have a replacement */
	if (type >= ARRAY_SIZE(lguest_insns) \|\| !lguest_insns[type].start)		if (type >= ARRAY_SIZE(lguest_insns) \|\| !lguest_insns[type].start)
	return paravirt_patch_default(type, clobber, insns, len);		return paravirt_patch_default(type, clobber, ibuf, addr, len);

	insn_len = lguest_insns[type].end - lguest_insns[type].start;		insn_len = lguest_insns[type].end - lguest_insns[type].start;

	/* Similarly if we can't fit replacement (shouldn't happen, but let's		/* Similarly if we can't fit replacement (shouldn't happen, but let's
	* be thorough). */		* be thorough). */
	if (len < insn_len)		if (len < insn_len)
	return paravirt_patch_default(type, clobber, insns, len);		return paravirt_patch_default(type, clobber, ibuf, addr, len);

	/* Copy in our instructions. */		/* Copy in our instructions. */
	memcpy(insns, lguest_insns[type].start, insn_len);		memcpy(ibuf, lguest_insns[type].start, insn_len);
	return insn_len;		return insn_len;
	}		}