Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5fdbf976 authored by Marcelo Tosatti's avatar Marcelo Tosatti Committed by Avi Kivity
Browse files

KVM: x86: accessors for guest registers 



As suggested by Avi, introduce accessors to read/write guest registers.
This simplifies the ->cache_regs/->decache_regs interface, and improves
register caching which is important for VMX, where the cost of
vmcs_read/vmcs_write is significant.

[avi: fix warnings]

Signed-off-by: default avatarMarcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: default avatarAvi Kivity <avi@qumranet.com>
parent ca60dfbb

arch/x86/kvm/kvm_cache_regs.h

0 → 100644
+32 −0
Original line number Diff line number Diff line 
#ifndef ASM_KVM_CACHE_REGS_H

#define ASM_KVM_CACHE_REGS_H



static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu,

					      enum kvm_reg reg)

{

	if (!test_bit(reg, (unsigned long *)&vcpu->arch.regs_avail))

		kvm_x86_ops->cache_reg(vcpu, reg);



	return vcpu->arch.regs[reg];

}



static inline void kvm_register_write(struct kvm_vcpu *vcpu,

				      enum kvm_reg reg,

				      unsigned long val)

{

	vcpu->arch.regs[reg] = val;

	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty);

	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);

}



static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu)

{

	return kvm_register_read(vcpu, VCPU_REGS_RIP);

}



static inline void kvm_rip_write(struct kvm_vcpu *vcpu, unsigned long val)

{

	kvm_register_write(vcpu, VCPU_REGS_RIP, val);

}



#endif

arch/x86/kvm/lapic.c

+2 −2
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ 
#include <asm/current.h>

#include <asm/apicdef.h>

#include <asm/atomic.h>

#include "kvm_cache_regs.h"

#include "irq.h"



#define PRId64 "d"
@@ -558,8 +559,7 @@ static void __report_tpr_access(struct kvm_lapic *apic, bool write) 
	struct kvm_run *run = vcpu->run;



	set_bit(KVM_REQ_REPORT_TPR_ACCESS, &vcpu->requests);

	kvm_x86_ops->cache_regs(vcpu);

	run->tpr_access.rip = vcpu->arch.rip;

	run->tpr_access.rip = kvm_rip_read(vcpu);

	run->tpr_access.is_write = write;

}

arch/x86/kvm/svm.c

+23 −33
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include "kvm_svm.h"

#include "irq.h"

#include "mmu.h"

#include "kvm_cache_regs.h"



#include <linux/module.h>

#include <linux/kernel.h>
@@ -236,13 +237,11 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) 
		printk(KERN_DEBUG "%s: NOP\n", __func__);

		return;

	}

	if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE)

		printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n",

		       __func__,

		       svm->vmcb->save.rip,

		       svm->next_rip);

	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)

		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",

		       __func__, kvm_rip_read(vcpu), svm->next_rip);



	vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip;

	kvm_rip_write(vcpu, svm->next_rip);

	svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;



	vcpu->arch.interrupt_window_open = 1;
@@ -581,6 +580,7 @@ static void init_vmcb(struct vcpu_svm *svm) 
	save->dr7 = 0x400;

	save->rflags = 2;

	save->rip = 0x0000fff0;

	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;



	/*

	 * cr0 val on cpu init should be 0x60000010, we enable cpu
@@ -615,10 +615,12 @@ static int svm_vcpu_reset(struct kvm_vcpu *vcpu) 
	init_vmcb(svm);



	if (vcpu->vcpu_id != 0) {

		svm->vmcb->save.rip = 0;

		kvm_rip_write(vcpu, 0);

		svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;

		svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;

	}

	vcpu->arch.regs_avail = ~0;

	vcpu->arch.regs_dirty = ~0;



	return 0;

}
@@ -721,23 +723,6 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) 
	rdtscll(vcpu->arch.host_tsc);

}



static void svm_cache_regs(struct kvm_vcpu *vcpu)

{

	struct vcpu_svm *svm = to_svm(vcpu);



	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;

	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;

	vcpu->arch.rip = svm->vmcb->save.rip;

}



static void svm_decache_regs(struct kvm_vcpu *vcpu)

{

	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];

	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];

	svm->vmcb->save.rip = vcpu->arch.rip;

}



static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)

{

	return to_svm(vcpu)->vmcb->save.rflags;
@@ -1139,14 +1124,14 @@ static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) 


static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)

{

	svm->next_rip = svm->vmcb->save.rip + 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;

	skip_emulated_instruction(&svm->vcpu);

	return kvm_emulate_halt(&svm->vcpu);

}



static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)

{

	svm->next_rip = svm->vmcb->save.rip + 3;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;

	skip_emulated_instruction(&svm->vcpu);

	kvm_emulate_hypercall(&svm->vcpu);

	return 1;
@@ -1178,7 +1163,7 @@ static int task_switch_interception(struct vcpu_svm *svm, 


static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)

{

	svm->next_rip = svm->vmcb->save.rip + 2;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;

	kvm_emulate_cpuid(&svm->vcpu);

	return 1;

}
@@ -1273,9 +1258,9 @@ static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) 
		KVMTRACE_3D(MSR_READ, &svm->vcpu, ecx, (u32)data,

			    (u32)(data >> 32), handler);



		svm->vmcb->save.rax = data & 0xffffffff;

		svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;

		svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;

		svm->next_rip = svm->vmcb->save.rip + 2;

		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;

		skip_emulated_instruction(&svm->vcpu);

	}

	return 1;
@@ -1359,13 +1344,13 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) 
static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)

{

	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];

	u64 data = (svm->vmcb->save.rax & -1u)

	u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)

		| ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);



	KVMTRACE_3D(MSR_WRITE, &svm->vcpu, ecx, (u32)data, (u32)(data >> 32),

		    handler);



	svm->next_rip = svm->vmcb->save.rip + 2;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;

	if (svm_set_msr(&svm->vcpu, ecx, data))

		kvm_inject_gp(&svm->vcpu, 0);

	else
@@ -1723,6 +1708,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	u16 gs_selector;

	u16 ldt_selector;



	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];

	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];

	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];



	pre_svm_run(svm);



	sync_lapic_to_cr8(vcpu);
@@ -1858,6 +1847,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
		load_db_regs(svm->host_db_regs);



	vcpu->arch.cr2 = svm->vmcb->save.cr2;

	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;

	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;

	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;



	write_dr6(svm->host_dr6);

	write_dr7(svm->host_dr7);
@@ -1977,8 +1969,6 @@ static struct kvm_x86_ops svm_x86_ops = { 
	.set_gdt = svm_set_gdt,

	.get_dr = svm_get_dr,

	.set_dr = svm_set_dr,

	.cache_regs = svm_cache_regs,

	.decache_regs = svm_decache_regs,

	.get_rflags = svm_get_rflags,

	.set_rflags = svm_set_rflags,

arch/x86/kvm/vmx.c

+49 −54
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
#include <linux/highmem.h>

#include <linux/sched.h>

#include <linux/moduleparam.h>

#include "kvm_cache_regs.h"



#include <asm/io.h>

#include <asm/desc.h>
@@ -715,9 +716,9 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) 
	unsigned long rip;

	u32 interruptibility;



	rip = vmcs_readl(GUEST_RIP);

	rip = kvm_rip_read(vcpu);

	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);

	vmcs_writel(GUEST_RIP, rip);

	kvm_rip_write(vcpu, rip);



	/*

	 * We emulated an instruction, so temporary interrupt blocking
@@ -947,24 +948,19 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 
	return ret;

}



/*

 * Sync the rsp and rip registers into the vcpu structure.  This allows

 * registers to be accessed by indexing vcpu->arch.regs.

 */

static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu)

static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)

{

	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);

	switch (reg) {

	case VCPU_REGS_RSP:

		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);

	vcpu->arch.rip = vmcs_readl(GUEST_RIP);

		break;

	case VCPU_REGS_RIP:

		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);

		break;

	default:

		break;

	}



/*

 * Syncs rsp and rip back into the vmcs.  Should be called after possible

 * modification.

 */

static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu)

{

	vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);

	vmcs_writel(GUEST_RIP, vcpu->arch.rip);

}



static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
@@ -2019,6 +2015,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) 
	u64 msr;

	int ret;



	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));

	down_read(&vcpu->kvm->slots_lock);

	if (!init_rmode(vmx->vcpu.kvm)) {

		ret = -ENOMEM;
@@ -2072,10 +2069,10 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) 


	vmcs_writel(GUEST_RFLAGS, 0x02);

	if (vmx->vcpu.vcpu_id == 0)

		vmcs_writel(GUEST_RIP, 0xfff0);

		kvm_rip_write(vcpu, 0xfff0);

	else

		vmcs_writel(GUEST_RIP, 0);

	vmcs_writel(GUEST_RSP, 0);

		kvm_rip_write(vcpu, 0);

	kvm_register_write(vcpu, VCPU_REGS_RSP, 0);



	/* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */

	vmcs_writel(GUEST_DR7, 0x400);
@@ -2139,11 +2136,11 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) 
	if (vcpu->arch.rmode.active) {

		vmx->rmode.irq.pending = true;

		vmx->rmode.irq.vector = irq;

		vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP);

		vmx->rmode.irq.rip = kvm_rip_read(vcpu);

		vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,

			     irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);

		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);

		vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip - 1);

		kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);

		return;

	}

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
@@ -2288,7 +2285,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	}



	error_code = 0;

	rip = vmcs_readl(GUEST_RIP);

	rip = kvm_rip_read(vcpu);

	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)

		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);

	if (is_page_fault(intr_info)) {
@@ -2386,27 +2383,25 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	reg = (exit_qualification >> 8) & 15;

	switch ((exit_qualification >> 4) & 3) {

	case 0: /* mov to cr */

		KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr, (u32)vcpu->arch.regs[reg],

			    (u32)((u64)vcpu->arch.regs[reg] >> 32), handler);

		KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr,

			    (u32)kvm_register_read(vcpu, reg),

			    (u32)((u64)kvm_register_read(vcpu, reg) >> 32),

			    handler);

		switch (cr) {

		case 0:

			vcpu_load_rsp_rip(vcpu);

			kvm_set_cr0(vcpu, vcpu->arch.regs[reg]);

			kvm_set_cr0(vcpu, kvm_register_read(vcpu, reg));

			skip_emulated_instruction(vcpu);

			return 1;

		case 3:

			vcpu_load_rsp_rip(vcpu);

			kvm_set_cr3(vcpu, vcpu->arch.regs[reg]);

			kvm_set_cr3(vcpu, kvm_register_read(vcpu, reg));

			skip_emulated_instruction(vcpu);

			return 1;

		case 4:

			vcpu_load_rsp_rip(vcpu);

			kvm_set_cr4(vcpu, vcpu->arch.regs[reg]);

			kvm_set_cr4(vcpu, kvm_register_read(vcpu, reg));

			skip_emulated_instruction(vcpu);

			return 1;

		case 8:

			vcpu_load_rsp_rip(vcpu);

			kvm_set_cr8(vcpu, vcpu->arch.regs[reg]);

			kvm_set_cr8(vcpu, kvm_register_read(vcpu, reg));

			skip_emulated_instruction(vcpu);

			if (irqchip_in_kernel(vcpu->kvm))

				return 1;
@@ -2415,7 +2410,6 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
		};

		break;

	case 2: /* clts */

		vcpu_load_rsp_rip(vcpu);

		vmx_fpu_deactivate(vcpu);

		vcpu->arch.cr0 &= ~X86_CR0_TS;

		vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
@@ -2426,21 +2420,17 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	case 1: /*mov from cr*/

		switch (cr) {

		case 3:

			vcpu_load_rsp_rip(vcpu);

			vcpu->arch.regs[reg] = vcpu->arch.cr3;

			vcpu_put_rsp_rip(vcpu);

			kvm_register_write(vcpu, reg, vcpu->arch.cr3);

			KVMTRACE_3D(CR_READ, vcpu, (u32)cr,

				    (u32)vcpu->arch.regs[reg],

				    (u32)((u64)vcpu->arch.regs[reg] >> 32),

				    (u32)kvm_register_read(vcpu, reg),

				    (u32)((u64)kvm_register_read(vcpu, reg) >> 32),

				    handler);

			skip_emulated_instruction(vcpu);

			return 1;

		case 8:

			vcpu_load_rsp_rip(vcpu);

			vcpu->arch.regs[reg] = kvm_get_cr8(vcpu);

			vcpu_put_rsp_rip(vcpu);

			kvm_register_write(vcpu, reg, kvm_get_cr8(vcpu));

			KVMTRACE_2D(CR_READ, vcpu, (u32)cr,

				    (u32)vcpu->arch.regs[reg], handler);

				    (u32)kvm_register_read(vcpu, reg), handler);

			skip_emulated_instruction(vcpu);

			return 1;

		}
@@ -2472,7 +2462,6 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	dr = exit_qualification & 7;

	reg = (exit_qualification >> 8) & 15;

	vcpu_load_rsp_rip(vcpu);

	if (exit_qualification & 16) {

		/* mov from dr */

		switch (dr) {
@@ -2485,12 +2474,11 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
		default:

			val = 0;

		}

		vcpu->arch.regs[reg] = val;

		kvm_register_write(vcpu, reg, val);

		KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);

	} else {

		/* mov to dr */

	}

	vcpu_put_rsp_rip(vcpu);

	skip_emulated_instruction(vcpu);

	return 1;

}
@@ -2735,8 +2723,8 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	u32 vectoring_info = vmx->idt_vectoring_info;



	KVMTRACE_3D(VMEXIT, vcpu, exit_reason, (u32)vmcs_readl(GUEST_RIP),

		    (u32)((u64)vmcs_readl(GUEST_RIP) >> 32), entryexit);

	KVMTRACE_3D(VMEXIT, vcpu, exit_reason, (u32)kvm_rip_read(vcpu),

		    (u32)((u64)kvm_rip_read(vcpu) >> 32), entryexit);



	/* Access CR3 don't cause VMExit in paging mode, so we need

	 * to sync with guest real CR3. */
@@ -2922,9 +2910,9 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) 
static void fixup_rmode_irq(struct vcpu_vmx *vmx)

{

	vmx->rmode.irq.pending = 0;

	if (vmcs_readl(GUEST_RIP) + 1 != vmx->rmode.irq.rip)

	if (kvm_rip_read(&vmx->vcpu) + 1 != vmx->rmode.irq.rip)

		return;

	vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip);

	kvm_rip_write(&vmx->vcpu, vmx->rmode.irq.rip);

	if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {

		vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK;

		vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR;
@@ -2941,6 +2929,11 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	u32 intr_info;



	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))

		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);

	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))

		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);



	/*

	 * Loading guest fpu may have cleared host cr0.ts

	 */
@@ -3061,6 +3054,9 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 
#endif

	      );



	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));

	vcpu->arch.regs_dirty = 0;



	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

	if (vmx->rmode.irq.pending)

		fixup_rmode_irq(vmx);
@@ -3224,8 +3220,7 @@ static struct kvm_x86_ops vmx_x86_ops = { 
	.set_idt = vmx_set_idt,

	.get_gdt = vmx_get_gdt,

	.set_gdt = vmx_set_gdt,

	.cache_regs = vcpu_load_rsp_rip,

	.decache_regs = vcpu_put_rsp_rip,

	.cache_reg = vmx_cache_reg,

	.get_rflags = vmx_get_rflags,

	.set_rflags = vmx_set_rflags,

arch/x86/kvm/x86.c

+138 −130

File changed.

Preview size limit exceeded, changes collapsed.