Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ad218f85 authored by Marcelo Tosatti's avatar Marcelo Tosatti Committed by Avi Kivity
Browse files

KVM: MMU: prepopulate the shadow on invlpg 



If the guest executes invlpg, peek into the pagetable and attempt to
prepopulate the shadow entry.

Also stop dirty fault updates from interfering with the fork detector.

2% improvement on RHEL3/AIM7.

Signed-off-by: default avatarMarcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: default avatarAvi Kivity <avi@redhat.com>
parent 6cffe8ca

arch/x86/include/asm/kvm_host.h

+2 −1
Original line number Original line Diff line number Diff line
@@ -602,7 +602,8 @@ unsigned long segment_base(u16 selector); 




void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu);

void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu);

void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,

void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,

		       const u8 *new, int bytes);

		       const u8 *new, int bytes,

		       bool guest_initiated);

int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva);

int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva);

void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);

void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);

int kvm_mmu_load(struct kvm_vcpu *vcpu);

int kvm_mmu_load(struct kvm_vcpu *vcpu);

arch/x86/kvm/mmu.c

+13 −12
Original line number Original line Diff line number Diff line
@@ -2441,7 +2441,8 @@ static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn) 
}

}





void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,

void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,

		       const u8 *new, int bytes)

		       const u8 *new, int bytes,

		       bool guest_initiated)

{

{

	gfn_t gfn = gpa >> PAGE_SHIFT;

	gfn_t gfn = gpa >> PAGE_SHIFT;

	struct kvm_mmu_page *sp;

	struct kvm_mmu_page *sp;
@@ -2467,6 +2468,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 
	kvm_mmu_free_some_pages(vcpu);

	kvm_mmu_free_some_pages(vcpu);

	++vcpu->kvm->stat.mmu_pte_write;

	++vcpu->kvm->stat.mmu_pte_write;

	kvm_mmu_audit(vcpu, "pre pte write");

	kvm_mmu_audit(vcpu, "pre pte write");

	if (guest_initiated) {

		if (gfn == vcpu->arch.last_pt_write_gfn

		if (gfn == vcpu->arch.last_pt_write_gfn

		    && !last_updated_pte_accessed(vcpu)) {

		    && !last_updated_pte_accessed(vcpu)) {

			++vcpu->arch.last_pt_write_count;

			++vcpu->arch.last_pt_write_count;
@@ -2477,6 +2479,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 
			vcpu->arch.last_pt_write_count = 1;

			vcpu->arch.last_pt_write_count = 1;

			vcpu->arch.last_pte_updated = NULL;

			vcpu->arch.last_pte_updated = NULL;

		}

		}

	}

	index = kvm_page_table_hashfn(gfn);

	index = kvm_page_table_hashfn(gfn);

	bucket = &vcpu->kvm->arch.mmu_page_hash[index];

	bucket = &vcpu->kvm->arch.mmu_page_hash[index];

	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {

	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
@@ -2615,9 +2618,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); 




void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)

void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)

{

{

	spin_lock(&vcpu->kvm->mmu_lock);

	vcpu->arch.mmu.invlpg(vcpu, gva);

	vcpu->arch.mmu.invlpg(vcpu, gva);

	spin_unlock(&vcpu->kvm->mmu_lock);

	kvm_mmu_flush_tlb(vcpu);

	kvm_mmu_flush_tlb(vcpu);

	++vcpu->stat.invlpg;

	++vcpu->stat.invlpg;

}

}

arch/x86/kvm/paging_tmpl.h

+24 −1
Original line number Original line Diff line number Diff line
@@ -82,6 +82,7 @@ struct shadow_walker { 
	int *ptwrite;

	int *ptwrite;

	pfn_t pfn;

	pfn_t pfn;

	u64 *sptep;

	u64 *sptep;

	gpa_t pte_gpa;

};

};





static gfn_t gpte_to_gfn(pt_element_t gpte)

static gfn_t gpte_to_gfn(pt_element_t gpte)
@@ -222,7 +223,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, 
		if (ret)

		if (ret)

			goto walk;

			goto walk;

		pte |= PT_DIRTY_MASK;

		pte |= PT_DIRTY_MASK;

		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));

		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte), 0);

		walker->ptes[walker->level - 1] = pte;

		walker->ptes[walker->level - 1] = pte;

	}

	}
@@ -468,8 +469,15 @@ static int FNAME(shadow_invlpg_entry)(struct kvm_shadow_walk *_sw, 
				      struct kvm_vcpu *vcpu, u64 addr,

				      struct kvm_vcpu *vcpu, u64 addr,

				      u64 *sptep, int level)

				      u64 *sptep, int level)

{

{

	struct shadow_walker *sw =

		container_of(_sw, struct shadow_walker, walker);





	if (level == PT_PAGE_TABLE_LEVEL) {

	if (level == PT_PAGE_TABLE_LEVEL) {

		struct kvm_mmu_page *sp = page_header(__pa(sptep));



		sw->pte_gpa = (sp->gfn << PAGE_SHIFT);

		sw->pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t);



		if (is_shadow_present_pte(*sptep))

		if (is_shadow_present_pte(*sptep))

			rmap_remove(vcpu->kvm, sptep);

			rmap_remove(vcpu->kvm, sptep);

		set_shadow_pte(sptep, shadow_trap_nonpresent_pte);

		set_shadow_pte(sptep, shadow_trap_nonpresent_pte);
@@ -482,11 +490,26 @@ static int FNAME(shadow_invlpg_entry)(struct kvm_shadow_walk *_sw, 




static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)

static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)

{

{

	pt_element_t gpte;

	struct shadow_walker walker = {

	struct shadow_walker walker = {

		.walker = { .entry = FNAME(shadow_invlpg_entry), },

		.walker = { .entry = FNAME(shadow_invlpg_entry), },

		.pte_gpa = -1,

	};

	};





	spin_lock(&vcpu->kvm->mmu_lock);

	walk_shadow(&walker.walker, vcpu, gva);

	walk_shadow(&walker.walker, vcpu, gva);

	spin_unlock(&vcpu->kvm->mmu_lock);

	if (walker.pte_gpa == -1)

		return;

	if (kvm_read_guest_atomic(vcpu->kvm, walker.pte_gpa, &gpte,

				  sizeof(pt_element_t)))

		return;

	if (is_present_pte(gpte) && (gpte & PT_ACCESSED_MASK)) {

		if (mmu_topup_memory_caches(vcpu))

			return;

		kvm_mmu_pte_write(vcpu, walker.pte_gpa, (const u8 *)&gpte,

				  sizeof(pt_element_t), 0);

	}

}

}





static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr)

static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr)

arch/x86/kvm/x86.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -2046,7 +2046,7 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, 
	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);

	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);

	if (ret < 0)

	if (ret < 0)

		return 0;

		return 0;

	kvm_mmu_pte_write(vcpu, gpa, val, bytes);

	kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);

	return 1;

	return 1;

}

}