kvm: vmx: Support INVPCID in shadow paging mode (eb4b248e) · Commits · e / devices / android_kernel_fairphone_FP4

arch/x86/include/asm/kvm_host.h

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -1317,6 +1317,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
	int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,		int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,
	void *insn, int insn_len);		void *insn, int insn_len);
	void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);		void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
			void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
	void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3);		void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3);

	void kvm_enable_tdp(void);		void kvm_enable_tdp(void);

arch/x86/kvm/mmu.c

+18 −0

Original line number	Original line	Diff line number	Diff line
	@@ -5187,6 +5187,24 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
	}		}
	EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);		EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

			void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
			{
			struct kvm_mmu *mmu = &vcpu->arch.mmu;

			if (pcid == kvm_get_active_pcid(vcpu)) {
			mmu->invlpg(vcpu, gva);
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
			}

			++vcpu->stat.invlpg;

			/*
			* Mappings not reachable via the current cr3 will be synced when
			* switching to that cr3, so nothing needs to be done here for them.
			*/
			}
			EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva);

	void kvm_enable_tdp(void)		void kvm_enable_tdp(void)
	{		{
	tdp_enabled = true;		tdp_enabled = true;

arch/x86/kvm/vmx.c

+93 −3

Original line number	Original line	Diff line number	Diff line
	@@ -3065,7 +3065,7 @@ static bool vmx_rdtscp_supported(void)

	static bool vmx_invpcid_supported(void)		static bool vmx_invpcid_supported(void)
	{		{
	return cpu_has_vmx_invpcid() && enable_ept;		return cpu_has_vmx_invpcid();
	}		}

	/*		/*
	@@ -6101,8 +6101,6 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
	if (!enable_ept) {		if (!enable_ept) {
	exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;		exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
	enable_unrestricted_guest = 0;		enable_unrestricted_guest = 0;
	/* Enable INVPCID for non-ept guests may cause performance regression. */
	exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
	}		}
	if (!enable_unrestricted_guest)		if (!enable_unrestricted_guest)
	exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;		exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
	@@ -8756,6 +8754,97 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
	return kvm_skip_emulated_instruction(vcpu);		return kvm_skip_emulated_instruction(vcpu);
	}		}

			static int handle_invpcid(struct kvm_vcpu *vcpu)
			{
			u32 vmx_instruction_info;
			unsigned long type;
			bool pcid_enabled;
			gva_t gva;
			struct x86_exception e;
			struct {
			u64 pcid;
			u64 gla;
			} operand;

			if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
			kvm_queue_exception(vcpu, UD_VECTOR);
			return 1;
			}

			vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
			type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);

			if (type > 3) {
			kvm_inject_gp(vcpu, 0);
			return 1;
			}

			/* According to the Intel instruction reference, the memory operand
			* is read even if it isn't needed (e.g., for type==all)
			*/
			if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmx_instruction_info, false, &gva))
			return 1;

			if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
			kvm_inject_page_fault(vcpu, &e);
			return 1;
			}

			if (operand.pcid >> 12 != 0) {
			kvm_inject_gp(vcpu, 0);
			return 1;
			}

			pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

			switch (type) {
			case INVPCID_TYPE_INDIV_ADDR:
			if ((!pcid_enabled && (operand.pcid != 0)) \|\|
			is_noncanonical_address(operand.gla, vcpu)) {
			kvm_inject_gp(vcpu, 0);
			return 1;
			}
			kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
			return kvm_skip_emulated_instruction(vcpu);

			case INVPCID_TYPE_SINGLE_CTXT:
			if (!pcid_enabled && (operand.pcid != 0)) {
			kvm_inject_gp(vcpu, 0);
			return 1;
			}

			if (kvm_get_active_pcid(vcpu) == operand.pcid) {
			kvm_mmu_sync_roots(vcpu);
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
			}

			/*
			* If the current cr3 does not use the given PCID, then nothing
			* needs to be done here because a resync will happen anyway
			* before switching to any other CR3.
			*/

			return kvm_skip_emulated_instruction(vcpu);

			case INVPCID_TYPE_ALL_NON_GLOBAL:
			/*
			* Currently, KVM doesn't mark global entries in the shadow
			* page tables, so a non-global flush just degenerates to a
			* global flush. If needed, we could optimize this later by
			* keeping track of global entries in shadow page tables.
			*/

			/* fall-through */
			case INVPCID_TYPE_ALL_INCL_GLOBAL:
			kvm_mmu_unload(vcpu);
			return kvm_skip_emulated_instruction(vcpu);

			default:
			BUG(); /* We have already checked above that type <= 3 */
			}
			}

	static int handle_pml_full(struct kvm_vcpu *vcpu)		static int handle_pml_full(struct kvm_vcpu *vcpu)
	{		{
	unsigned long exit_qualification;		unsigned long exit_qualification;
	@@ -8959,6 +9048,7 @@ static int (const kvm_vmx_exit_handlers[])(struct kvm_vcpu vcpu) = {
	[EXIT_REASON_XSAVES] = handle_xsaves,		[EXIT_REASON_XSAVES] = handle_xsaves,
	[EXIT_REASON_XRSTORS] = handle_xrstors,		[EXIT_REASON_XRSTORS] = handle_xrstors,
	[EXIT_REASON_PML_FULL] = handle_pml_full,		[EXIT_REASON_PML_FULL] = handle_pml_full,
			[EXIT_REASON_INVPCID] = handle_invpcid,
	[EXIT_REASON_VMFUNC] = handle_vmfunc,		[EXIT_REASON_VMFUNC] = handle_vmfunc,
	[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,		[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
	};		};