KVM: Move the memslot update in-progress flag to bit 63

out-of-date information, but with an up-to-date generation number.

To avoid this, the generation number is incremented again after synchronize_srcu

returns; thus, the low bit of kvm_memslots(kvm)->generation is only 1 during a

returns; thus, bit 63 of kvm_memslots(kvm)->generation set to 1 only during a

memslot update, while some SRCU readers might be using the old copy.  We do not

want to use an MMIO sptes created with an odd generation number, and we can do

this without losing a bit in the MMIO spte.  The low bit of the generation

is not stored in MMIO spte, and presumed zero when it is extracted out of the

spte.  If KVM is unlucky and creates an MMIO spte while the low bit is 1,

the next access to the spte will always be a cache miss.

this without losing a bit in the MMIO spte.  The "update in-progress" bit of the

generation is not stored in MMIO spte, and is so is implicitly zero when the

generation is extracted out of the spte.  If KVM is unlucky and creates an MMIO

spte while an update is in-progress, the next access to the spte will always be

a cache miss.  For example, a subsequent access during the update window will

miss due to the in-progress flag diverging, while an access after the update

window closes will have a higher generation number (as compared to the spte).

Further reading

 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of

 * the memslots generation and is derived as follows:

 *

 * Bits 1-9 of the memslot generation are propagated to spte bits 3-11

 * Bits 10-19 of the memslot generation are propagated to spte bits 52-61

 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11

 * Bits 9-18 of the MMIO generation are propagated to spte bits 52-61

 *

 * The MMIO generation starts at bit 1 of the memslots generation in order to

 * skip over bit 0, the KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag.  Including

 * the flag would require stealing a bit from the "real" generation number and

 * thus effectively halve the maximum number of MMIO generations that can be

 * handled before encountering a wrap (which requires a full MMU zap).  The

 * flag is instead explicitly queried when checking for MMIO spte cache hits.

 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in

 * the MMIO generation number, as doing so would require stealing a bit from

 * the "real" generation number and thus effectively halve the maximum number

 * of MMIO generations that can be handled before encountering a wrap (which

 * requires a full MMU zap).  The flag is instead explicitly queried when

 * checking for MMIO spte cache hits.

 */

#define MMIO_SPTE_GEN_MASK		GENMASK_ULL(19, 1)

#define MMIO_SPTE_GEN_SHIFT		1

#define MMIO_SPTE_GEN_MASK		GENMASK_ULL(18, 0)

#define MMIO_SPTE_GEN_LOW_START		3

#define MMIO_SPTE_GEN_LOW_END		11

	WARN_ON(gen & ~MMIO_SPTE_GEN_MASK);

	gen >>= MMIO_SPTE_GEN_SHIFT;

	mask = (gen << MMIO_SPTE_GEN_LOW_START) & MMIO_SPTE_GEN_LOW_MASK;

	mask |= (gen << MMIO_SPTE_GEN_HIGH_START) & MMIO_SPTE_GEN_HIGH_MASK;

	return mask;

	gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_START;

	gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_START;

	return gen << MMIO_SPTE_GEN_SHIFT;

	return gen;

}

static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,

void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)

{

	gen &= MMIO_SPTE_GEN_MASK;

	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);

	/*

	 * Shift to adjust for the "update in-progress" flag, which isn't

	 * included in the MMIO generation number.

	 */

	gen >>= MMIO_SPTE_GEN_SHIFT;

	gen &= MMIO_SPTE_GEN_MASK;

	/*

	 * Generation numbers are incremented in multiples of the number of

#define KVM_MEMSLOT_INVALID	(1UL << 16)

/*

 * Bit 0 of the memslot generation number is an "update in-progress flag",

 * Bit 63 of the memslot generation number is an "update in-progress flag",

 * e.g. is temporarily set for the duration of install_new_memslots().

 * This flag effectively creates a unique generation number that is used to

 * mark cached memslot data, e.g. MMIO accesses, as potentially being stale,

 * the actual generation number against accesses that were inserted into the

 * cache *before* the memslots were updated.

 */

#define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS	BIT_ULL(0)

#define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS	BIT_ULL(63)

/* Two fragments for cross MMIO pages. */

#define KVM_MAX_MMIO_FRAGMENTS	2

		if (!slots)

			goto out_err_no_srcu;

		/* Generations must be different for each address space. */

		slots->generation = i * 2;

		slots->generation = i;

		rcu_assign_pointer(kvm->memslots[i], slots);

	}

	 * Generations must be unique even across address spaces.  We do not need

	 * a global counter for that, instead the generation space is evenly split

	 * across address spaces.  For example, with two address spaces, address

	 * space 0 will use generations 0, 4, 8, ... while address space 1 will

	 * use generations 2, 6, 10, 14, ...

	 * space 0 will use generations 0, 2, 4, ... while address space 1 will

	 * use generations 1, 3, 5, ...

	 */

	gen += KVM_ADDRESS_SPACE_NUM * 2;

	gen += KVM_ADDRESS_SPACE_NUM;

	kvm_arch_memslots_updated(kvm, gen);