Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0a126abd authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

x86/mm: Clarify the whole ASID/kernel PCID/user PCID naming 



Ideally we'd also use sparse to enforce this separation so it becomes much
more difficult to mess up.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Greg KH <gregkh@linuxfoundation.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: aliguori@amazon.com
Cc: daniel.gruss@iaik.tugraz.at
Cc: hughd@google.com
Cc: keescook@google.com
Cc: linux-mm@kvack.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 6cff64b8

arch/x86/include/asm/tlbflush.h

+43 −12
Original line number Diff line number Diff line
@@ -13,16 +13,33 @@ 
#include <asm/pti.h>

#include <asm/processor-flags.h>



static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)

{

/*

	 * Bump the generation count.  This also serves as a full barrier

	 * that synchronizes with switch_mm(): callers are required to order

	 * their read of mm_cpumask after their writes to the paging

	 * structures.

 * The x86 feature is called PCID (Process Context IDentifier). It is similar

 * to what is traditionally called ASID on the RISC processors.

 *

 * We don't use the traditional ASID implementation, where each process/mm gets

 * its own ASID and flush/restart when we run out of ASID space.

 *

 * Instead we have a small per-cpu array of ASIDs and cache the last few mm's

 * that came by on this CPU, allowing cheaper switch_mm between processes on

 * this CPU.

 *

 * We end up with different spaces for different things. To avoid confusion we

 * use different names for each of them:

 *

 * ASID  - [0, TLB_NR_DYN_ASIDS-1]

 *         the canonical identifier for an mm

 *

 * kPCID - [1, TLB_NR_DYN_ASIDS]

 *         the value we write into the PCID part of CR3; corresponds to the

 *         ASID+1, because PCID 0 is special.

 *

 * uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS]

 *         for KPTI each mm has two address spaces and thus needs two

 *         PCID values, but we can still do with a single ASID denomination

 *         for each mm. Corresponds to kPCID + 2048.

 *

 */

	return atomic64_inc_return(&mm->context.tlb_gen);

}



/* There are 12 bits of space for ASIDS in CR3 */

#define CR3_HW_ASID_BITS		12
@@ -41,7 +58,7 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) 


/*

 * ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid.  -1 below to account

 * for them being zero-based.  Another -1 is because ASID 0 is reserved for

 * for them being zero-based.  Another -1 is because PCID 0 is reserved for

 * use by non-PCID-aware users.

 */

#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2)
@@ -52,6 +69,9 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) 
 */

#define TLB_NR_DYN_ASIDS	6



/*

 * Given @asid, compute kPCID

 */

static inline u16 kern_pcid(u16 asid)

{

	VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
@@ -86,7 +106,7 @@ static inline u16 kern_pcid(u16 asid) 
}



/*

 * The user PCID is just the kernel one, plus the "switch bit".

 * Given @asid, compute uPCID

 */

static inline u16 user_pcid(u16 asid)

{
@@ -484,6 +504,17 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a) 
void native_flush_tlb_others(const struct cpumask *cpumask,

			     const struct flush_tlb_info *info);



static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)

{

	/*

	 * Bump the generation count.  This also serves as a full barrier

	 * that synchronizes with switch_mm(): callers are required to order

	 * their read of mm_cpumask after their writes to the paging

	 * structures.

	 */

	return atomic64_inc_return(&mm->context.tlb_gen);

}



static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch,

					struct mm_struct *mm)

{