Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 48e11198 authored by Dave Hansen's avatar Dave Hansen Committed by Ingo Molnar
Browse files

x86/mm: Abstract switching CR3 



In preparation to adding additional PCID flushing, abstract the
loading of a new ASID into CR3.

[ PeterZ: Split out from big combo patch ]

Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Greg KH <gregkh@linuxfoundation.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: aliguori@amazon.com
Cc: daniel.gruss@iaik.tugraz.at
Cc: hughd@google.com
Cc: keescook@google.com
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 2ea907c4

arch/x86/mm/tlb.c

+20 −2
Original line number Original line Diff line number Diff line
@@ -100,6 +100,24 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, 
	*need_flush = true;

	*need_flush = true;

}

}





static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)

{

	unsigned long new_mm_cr3;



	if (need_flush) {

		new_mm_cr3 = build_cr3(pgdir, new_asid);

	} else {

		new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);

	}



	/*

	 * Caution: many callers of this function expect

	 * that load_cr3() is serializing and orders TLB

	 * fills with respect to the mm_cpumask writes.

	 */

	write_cr3(new_mm_cr3);

}



void leave_mm(int cpu)

void leave_mm(int cpu)

{

{

	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);

	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
@@ -230,7 +248,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, 
		if (need_flush) {

		if (need_flush) {

			this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);

			this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);

			this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);

			this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);

			write_cr3(build_cr3(next->pgd, new_asid));

			load_new_mm_cr3(next->pgd, new_asid, true);





			/*

			/*

			 * NB: This gets called via leave_mm() in the idle path

			 * NB: This gets called via leave_mm() in the idle path
@@ -243,7 +261,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, 
			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);

			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);

		} else {

		} else {

			/* The new ASID is already up to date. */

			/* The new ASID is already up to date. */

			write_cr3(build_cr3_noflush(next->pgd, new_asid));

			load_new_mm_cr3(next->pgd, new_asid, false);





			/* See above wrt _rcuidle. */

			/* See above wrt _rcuidle. */

			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);

			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);