Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2b047252 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Fix TLB gather virtual address range invalidation corner cases 

Ben Tebulin reported:

 "Since v3.7.2 on two independent machines a very specific Git
  repository fails in 9/10 cases on git-fsck due to an SHA1/memory
  failures.  This only occurs on a very specific repository and can be
  reproduced stably on two independent laptops.  Git mailing list ran
  out of ideas and for me this looks like some very exotic kernel issue"

and bisected the failure to the backport of commit 53a59fc6 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").

That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.

The real bug is that the TLB gather virtual memory range setup is subtly
buggered.  It was introduced in commit 597e1c35 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range...
parent f1d6e17f

arch/arm/include/asm/tlb.h

+5 −2
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ struct mmu_gather { 
	struct mm_struct	*mm;

	unsigned int		fullmm;

	struct vm_area_struct	*vma;

	unsigned long		start, end;

	unsigned long		range_start;

	unsigned long		range_end;

	unsigned int		nr;
@@ -107,10 +108,12 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) 
}



static inline void

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int fullmm)

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)

{

	tlb->mm = mm;

	tlb->fullmm = fullmm;

	tlb->fullmm = !(start | (end+1));

	tlb->start = start;

	tlb->end = end;

	tlb->vma = NULL;

	tlb->max = ARRAY_SIZE(tlb->local);

	tlb->pages = tlb->local;

arch/arm64/include/asm/tlb.h

+5 −2
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ struct mmu_gather { 
	struct mm_struct	*mm;

	unsigned int		fullmm;

	struct vm_area_struct	*vma;

	unsigned long		start, end;

	unsigned long		range_start;

	unsigned long		range_end;

	unsigned int		nr;
@@ -97,10 +98,12 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) 
}



static inline void

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int fullmm)

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)

{

	tlb->mm = mm;

	tlb->fullmm = fullmm;

	tlb->fullmm = !(start | (end+1));

	tlb->start = start;

	tlb->end = end;

	tlb->vma = NULL;

	tlb->max = ARRAY_SIZE(tlb->local);

	tlb->pages = tlb->local;

arch/ia64/include/asm/tlb.h

+6 −3
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ 
 * unmapping a portion of the virtual address space, these hooks are called according to

 * the following template:

 *

 *	tlb <- tlb_gather_mmu(mm, full_mm_flush);	// start unmap for address space MM

 *	tlb <- tlb_gather_mmu(mm, start, end);		// start unmap for address space MM

 *	{

 *	  for each vma that needs a shootdown do {

 *	    tlb_start_vma(tlb, vma);
@@ -58,6 +58,7 @@ struct mmu_gather { 
	unsigned int		max;

	unsigned char		fullmm;		/* non-zero means full mm flush */

	unsigned char		need_flush;	/* really unmapped some PTEs? */

	unsigned long		start, end;

	unsigned long		start_addr;

	unsigned long		end_addr;

	struct page		**pages;
@@ -155,13 +156,15 @@ static inline void __tlb_alloc_page(struct mmu_gather *tlb) 




static inline void

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int full_mm_flush)

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)

{

	tlb->mm = mm;

	tlb->max = ARRAY_SIZE(tlb->local);

	tlb->pages = tlb->local;

	tlb->nr = 0;

	tlb->fullmm = full_mm_flush;

	tlb->fullmm = !(start | (end+1));

	tlb->start = start;

	tlb->end = end;

	tlb->start_addr = ~0UL;

}

arch/s390/include/asm/tlb.h

+6 −2
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ struct mmu_gather { 
	struct mm_struct *mm;

	struct mmu_table_batch *batch;

	unsigned int fullmm;

	unsigned long start, unsigned long end;

};



struct mmu_table_batch {
@@ -48,10 +49,13 @@ extern void tlb_remove_table(struct mmu_gather *tlb, void *table); 


static inline void tlb_gather_mmu(struct mmu_gather *tlb,

				  struct mm_struct *mm,

				  unsigned int full_mm_flush)

				  unsigned long start,

				  unsigned long end)

{

	tlb->mm = mm;

	tlb->fullmm = full_mm_flush;

	tlb->start = start;

	tlb->end = end;

	tlb->fullmm = !(start | (end+1));

	tlb->batch = NULL;

	if (tlb->fullmm)

		__tlb_flush_mm(mm);

arch/sh/include/asm/tlb.h

+4 −2
Original line number Diff line number Diff line
@@ -36,10 +36,12 @@ static inline void init_tlb_gather(struct mmu_gather *tlb) 
}



static inline void

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int full_mm_flush)

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)

{

	tlb->mm = mm;

	tlb->fullmm = full_mm_flush;

	tlb->start = start;

	tlb->end = end;

	tlb->fullmm = !(start | (end+1));



	init_tlb_gather(tlb);

}