Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 962734fc authored by Christian Borntraeger's avatar Christian Borntraeger
Browse files

Merge branch 'tlb-flushing' of... 

Merge branch 'tlb-flushing' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux  into kernelorgnext

base patches for tlb flushing optimizations
parents 520eccdf cd774b90

arch/s390/include/asm/page-states.h

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#define ESSA_SET_POT_VOLATILE		4

#define ESSA_SET_STABLE_RESIDENT	5

#define ESSA_SET_STABLE_IF_RESIDENT	6

#define ESSA_SET_STABLE_NODAT		7



#define ESSA_MAX	ESSA_SET_STABLE_IF_RESIDENT

arch/s390/include/asm/page.h

+3 −0
Original line number Diff line number Diff line
@@ -133,6 +133,9 @@ static inline int page_reset_referenced(unsigned long addr) 
struct page;

void arch_free_page(struct page *page, int order);

void arch_alloc_page(struct page *page, int order);

void arch_set_page_dat(struct page *page, int order);

void arch_set_page_nodat(struct page *page, int order);

int arch_test_page_nodat(struct page *page);

void arch_set_page_states(int make_stable);



static inline int devmem_is_allowed(unsigned long pfn)

arch/s390/include/asm/pgtable.h

+67 −21
Original line number Diff line number Diff line
@@ -376,6 +376,7 @@ static inline int is_module_addr(void *addr) 


/* Guest Page State used for virtualization */

#define _PGSTE_GPS_ZERO			0x0000000080000000UL

#define _PGSTE_GPS_NODAT		0x0000000040000000UL

#define _PGSTE_GPS_USAGE_MASK		0x0000000003000000UL

#define _PGSTE_GPS_USAGE_STABLE		0x0000000000000000UL

#define _PGSTE_GPS_USAGE_UNUSED		0x0000000001000000UL
@@ -952,15 +953,30 @@ static inline pte_t pte_mkhuge(pte_t pte) 
#define IPTE_GLOBAL	0

#define	IPTE_LOCAL	1



static inline void __ptep_ipte(unsigned long address, pte_t *ptep, int local)

#define IPTE_NODAT	0x400

#define IPTE_GUEST_ASCE	0x800



static inline void __ptep_ipte(unsigned long address, pte_t *ptep,

			       unsigned long opt, unsigned long asce,

			       int local)

{

	unsigned long pto = (unsigned long) ptep;



	if (__builtin_constant_p(opt) && opt == 0) {

		/* Invalidation + TLB flush for the pte */

		asm volatile(

			"	.insn	rrf,0xb2210000,%[r1],%[r2],0,%[m4]"

			: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),

			  [m4] "i" (local));

		return;

	}



	/* Invalidate ptes with options + TLB flush of the ptes */

	opt = opt | (asce & _ASCE_ORIGIN);

	asm volatile(

		"	.insn	rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"

		: [r2] "+a" (address), [r3] "+a" (opt)

		: [r1] "a" (pto), [m4] "i" (local) : "memory");

}



static inline void __ptep_ipte_range(unsigned long address, int nr,
@@ -1341,31 +1357,61 @@ static inline void __pmdp_csp(pmd_t *pmdp) 
#define IDTE_GLOBAL	0

#define IDTE_LOCAL	1



static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local)

#define IDTE_PTOA	0x0800

#define IDTE_NODAT	0x1000

#define IDTE_GUEST_ASCE	0x2000



static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,

			       unsigned long opt, unsigned long asce,

			       int local)

{

	unsigned long sto;



	sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);

	sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);

	if (__builtin_constant_p(opt) && opt == 0) {

		/* flush without guest asce */

		asm volatile(

			"	.insn	rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"

			: "+m" (*pmdp)

		: [r1] "a" (sto), [r2] "a" ((address & HPAGE_MASK)),

			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),

			  [m4] "i" (local)

			: "cc" );

	} else {

		/* flush with guest asce */

		asm volatile(

			"	.insn	rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"

			: "+m" (*pmdp)

			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),

			  [r3] "a" (asce), [m4] "i" (local)

			: "cc" );

	}

}



static inline void __pudp_idte(unsigned long address, pud_t *pudp, int local)

static inline void __pudp_idte(unsigned long addr, pud_t *pudp,

			       unsigned long opt, unsigned long asce,

			       int local)

{

	unsigned long r3o;



	r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);

	r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);

	r3o |= _ASCE_TYPE_REGION3;

	if (__builtin_constant_p(opt) && opt == 0) {

		/* flush without guest asce */

		asm volatile(

			"	.insn	rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"

			: "+m" (*pudp)

		: [r1] "a" (r3o), [r2] "a" ((address & PUD_MASK)),

			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),

			  [m4] "i" (local)

			: "cc");

	} else {

		/* flush with guest asce */

		asm volatile(

			"	.insn	rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"

			: "+m" (*pudp)

			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),

			  [r3] "a" (asce), [m4] "i" (local)

			: "cc" );

	}

}



pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

arch/s390/include/asm/setup.h

+6 −3
Original line number Diff line number Diff line
@@ -29,8 +29,9 @@ 
#define MACHINE_FLAG_TE		_BITUL(11)

#define MACHINE_FLAG_TLB_LC	_BITUL(12)

#define MACHINE_FLAG_VX		_BITUL(13)

#define MACHINE_FLAG_NX		_BITUL(14)

#define MACHINE_FLAG_GS		_BITUL(15)

#define MACHINE_FLAG_TLB_GUEST	_BITUL(14)

#define MACHINE_FLAG_NX		_BITUL(15)

#define MACHINE_FLAG_GS		_BITUL(16)



#define LPP_MAGIC		_BITUL(31)

#define LPP_PFAULT_PID_MASK	_AC(0xffffffff, UL)
@@ -68,6 +69,7 @@ extern void detect_memory_memblock(void); 
#define MACHINE_HAS_TE		(S390_lowcore.machine_flags & MACHINE_FLAG_TE)

#define MACHINE_HAS_TLB_LC	(S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC)

#define MACHINE_HAS_VX		(S390_lowcore.machine_flags & MACHINE_FLAG_VX)

#define MACHINE_HAS_TLB_GUEST	(S390_lowcore.machine_flags & MACHINE_FLAG_TLB_GUEST)

#define MACHINE_HAS_NX		(S390_lowcore.machine_flags & MACHINE_FLAG_NX)

#define MACHINE_HAS_GS		(S390_lowcore.machine_flags & MACHINE_FLAG_GS)
@@ -106,7 +108,8 @@ extern void pfault_fini(void); 


void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault);



extern void cmma_init(void);

void cmma_init(void);

void cmma_init_nodat(void);



extern void (*_machine_restart)(char *command);

extern void (*_machine_halt)(void);

arch/s390/include/asm/tlbflush.h

+6 −1
Original line number Diff line number Diff line
@@ -20,10 +20,15 @@ static inline void __tlb_flush_local(void) 
 */

static inline void __tlb_flush_idte(unsigned long asce)

{

	unsigned long opt;



	opt = IDTE_PTOA;

	if (MACHINE_HAS_TLB_GUEST)

		opt |= IDTE_GUEST_ASCE;

	/* Global TLB flush for the mm */

	asm volatile(

		"	.insn	rrf,0xb98e0000,0,%0,%1,0"

		: : "a" (2048), "a" (asce) : "cc");

		: : "a" (opt), "a" (asce) : "cc");

}



#ifdef CONFIG_SMP