Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0aed55af authored by Dan Williams's avatar Dan Williams
Browse files

x86, uaccess: introduce copy_from_iter_flushcache for pmem / cache-bypass operations 

The pmem driver has a need to transfer data with a persistent memory
destination and be able to rely on the fact that the destination writes are not
cached. It is sufficient for the writes to be flushed to a cpu-store-buffer
(non-temporal / "movnt" in x86 terms), as we expect userspace to call fsync()
to ensure data-writes have reached a power-fail-safe zone in the platform. The
fsync() triggers a REQ_FUA or REQ_FLUSH to the pmem driver which will turn
around and fence previous writes with an "sfence".

Implement a __copy_from_user_inatomic_flushcache, memcpy_page_flushcache, and
memcpy_flushcache, that guarantee that the destination buffer is not dirty in
the cpu cache on completion. The new copy_from_iter_flushcache and sub-routines
will be used to replace the "pmem api" (include/linux/pmem.h +
arch/x86/include/asm/pmem.h). The availability of copy_from_iter_flushcache()
and memcpy_flushcache() are gated by the CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
config symbol, and fallback to copy_from_iter_nocache() and plain memcpy()
otherwise.

This is meant to satisfy the concern from Linus that if a driver wants to do
something beyond the normal nocache semantics it should be something private to
that driver [1], and Al's concern that anything uaccess related belongs with
the rest of the uaccess code [2].

The first consumer of this interface is a new 'copy_from_iter' dax operation so
that pmem can inject cache maintenance operations without imposing this
overhead on other dax-capable drivers.

[1]: https://lists.01.org/pipermail/linux-nvdimm/2017-January/008364.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2017-April/009942.html



Cc: <x86@kernel.org>
Cc: Jan Kara <jack@suse.cz>
Cc: Jeff Moyer <jmoyer@redhat.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Toshi Kani <toshi.kani@hpe.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Matthew Wilcox <mawilcox@microsoft.com>
Reviewed-by: default avatarRoss Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
parent 3c2993b8

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ config X86 
	select ARCH_HAS_KCOV			if X86_64

	select ARCH_HAS_MMIO_FLUSH

	select ARCH_HAS_PMEM_API		if X86_64

	select ARCH_HAS_UACCESS_FLUSHCACHE	if X86_64

	select ARCH_HAS_SET_MEMORY

	select ARCH_HAS_SG_CHAIN

	select ARCH_HAS_STRICT_KERNEL_RWX

arch/x86/include/asm/string_64.h

+5 −0
Original line number Diff line number Diff line
@@ -109,6 +109,11 @@ memcpy_mcsafe(void *dst, const void *src, size_t cnt) 
	return 0;

}



#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE

#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1

void memcpy_flushcache(void *dst, const void *src, size_t cnt);

#endif



#endif /* __KERNEL__ */



#endif /* _ASM_X86_STRING_64_H */

arch/x86/include/asm/uaccess_64.h

+11 −0
Original line number Diff line number Diff line
@@ -171,6 +171,10 @@ unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigne 
extern long __copy_user_nocache(void *dst, const void __user *src,

				unsigned size, int zerorest);



extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);

extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset,

			   size_t len);



static inline int

__copy_from_user_inatomic_nocache(void *dst, const void __user *src,

				  unsigned size)
@@ -179,6 +183,13 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 
	return __copy_user_nocache(dst, src, size, 0);

}



static inline int

__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)

{

	kasan_check_write(dst, size);

	return __copy_user_flushcache(dst, src, size);

}



unsigned long

copy_user_handle_tail(char *to, char *from, unsigned len);

arch/x86/lib/usercopy_64.c

+128 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
 */

#include <linux/export.h>

#include <linux/uaccess.h>

#include <linux/highmem.h>



/*

 * Zero Userspace
@@ -73,3 +74,130 @@ copy_user_handle_tail(char *to, char *from, unsigned len) 
	clac();

	return len;

}



#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE

/**

 * clean_cache_range - write back a cache range with CLWB

 * @vaddr:	virtual start address

 * @size:	number of bytes to write back

 *

 * Write back a cache range using the CLWB (cache line write back)

 * instruction. Note that @size is internally rounded up to be cache

 * line size aligned.

 */

static void clean_cache_range(void *addr, size_t size)

{

	u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;

	unsigned long clflush_mask = x86_clflush_size - 1;

	void *vend = addr + size;

	void *p;



	for (p = (void *)((unsigned long)addr & ~clflush_mask);

	     p < vend; p += x86_clflush_size)

		clwb(p);

}



long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)

{

	unsigned long flushed, dest = (unsigned long) dst;

	long rc = __copy_user_nocache(dst, src, size, 0);



	/*

	 * __copy_user_nocache() uses non-temporal stores for the bulk

	 * of the transfer, but we need to manually flush if the

	 * transfer is unaligned. A cached memory copy is used when

	 * destination or size is not naturally aligned. That is:

	 *   - Require 8-byte alignment when size is 8 bytes or larger.

	 *   - Require 4-byte alignment when size is 4 bytes.

	 */

	if (size < 8) {

		if (!IS_ALIGNED(dest, 4) || size != 4)

			clean_cache_range(dst, 1);

	} else {

		if (!IS_ALIGNED(dest, 8)) {

			dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);

			clean_cache_range(dst, 1);

		}



		flushed = dest - (unsigned long) dst;

		if (size > flushed && !IS_ALIGNED(size - flushed, 8))

			clean_cache_range(dst + size - 1, 1);

	}



	return rc;

}



void memcpy_flushcache(void *_dst, const void *_src, size_t size)

{

	unsigned long dest = (unsigned long) _dst;

	unsigned long source = (unsigned long) _src;



	/* cache copy and flush to align dest */

	if (!IS_ALIGNED(dest, 8)) {

		unsigned len = min_t(unsigned, size, ALIGN(dest, 8) - dest);



		memcpy((void *) dest, (void *) source, len);

		clean_cache_range((void *) dest, len);

		dest += len;

		source += len;

		size -= len;

		if (!size)

			return;

	}



	/* 4x8 movnti loop */

	while (size >= 32) {

		asm("movq    (%0), %%r8\n"

		    "movq   8(%0), %%r9\n"

		    "movq  16(%0), %%r10\n"

		    "movq  24(%0), %%r11\n"

		    "movnti  %%r8,   (%1)\n"

		    "movnti  %%r9,  8(%1)\n"

		    "movnti %%r10, 16(%1)\n"

		    "movnti %%r11, 24(%1)\n"

		    :: "r" (source), "r" (dest)

		    : "memory", "r8", "r9", "r10", "r11");

		dest += 32;

		source += 32;

		size -= 32;

	}



	/* 1x8 movnti loop */

	while (size >= 8) {

		asm("movq    (%0), %%r8\n"

		    "movnti  %%r8,   (%1)\n"

		    :: "r" (source), "r" (dest)

		    : "memory", "r8");

		dest += 8;

		source += 8;

		size -= 8;

	}



	/* 1x4 movnti loop */

	while (size >= 4) {

		asm("movl    (%0), %%r8d\n"

		    "movnti  %%r8d,   (%1)\n"

		    :: "r" (source), "r" (dest)

		    : "memory", "r8");

		dest += 4;

		source += 4;

		size -= 4;

	}



	/* cache copy for remaining bytes */

	if (size) {

		memcpy((void *) dest, (void *) source, size);

		clean_cache_range((void *) dest, size);

	}

}

EXPORT_SYMBOL_GPL(memcpy_flushcache);



void memcpy_page_flushcache(char *to, struct page *page, size_t offset,

		size_t len)

{

	char *from = kmap_atomic(page);



	memcpy_flushcache(to, from + offset, len);

	kunmap_atomic(from);

}

#endif

drivers/acpi/nfit/core.c

+1 −2
Original line number Diff line number Diff line
@@ -1842,8 +1842,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 
		}



		if (rw)

			memcpy_to_pmem(mmio->addr.aperture + offset,

					iobuf + copied, c);

			memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);

		else {

			if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)

				mmio_flush_range((void __force *)