Merge "arm: optimize memcpy_{from,to}io() and memset_io" (930303d6) · Commits · e / devices / android_kernel_xiaomi_markw

arch/arm/kernel/io.c

+61 −16

Original line number	Diff line number	Diff line
		@@ -3,6 +3,8 @@
		#include <linux/io.h>
		#include <linux/spinlock.h>

		#define IO_CHECK_ALIGN(v, a) ((((unsigned long)(v)) & ((a) - 1)) == 0)

		static DEFINE_RAW_SPINLOCK(__io_lock);

		/*
		@@ -39,47 +41,90 @@ EXPORT_SYMBOL(atomic_io_modify);

		/*
		* Copy data from IO memory space to "real" memory space.
		* This needs to be optimized.
		*/
		void _memcpy_fromio(void to, const volatile void __iomem from, size_t count)
		{
		unsigned char *t = to;
		while (count) {
		while (count && (!IO_CHECK_ALIGN(from, 8) \|\| !IO_CHECK_ALIGN(to, 8))) {
		(u8 )to = readb_relaxed_no_log(from);
		from++;
		to++;
		count--;
		*t = readb(from);
		t++;
		}

		while (count >= 8) {
		(u64 )to = readq_relaxed_no_log(from);
		from += 8;
		to += 8;
		count -= 8;
		}

		while (count) {
		(u8 )to = readb_relaxed_no_log(from);
		from++;
		to++;
		count--;
		}
		}
		EXPORT_SYMBOL(_memcpy_fromio);

		/*
		* Copy data from "real" memory space to IO memory space.
		* This needs to be optimized.
		*/
		void _memcpy_toio(volatile void __iomem to, const void from, size_t count)
		{
		const unsigned char *f = from;
		void p = (void __force )to;

		while (count && (!IO_CHECK_ALIGN(p, 8) \|\| !IO_CHECK_ALIGN(from, 8))) {
		writeb_relaxed_no_log((volatile u8 )from, p);
		from++;
		p++;
		count--;
		}

		while (count >= 8) {
		writeq_relaxed_no_log((volatile u64 )from, p);
		from += 8;
		p += 8;
		count -= 8;
		}

		while (count) {
		writeb_relaxed_no_log((volatile u8 )from, p);
		from++;
		p++;
		count--;
		writeb(*f, to);
		f++;
		to++;
		}
		}
		EXPORT_SYMBOL(_memcpy_toio);

		/*
		* "memset" on IO memory space.
		* This needs to be optimized.
		*/
		void _memset_io(volatile void __iomem *dst, int c, size_t count)
		{
		while (count) {
		void p = (void __force )dst;
		u64 qc = c;

		qc \|= qc << 8;
		qc \|= qc << 16;
		qc \|= qc << 32;

		while (count && !IO_CHECK_ALIGN(p, 8)) {
		writeb_relaxed_no_log(c, p);
		p++;
		count--;
		writeb(c, dst);
		dst++;
		}

		while (count >= 8) {
		writeq_relaxed_no_log(qc, p);
		p += 8;
		count -= 8;
		}

		EXPORT_SYMBOL(_memcpy_fromio);
		EXPORT_SYMBOL(_memcpy_toio);
		while (count) {
		writeb_relaxed_no_log(c, p);
		p++;
		count--;
		}
		}
		EXPORT_SYMBOL(_memset_io);