Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6dd7a82c authored by Anton Blanchard's avatar Anton Blanchard Committed by Herbert Xu
Browse files

crypto: powerpc - Add POWER8 optimised crc32c 



Use the vector polynomial multiply-sum instructions in POWER8 to
speed up crc32c.

This is just over 41x faster than the slice-by-8 method that it
replaces. Measurements on a 4.1 GHz POWER8 show it sustaining
52 GiB/sec.

A simple btrfs write performance test:

    dd if=/dev/zero of=/mnt/tmpfile bs=1M count=4096
    sync

is over 3.7x faster.

Signed-off-by: default avatarAnton Blanchard <anton@samba.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 151f2511

arch/powerpc/crypto/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -9,9 +9,11 @@ obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o 
obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o

obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o

obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o

obj-$(CONFIG_CRYPT_CRC32C_VPMSUM) += crc32c-vpmsum.o



aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o

md5-ppc-y := md5-asm.o md5-glue.o

sha1-powerpc-y := sha1-powerpc-asm.o sha1.o

sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o

sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o

crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o

arch/powerpc/crypto/crc32c-vpmsum_asm.S

0 → 100644
+1553 −0

File added.

Preview size limit exceeded, changes collapsed.

arch/powerpc/crypto/crc32c-vpmsum_glue.c

0 → 100644
+167 −0
Original line number Diff line number Diff line 
#include <linux/crc32.h>

#include <crypto/internal/hash.h>

#include <linux/init.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/kernel.h>

#include <asm/switch_to.h>



#define CHKSUM_BLOCK_SIZE	1

#define CHKSUM_DIGEST_SIZE	4



#define VMX_ALIGN		16

#define VMX_ALIGN_MASK		(VMX_ALIGN-1)



#define VECTOR_BREAKPOINT	512



u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);



static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)

{

	unsigned int prealign;

	unsigned int tail;



	if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt())

		return __crc32c_le(crc, p, len);



	if ((unsigned long)p & VMX_ALIGN_MASK) {

		prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);

		crc = __crc32c_le(crc, p, prealign);

		len -= prealign;

		p += prealign;

	}



	if (len & ~VMX_ALIGN_MASK) {

		pagefault_disable();

		enable_kernel_altivec();

		crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);

		pagefault_enable();

	}



	tail = len & VMX_ALIGN_MASK;

	if (tail) {

		p += len & ~VMX_ALIGN_MASK;

		crc = __crc32c_le(crc, p, tail);

	}



	return crc;

}



static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)

{

	u32 *key = crypto_tfm_ctx(tfm);



	*key = 0;



	return 0;

}



/*

 * Setting the seed allows arbitrary accumulators and flexible XOR policy

 * If your algorithm starts with ~0, then XOR with ~0 before you set

 * the seed.

 */

static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key,

			       unsigned int keylen)

{

	u32 *mctx = crypto_shash_ctx(hash);



	if (keylen != sizeof(u32)) {

		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);

		return -EINVAL;

	}

	*mctx = le32_to_cpup((__le32 *)key);

	return 0;

}



static int crc32c_vpmsum_init(struct shash_desc *desc)

{

	u32 *mctx = crypto_shash_ctx(desc->tfm);

	u32 *crcp = shash_desc_ctx(desc);



	*crcp = *mctx;



	return 0;

}



static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data,

			       unsigned int len)

{

	u32 *crcp = shash_desc_ctx(desc);



	*crcp = crc32c_vpmsum(*crcp, data, len);



	return 0;

}



static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len,

				u8 *out)

{

	*(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));



	return 0;

}



static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data,

			      unsigned int len, u8 *out)

{

	return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);

}



static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out)

{

	u32 *crcp = shash_desc_ctx(desc);



	*(__le32 *)out = ~cpu_to_le32p(crcp);



	return 0;

}



static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data,

			       unsigned int len, u8 *out)

{

	return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,

				     out);

}



static struct shash_alg alg = {

	.setkey		= crc32c_vpmsum_setkey,

	.init		= crc32c_vpmsum_init,

	.update		= crc32c_vpmsum_update,

	.final		= crc32c_vpmsum_final,

	.finup		= crc32c_vpmsum_finup,

	.digest		= crc32c_vpmsum_digest,

	.descsize	= sizeof(u32),

	.digestsize	= CHKSUM_DIGEST_SIZE,

	.base		= {

		.cra_name		= "crc32c",

		.cra_driver_name	= "crc32c-vpmsum",

		.cra_priority		= 200,

		.cra_blocksize		= CHKSUM_BLOCK_SIZE,

		.cra_ctxsize		= sizeof(u32),

		.cra_module		= THIS_MODULE,

		.cra_init		= crc32c_vpmsum_cra_init,

	}

};



static int __init crc32c_vpmsum_mod_init(void)

{

	if (!cpu_has_feature(CPU_FTR_ARCH_207S))

		return -ENODEV;



	return crypto_register_shash(&alg);

}



static void __exit crc32c_vpmsum_mod_fini(void)

{

	crypto_unregister_shash(&alg);

}



module_init(crc32c_vpmsum_mod_init);

module_exit(crc32c_vpmsum_mod_fini);



MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");

MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");

MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crc32c");

MODULE_ALIAS_CRYPTO("crc32c-vpmsum");

arch/powerpc/include/asm/ppc-opcode.h

+12 −0
Original line number Diff line number Diff line
@@ -174,6 +174,8 @@ 
#define PPC_INST_MFSPR_DSCR_USER_MASK	0xfc1fffff

#define PPC_INST_MTSPR_DSCR_USER	0x7c0303a6

#define PPC_INST_MTSPR_DSCR_USER_MASK	0xfc1fffff

#define PPC_INST_MFVSRD			0x7c000066

#define PPC_INST_MTVSRD			0x7c000166

#define PPC_INST_SLBFEE			0x7c0007a7



#define PPC_INST_STRING			0x7c00042a
@@ -188,6 +190,8 @@ 
#define PPC_INST_WAIT			0x7c00007c

#define PPC_INST_TLBIVAX		0x7c000624

#define PPC_INST_TLBSRX_DOT		0x7c0006a5

#define PPC_INST_VPMSUMW		0x10000488

#define PPC_INST_VPMSUMD		0x100004c8

#define PPC_INST_XXLOR			0xf0000510

#define PPC_INST_XXSWAPD		0xf0000250

#define PPC_INST_XVCPSGNDP		0xf0000780
@@ -359,6 +363,14 @@ 
					       VSX_XX1((s), a, b))

#define LXVD2X(s, a, b)		stringify_in_c(.long PPC_INST_LXVD2X | \

					       VSX_XX1((s), a, b))

#define MFVRD(a, t)		stringify_in_c(.long PPC_INST_MFVSRD | \

					       VSX_XX1((t)+32, a, R0))

#define MTVRD(t, a)		stringify_in_c(.long PPC_INST_MTVSRD | \

					       VSX_XX1((t)+32, a, R0))

#define VPMSUMW(t, a, b)	stringify_in_c(.long PPC_INST_VPMSUMW | \

					       VSX_XX3((t), a, b))

#define VPMSUMD(t, a, b)	stringify_in_c(.long PPC_INST_VPMSUMD | \

					       VSX_XX3((t), a, b))

#define XXLOR(t, a, b)		stringify_in_c(.long PPC_INST_XXLOR | \

					       VSX_XX3((t), a, b))

#define XXSWAPD(t, a)		stringify_in_c(.long PPC_INST_XXSWAPD | \

crypto/Kconfig

+11 −0
Original line number Diff line number Diff line
@@ -437,6 +437,17 @@ config CRYPTO_CRC32C_INTEL 
	  gain performance compared with software implementation.

	  Module will be crc32c-intel.



config CRYPT_CRC32C_VPMSUM

	tristate "CRC32c CRC algorithm (powerpc64)"

	depends on PPC64

	select CRYPTO_HASH

	select CRC32

	help

	  CRC32c algorithm implemented using vector polynomial multiply-sum

	  (vpmsum) instructions, introduced in POWER8. Enable on POWER8

	  and newer processors for improved performance.





config CRYPTO_CRC32C_SPARC64

	tristate "CRC32c CRC algorithm (SPARC64)"

	depends on SPARC64