Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a20acf99 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next 

Pull sparc updates from David Miller:
 "Largely this is simply adding support for the Niagara 4 cpu.

  Major areas are perf events (chip now supports 4 counters and can
  monitor any event on each counter), crypto (opcodes are availble for
  sha1, sha256, sha512, md5, crc32c, AES, DES, CAMELLIA, and Kasumi
  although the last is unsupported since we lack a generic crypto layer
  Kasumi implementation), and an optimized memcpy.

  Finally some cleanups by Peter Senna Tschudin."

* git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next: (47 commits)
  sparc64: Fix trailing whitespace in NG4 memcpy.
  sparc64: Fix comment type in NG4 copy from user.
  sparc64: Add SPARC-T4 optimized memcpy.
  drivers/sbus/char: removes unnecessary semicolon
  arch/sparc/kernel/pci_sun4v.c: removes unnecessary semicolon
  sparc64: Fix function argument comment in camellia_sparc64_key_expand asm.
  sparc64: Fix IV handling bug in des_sparc64_cbc_decrypt
  sparc64: Add auto-loading mechanism to crypto-opcode drivers.
  sparc64: Add missing pr_fmt define to crypto opcode drivers.
  sparc64: Adjust crypto priorities.
  sparc64: Use cpu_pgsz_mask for linear kernel mapping config.
  sparc64: Probe cpu page size support more portably.
  sparc64: Support 2GB and 16GB page sizes for kernel linear mappings.
  sparc64: Fix bugs in unrolled 256-bit loops.
  sparc64: Avoid code duplication in crypto assembler.
  sparc64: Unroll CTR crypt loops in AES driver.
  sparc64: Unroll ECB decryption loops in AES driver.
  sparc64: Unroll ECB encryption loops in AES driver.
  sparc64: Add ctr mode support to AES driver.
  sparc64: Move AES driver over to a methods based implementation.
  ...
parents 437589a7 42a4172b

arch/sparc/Kbuild

+1 −0
Original line number Original line Diff line number Diff line
@@ -6,3 +6,4 @@ obj-y += kernel/ 
obj-y += mm/

obj-y += mm/

obj-y += math-emu/

obj-y += math-emu/

obj-y += net/

obj-y += net/

obj-y += crypto/

arch/sparc/crypto/Makefile

0 → 100644
+25 −0
Original line number Original line Diff line number Diff line 
#

# Arch-specific CryptoAPI modules.

#



obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o

obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o

obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o

obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o



obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o

obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o

obj-$(CONFIG_CRYPTO_DES_SPARC64) += camellia-sparc64.o



obj-$(CONFIG_CRYPTO_CRC32C_SPARC64) += crc32c-sparc64.o



sha1-sparc64-y := sha1_asm.o sha1_glue.o crop_devid.o

sha256-sparc64-y := sha256_asm.o sha256_glue.o crop_devid.o

sha512-sparc64-y := sha512_asm.o sha512_glue.o crop_devid.o

md5-sparc64-y := md5_asm.o md5_glue.o crop_devid.o



aes-sparc64-y := aes_asm.o aes_glue.o crop_devid.o

des-sparc64-y := des_asm.o des_glue.o crop_devid.o

camellia-sparc64-y := camellia_asm.o camellia_glue.o crop_devid.o



crc32c-sparc64-y := crc32c_asm.o crc32c_glue.o crop_devid.o

arch/sparc/crypto/aes_asm.S

0 → 100644
+1535 −0

File added.

Preview size limit exceeded, changes collapsed.

arch/sparc/crypto/aes_glue.c

0 → 100644
+477 −0
Original line number Original line Diff line number Diff line 
/* Glue code for AES encryption optimized for sparc64 crypto opcodes.

 *

 * This is based largely upon arch/x86/crypto/aesni-intel_glue.c

 *

 * Copyright (C) 2008, Intel Corp.

 *    Author: Huang Ying <ying.huang@intel.com>

 *

 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD

 * interface for 64-bit kernels.

 *    Authors: Adrian Hoban <adrian.hoban@intel.com>

 *             Gabriele Paoloni <gabriele.paoloni@intel.com>

 *             Tadeusz Struk (tadeusz.struk@intel.com)

 *             Aidan O'Mahony (aidan.o.mahony@intel.com)

 *    Copyright (c) 2010, Intel Corporation.

 */



#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt



#include <linux/crypto.h>

#include <linux/init.h>

#include <linux/module.h>

#include <linux/mm.h>

#include <linux/types.h>

#include <crypto/algapi.h>

#include <crypto/aes.h>



#include <asm/fpumacro.h>

#include <asm/pstate.h>

#include <asm/elf.h>



#include "opcodes.h"



struct aes_ops {

	void (*encrypt)(const u64 *key, const u32 *input, u32 *output);

	void (*decrypt)(const u64 *key, const u32 *input, u32 *output);

	void (*load_encrypt_keys)(const u64 *key);

	void (*load_decrypt_keys)(const u64 *key);

	void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len);

	void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len);

	void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len, u64 *iv);

	void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,

			    unsigned int len, u64 *iv);

	void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,

			  unsigned int len, u64 *iv);

};



struct crypto_sparc64_aes_ctx {

	struct aes_ops *ops;

	u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];

	u32 key_length;

	u32 expanded_key_length;

};



extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,

				    u32 *output);



extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,

				    u32 *output);

extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,

				    u32 *output);



extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);

extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);

extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);



extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);

extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);

extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);



extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);



extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);

extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len);



extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,

					u64 *output, unsigned int len,

					u64 *iv);



extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,

				      u64 *output, unsigned int len,

				      u64 *iv);

extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,

				      u64 *output, unsigned int len,

				      u64 *iv);

extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,

				      u64 *output, unsigned int len,

				      u64 *iv);



struct aes_ops aes128_ops = {

	.encrypt		= aes_sparc64_encrypt_128,

	.decrypt		= aes_sparc64_decrypt_128,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_128,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_128,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_128,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_128,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_128,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_128,

	.ctr_crypt		= aes_sparc64_ctr_crypt_128,

};



struct aes_ops aes192_ops = {

	.encrypt		= aes_sparc64_encrypt_192,

	.decrypt		= aes_sparc64_decrypt_192,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_192,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_192,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_192,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_192,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_192,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_192,

	.ctr_crypt		= aes_sparc64_ctr_crypt_192,

};



struct aes_ops aes256_ops = {

	.encrypt		= aes_sparc64_encrypt_256,

	.decrypt		= aes_sparc64_decrypt_256,

	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_256,

	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_256,

	.ecb_encrypt		= aes_sparc64_ecb_encrypt_256,

	.ecb_decrypt		= aes_sparc64_ecb_decrypt_256,

	.cbc_encrypt		= aes_sparc64_cbc_encrypt_256,

	.cbc_decrypt		= aes_sparc64_cbc_decrypt_256,

	.ctr_crypt		= aes_sparc64_ctr_crypt_256,

};



extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,

				   unsigned int key_len);



static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,

		       unsigned int key_len)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	u32 *flags = &tfm->crt_flags;



	switch (key_len) {

	case AES_KEYSIZE_128:

		ctx->expanded_key_length = 0xb0;

		ctx->ops = &aes128_ops;

		break;



	case AES_KEYSIZE_192:

		ctx->expanded_key_length = 0xd0;

		ctx->ops = &aes192_ops;

		break;



	case AES_KEYSIZE_256:

		ctx->expanded_key_length = 0xf0;

		ctx->ops = &aes256_ops;

		break;



	default:

		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;

		return -EINVAL;

	}



	aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);

	ctx->key_length = key_len;



	return 0;

}



static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);

}



static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);

}



#define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))



static int ecb_encrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	ctx->ops->load_encrypt_keys(&ctx->key[0]);

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;



		if (likely(block_len)) {

			ctx->ops->ecb_encrypt(&ctx->key[0],

					      (const u64 *)walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr,

					      block_len);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	fprs_write(0);

	return err;

}



static int ecb_decrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	struct blkcipher_walk walk;

	u64 *key_end;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	ctx->ops->load_decrypt_keys(&ctx->key[0]);

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;



		if (likely(block_len)) {

			ctx->ops->ecb_decrypt(key_end,

					      (const u64 *) walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr, block_len);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	fprs_write(0);



	return err;

}



static int cbc_encrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	ctx->ops->load_encrypt_keys(&ctx->key[0]);

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;



		if (likely(block_len)) {

			ctx->ops->cbc_encrypt(&ctx->key[0],

					      (const u64 *)walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr,

					      block_len, (u64 *) walk.iv);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	fprs_write(0);

	return err;

}



static int cbc_decrypt(struct blkcipher_desc *desc,

		       struct scatterlist *dst, struct scatterlist *src,

		       unsigned int nbytes)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	struct blkcipher_walk walk;

	u64 *key_end;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	ctx->ops->load_decrypt_keys(&ctx->key[0]);

	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;



		if (likely(block_len)) {

			ctx->ops->cbc_decrypt(key_end,

					      (const u64 *) walk.src.virt.addr,

					      (u64 *) walk.dst.virt.addr,

					      block_len, (u64 *) walk.iv);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	fprs_write(0);



	return err;

}



static int ctr_crypt(struct blkcipher_desc *desc,

		     struct scatterlist *dst, struct scatterlist *src,

		     unsigned int nbytes)

{

	struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	ctx->ops->load_encrypt_keys(&ctx->key[0]);

	while ((nbytes = walk.nbytes)) {

		unsigned int block_len = nbytes & AES_BLOCK_MASK;



		if (likely(block_len)) {

			ctx->ops->ctr_crypt(&ctx->key[0],

					    (const u64 *)walk.src.virt.addr,

					    (u64 *) walk.dst.virt.addr,

					    block_len, (u64 *) walk.iv);

		}

		nbytes &= AES_BLOCK_SIZE - 1;

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}

	fprs_write(0);

	return err;

}



static struct crypto_alg algs[] = { {

	.cra_name		= "aes",

	.cra_driver_name	= "aes-sparc64",

	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,

	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,

	.cra_blocksize		= AES_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),

	.cra_alignmask		= 3,

	.cra_module		= THIS_MODULE,

	.cra_u	= {

		.cipher	= {

			.cia_min_keysize	= AES_MIN_KEY_SIZE,

			.cia_max_keysize	= AES_MAX_KEY_SIZE,

			.cia_setkey		= aes_set_key,

			.cia_encrypt		= aes_encrypt,

			.cia_decrypt		= aes_decrypt

		}

	}

}, {

	.cra_name		= "ecb(aes)",

	.cra_driver_name	= "ecb-aes-sparc64",

	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= AES_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),

	.cra_alignmask		= 7,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_u = {

		.blkcipher = {

			.min_keysize	= AES_MIN_KEY_SIZE,

			.max_keysize	= AES_MAX_KEY_SIZE,

			.setkey		= aes_set_key,

			.encrypt	= ecb_encrypt,

			.decrypt	= ecb_decrypt,

		},

	},

}, {

	.cra_name		= "cbc(aes)",

	.cra_driver_name	= "cbc-aes-sparc64",

	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= AES_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),

	.cra_alignmask		= 7,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_u = {

		.blkcipher = {

			.min_keysize	= AES_MIN_KEY_SIZE,

			.max_keysize	= AES_MAX_KEY_SIZE,

			.setkey		= aes_set_key,

			.encrypt	= cbc_encrypt,

			.decrypt	= cbc_decrypt,

		},

	},

}, {

	.cra_name		= "ctr(aes)",

	.cra_driver_name	= "ctr-aes-sparc64",

	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= AES_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),

	.cra_alignmask		= 7,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_u = {

		.blkcipher = {

			.min_keysize	= AES_MIN_KEY_SIZE,

			.max_keysize	= AES_MAX_KEY_SIZE,

			.setkey		= aes_set_key,

			.encrypt	= ctr_crypt,

			.decrypt	= ctr_crypt,

		},

	},

} };



static bool __init sparc64_has_aes_opcode(void)

{

	unsigned long cfr;



	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))

		return false;



	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));

	if (!(cfr & CFR_AES))

		return false;



	return true;

}



static int __init aes_sparc64_mod_init(void)

{

	int i;



	for (i = 0; i < ARRAY_SIZE(algs); i++)

		INIT_LIST_HEAD(&algs[i].cra_list);



	if (sparc64_has_aes_opcode()) {

		pr_info("Using sparc64 aes opcodes optimized AES implementation\n");

		return crypto_register_algs(algs, ARRAY_SIZE(algs));

	}

	pr_info("sparc64 aes opcodes not available.\n");

	return -ENODEV;

}



static void __exit aes_sparc64_mod_fini(void)

{

	crypto_unregister_algs(algs, ARRAY_SIZE(algs));

}



module_init(aes_sparc64_mod_init);

module_exit(aes_sparc64_mod_fini);



MODULE_LICENSE("GPL");

MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");



MODULE_ALIAS("aes");

arch/sparc/crypto/camellia_asm.S

0 → 100644
+563 −0

File added.

Preview size limit exceeded, changes collapsed.