Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit dc47d381 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://github.com/herbertx/crypto 

* git://github.com/herbertx/crypto: (48 commits)
  crypto: user - Depend on NET instead of selecting it
  crypto: user - Add dependency on NET
  crypto: talitos - handle descriptor not found in error path
  crypto: user - Initialise match in crypto_alg_match
  crypto: testmgr - add twofish tests
  crypto: testmgr - add blowfish test-vectors
  crypto: Make hifn_795x build depend on !ARCH_DMA_ADDR_T_64BIT
  crypto: twofish-x86_64-3way - fix ctr blocksize to 1
  crypto: blowfish-x86_64 - fix ctr blocksize to 1
  crypto: whirlpool - count rounds from 0
  crypto: Add userspace report for compress type algorithms
  crypto: Add userspace report for cipher type algorithms
  crypto: Add userspace report for rng type algorithms
  crypto: Add userspace report for pcompress type algorithms
  crypto: Add userspace report for nivaead type algorithms
  crypto: Add userspace report for aead type algorithms
  crypto: Add userspace report for givcipher type algorithms
  crypto: Add userspace report for ablkcipher type algorithms
  crypto: Add userspace report for blkcipher type algorithms
  crypto: Add userspace report for ahash type algorithms
  ...
parents f6d90b4f 5db017aa

Documentation/devicetree/bindings/crypto/picochip-spacc.txt

0 → 100644
+23 −0
Original line number Diff line number Diff line 
Picochip picoXcell SPAcc (Security Protocol Accelerator) bindings



Picochip picoXcell devices contain crypto offload engines that may be used for

IPSEC and femtocell layer 2 ciphering.



Required properties:

  - compatible : "picochip,spacc-ipsec" for the IPSEC offload engine

    "picochip,spacc-l2" for the femtocell layer 2 ciphering engine.

  - reg : Offset and length of the register set for this device

  - interrupt-parent : The interrupt controller that controls the SPAcc

    interrupt.

  - interrupts : The interrupt line from the SPAcc.

  - ref-clock : The input clock that drives the SPAcc.



Example SPAcc node:



spacc@10000 {

	compatible = "picochip,spacc-ipsec";

	reg = <0x100000 0x10000>;

	interrupt-parent = <&vic0>;

	interrupts = <24>;

	ref-clock = <&ipsec_clk>, "ref";

};

arch/x86/crypto/Makefile

+12 −0
Original line number Diff line number Diff line
@@ -7,21 +7,33 @@ obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o 
obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o



obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o

obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o

obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o

obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o

obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o

obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o

obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o



obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o

obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o



aes-i586-y := aes-i586-asm_32.o aes_glue.o

twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o

salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o



aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o

blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o

twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o

twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o

salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o



aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o



ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o



# enable AVX support only when $(AS) can actually assemble the instructions

ifeq ($(call as-instr,vpxor %xmm0$(comma)%xmm1$(comma)%xmm2,yes,no),yes)

AFLAGS_sha1_ssse3_asm.o += -DSHA1_ENABLE_AVX_SUPPORT

CFLAGS_sha1_ssse3_glue.o += -DSHA1_ENABLE_AVX_SUPPORT

endif

sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o

arch/x86/crypto/aes_glue.c

+1 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
 */



#include <crypto/aes.h>

#include <asm/aes.h>



asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);

asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);

arch/x86/crypto/blowfish-x86_64-asm_64.S

0 → 100644
+390 −0
Original line number Diff line number Diff line 
/*

 * Blowfish Cipher Algorithm (x86_64)

 *

 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307

 * USA

 *

 */



.file "blowfish-x86_64-asm.S"

.text



/* structure of crypto context */

#define p	0

#define s0	((16 + 2) * 4)

#define s1	((16 + 2 + (1 * 256)) * 4)

#define s2	((16 + 2 + (2 * 256)) * 4)

#define s3	((16 + 2 + (3 * 256)) * 4)



/* register macros */

#define CTX %rdi

#define RIO %rsi



#define RX0 %rax

#define RX1 %rbx

#define RX2 %rcx

#define RX3 %rdx



#define RX0d %eax

#define RX1d %ebx

#define RX2d %ecx

#define RX3d %edx



#define RX0bl %al

#define RX1bl %bl

#define RX2bl %cl

#define RX3bl %dl



#define RX0bh %ah

#define RX1bh %bh

#define RX2bh %ch

#define RX3bh %dh



#define RT0 %rbp

#define RT1 %rsi

#define RT2 %r8

#define RT3 %r9



#define RT0d %ebp

#define RT1d %esi

#define RT2d %r8d

#define RT3d %r9d



#define RKEY %r10



/***********************************************************************

 * 1-way blowfish

 ***********************************************************************/

#define F() \

	rorq $16,		RX0; \

	movzbl RX0bh,		RT0d; \

	movzbl RX0bl,		RT1d; \

	rolq $16,		RX0; \

	movl s0(CTX,RT0,4),	RT0d; \

	addl s1(CTX,RT1,4),	RT0d; \

	movzbl RX0bh,		RT1d; \

	movzbl RX0bl,		RT2d; \

	rolq $32,		RX0; \

	xorl s2(CTX,RT1,4),	RT0d; \

	addl s3(CTX,RT2,4),	RT0d; \

	xorq RT0,		RX0;



#define add_roundkey_enc(n) \

	xorq p+4*(n)(CTX), 	RX0;



#define round_enc(n) \

	add_roundkey_enc(n); \

	\

	F(); \

	F();



#define add_roundkey_dec(n) \

	movq p+4*(n-1)(CTX),	RT0; \

	rorq $32,		RT0; \

	xorq RT0,		RX0;



#define round_dec(n) \

	add_roundkey_dec(n); \

	\

	F(); \

	F(); \



#define read_block() \

	movq (RIO), 		RX0; \

	rorq $32, 		RX0; \

	bswapq 			RX0;



#define write_block() \

	bswapq 			RX0; \

	movq RX0, 		(RIO);



#define xor_block() \

	bswapq 			RX0; \

	xorq RX0, 		(RIO);



.align 8

.global __blowfish_enc_blk

.type   __blowfish_enc_blk,@function;



__blowfish_enc_blk:

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 *	%rcx: bool, if true: xor output

	 */

	movq %rbp, %r11;



	movq %rsi, %r10;

	movq %rdx, RIO;



	read_block();



	round_enc(0);

	round_enc(2);

	round_enc(4);

	round_enc(6);

	round_enc(8);

	round_enc(10);

	round_enc(12);

	round_enc(14);

	add_roundkey_enc(16);



	movq %r11, %rbp;



	movq %r10, RIO;

	test %cl, %cl;

	jnz __enc_xor;



	write_block();

	ret;

__enc_xor:

	xor_block();

	ret;



.align 8

.global blowfish_dec_blk

.type   blowfish_dec_blk,@function;



blowfish_dec_blk:

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 */

	movq %rbp, %r11;



	movq %rsi, %r10;

	movq %rdx, RIO;



	read_block();



	round_dec(17);

	round_dec(15);

	round_dec(13);

	round_dec(11);

	round_dec(9);

	round_dec(7);

	round_dec(5);

	round_dec(3);

	add_roundkey_dec(1);



	movq %r10, RIO;

	write_block();



	movq %r11, %rbp;



	ret;



/**********************************************************************

  4-way blowfish, four blocks parallel

 **********************************************************************/



/* F() for 4-way. Slower when used alone/1-way, but faster when used

 * parallel/4-way (tested on AMD Phenom II & Intel Xeon E7330).

 */

#define F4(x) \

	movzbl x ## bh,		RT1d; \

	movzbl x ## bl,		RT3d; \

	rorq $16,		x; \

	movzbl x ## bh,		RT0d; \

	movzbl x ## bl,		RT2d; \

	rorq $16,		x; \

	movl s0(CTX,RT0,4),	RT0d; \

	addl s1(CTX,RT2,4),	RT0d; \

	xorl s2(CTX,RT1,4),	RT0d; \

	addl s3(CTX,RT3,4),	RT0d; \

	xorq RT0,		x;



#define add_preloaded_roundkey4() \

	xorq RKEY,		RX0; \

	xorq RKEY,		RX1; \

	xorq RKEY,		RX2; \

	xorq RKEY,		RX3;



#define preload_roundkey_enc(n) \

	movq p+4*(n)(CTX),	RKEY;



#define add_roundkey_enc4(n) \

	add_preloaded_roundkey4(); \

	preload_roundkey_enc(n + 2);



#define round_enc4(n) \

	add_roundkey_enc4(n); \

	\

	F4(RX0); \

	F4(RX1); \

	F4(RX2); \

	F4(RX3); \

	\

	F4(RX0); \

	F4(RX1); \

	F4(RX2); \

	F4(RX3);



#define preload_roundkey_dec(n) \

	movq p+4*((n)-1)(CTX),	RKEY; \

	rorq $32,		RKEY;



#define add_roundkey_dec4(n) \

	add_preloaded_roundkey4(); \

	preload_roundkey_dec(n - 2);



#define round_dec4(n) \

	add_roundkey_dec4(n); \

	\

	F4(RX0); \

	F4(RX1); \

	F4(RX2); \

	F4(RX3); \

	\

	F4(RX0); \

	F4(RX1); \

	F4(RX2); \

	F4(RX3);



#define read_block4() \

	movq (RIO),		RX0; \

	rorq $32,		RX0; \

	bswapq 			RX0; \

	\

	movq 8(RIO),		RX1; \

	rorq $32,		RX1; \

	bswapq 			RX1; \

	\

	movq 16(RIO),		RX2; \

	rorq $32,		RX2; \

	bswapq 			RX2; \

	\

	movq 24(RIO),		RX3; \

	rorq $32,		RX3; \

	bswapq 			RX3;



#define write_block4() \

	bswapq 			RX0; \

	movq RX0,		(RIO); \

	\

	bswapq 			RX1; \

	movq RX1,		8(RIO); \

	\

	bswapq 			RX2; \

	movq RX2,		16(RIO); \

	\

	bswapq 			RX3; \

	movq RX3,		24(RIO);



#define xor_block4() \

	bswapq 			RX0; \

	xorq RX0,		(RIO); \

	\

	bswapq 			RX1; \

	xorq RX1,		8(RIO); \

	\

	bswapq 			RX2; \

	xorq RX2,		16(RIO); \

	\

	bswapq 			RX3; \

	xorq RX3,		24(RIO);



.align 8

.global __blowfish_enc_blk_4way

.type   __blowfish_enc_blk_4way,@function;



__blowfish_enc_blk_4way:

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 *	%rcx: bool, if true: xor output

	 */

	pushq %rbp;

	pushq %rbx;

	pushq %rcx;



	preload_roundkey_enc(0);



	movq %rsi, %r11;

	movq %rdx, RIO;



	read_block4();



	round_enc4(0);

	round_enc4(2);

	round_enc4(4);

	round_enc4(6);

	round_enc4(8);

	round_enc4(10);

	round_enc4(12);

	round_enc4(14);

	add_preloaded_roundkey4();



	popq %rbp;

	movq %r11, RIO;



	test %bpl, %bpl;

	jnz __enc_xor4;



	write_block4();



	popq %rbx;

	popq %rbp;

	ret;



__enc_xor4:

	xor_block4();



	popq %rbx;

	popq %rbp;

	ret;



.align 8

.global blowfish_dec_blk_4way

.type   blowfish_dec_blk_4way,@function;



blowfish_dec_blk_4way:

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 */

	pushq %rbp;

	pushq %rbx;

	preload_roundkey_dec(17);



	movq %rsi, %r11;

	movq %rdx, RIO;



	read_block4();



	round_dec4(17);

	round_dec4(15);

	round_dec4(13);

	round_dec4(11);

	round_dec4(9);

	round_dec4(7);

	round_dec4(5);

	round_dec4(3);

	add_preloaded_roundkey4();



	movq %r11, RIO;

	write_block4();



	popq %rbx;

	popq %rbp;



	ret;

arch/x86/crypto/blowfish_glue.c

0 → 100644
+492 −0
Original line number Diff line number Diff line 
/*

 * Glue Code for assembler optimized version of Blowfish

 *

 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>

 *

 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:

 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>

 * CTR part based on code (crypto/ctr.c) by:

 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307

 * USA

 *

 */



#include <crypto/blowfish.h>

#include <linux/crypto.h>

#include <linux/init.h>

#include <linux/module.h>

#include <linux/types.h>

#include <crypto/algapi.h>



/* regular block cipher functions */

asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,

				   bool xor);

asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);



/* 4-way parallel cipher functions */

asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,

					const u8 *src, bool xor);

asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,

				      const u8 *src);



static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)

{

	__blowfish_enc_blk(ctx, dst, src, false);

}



static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,

					const u8 *src)

{

	__blowfish_enc_blk(ctx, dst, src, true);

}



static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,

					 const u8 *src)

{

	__blowfish_enc_blk_4way(ctx, dst, src, false);

}



static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,

				      const u8 *src)

{

	__blowfish_enc_blk_4way(ctx, dst, src, true);

}



static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);

}



static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);

}



static struct crypto_alg bf_alg = {

	.cra_name		=	"blowfish",

	.cra_driver_name	=	"blowfish-asm",

	.cra_priority		=	200,

	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,

	.cra_blocksize		=	BF_BLOCK_SIZE,

	.cra_ctxsize		=	sizeof(struct bf_ctx),

	.cra_alignmask		=	3,

	.cra_module		=	THIS_MODULE,

	.cra_list		=	LIST_HEAD_INIT(bf_alg.cra_list),

	.cra_u			=	{

		.cipher = {

			.cia_min_keysize	=	BF_MIN_KEY_SIZE,

			.cia_max_keysize	=	BF_MAX_KEY_SIZE,

			.cia_setkey		=	blowfish_setkey,

			.cia_encrypt		=	blowfish_encrypt,

			.cia_decrypt		=	blowfish_decrypt,

		}

	}

};



static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,

		     void (*fn)(struct bf_ctx *, u8 *, const u8 *),

		     void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))

{

	struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	unsigned int bsize = BF_BLOCK_SIZE;

	unsigned int nbytes;

	int err;



	err = blkcipher_walk_virt(desc, walk);



	while ((nbytes = walk->nbytes)) {

		u8 *wsrc = walk->src.virt.addr;

		u8 *wdst = walk->dst.virt.addr;



		/* Process four block batch */

		if (nbytes >= bsize * 4) {

			do {

				fn_4way(ctx, wdst, wsrc);



				wsrc += bsize * 4;

				wdst += bsize * 4;

				nbytes -= bsize * 4;

			} while (nbytes >= bsize * 4);



			if (nbytes < bsize)

				goto done;

		}



		/* Handle leftovers */

		do {

			fn(ctx, wdst, wsrc);



			wsrc += bsize;

			wdst += bsize;

			nbytes -= bsize;

		} while (nbytes >= bsize);



done:

		err = blkcipher_walk_done(desc, walk, nbytes);

	}



	return err;

}



static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,

		       struct scatterlist *src, unsigned int nbytes)

{

	struct blkcipher_walk walk;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);

}



static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,

		       struct scatterlist *src, unsigned int nbytes)

{

	struct blkcipher_walk walk;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);

}



static struct crypto_alg blk_ecb_alg = {

	.cra_name		= "ecb(blowfish)",

	.cra_driver_name	= "ecb-blowfish-asm",

	.cra_priority		= 300,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= BF_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct bf_ctx),

	.cra_alignmask		= 0,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_list		= LIST_HEAD_INIT(blk_ecb_alg.cra_list),

	.cra_u = {

		.blkcipher = {

			.min_keysize	= BF_MIN_KEY_SIZE,

			.max_keysize	= BF_MAX_KEY_SIZE,

			.setkey		= blowfish_setkey,

			.encrypt	= ecb_encrypt,

			.decrypt	= ecb_decrypt,

		},

	},

};



static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,

				  struct blkcipher_walk *walk)

{

	struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	unsigned int bsize = BF_BLOCK_SIZE;

	unsigned int nbytes = walk->nbytes;

	u64 *src = (u64 *)walk->src.virt.addr;

	u64 *dst = (u64 *)walk->dst.virt.addr;

	u64 *iv = (u64 *)walk->iv;



	do {

		*dst = *src ^ *iv;

		blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);

		iv = dst;



		src += 1;

		dst += 1;

		nbytes -= bsize;

	} while (nbytes >= bsize);



	*(u64 *)walk->iv = *iv;

	return nbytes;

}



static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,

		       struct scatterlist *src, unsigned int nbytes)

{

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	while ((nbytes = walk.nbytes)) {

		nbytes = __cbc_encrypt(desc, &walk);

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}



	return err;

}



static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,

				  struct blkcipher_walk *walk)

{

	struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	unsigned int bsize = BF_BLOCK_SIZE;

	unsigned int nbytes = walk->nbytes;

	u64 *src = (u64 *)walk->src.virt.addr;

	u64 *dst = (u64 *)walk->dst.virt.addr;

	u64 ivs[4 - 1];

	u64 last_iv;



	/* Start of the last block. */

	src += nbytes / bsize - 1;

	dst += nbytes / bsize - 1;



	last_iv = *src;



	/* Process four block batch */

	if (nbytes >= bsize * 4) {

		do {

			nbytes -= bsize * 4 - bsize;

			src -= 4 - 1;

			dst -= 4 - 1;



			ivs[0] = src[0];

			ivs[1] = src[1];

			ivs[2] = src[2];



			blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);



			dst[1] ^= ivs[0];

			dst[2] ^= ivs[1];

			dst[3] ^= ivs[2];



			nbytes -= bsize;

			if (nbytes < bsize)

				goto done;



			*dst ^= *(src - 1);

			src -= 1;

			dst -= 1;

		} while (nbytes >= bsize * 4);



		if (nbytes < bsize)

			goto done;

	}



	/* Handle leftovers */

	for (;;) {

		blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);



		nbytes -= bsize;

		if (nbytes < bsize)

			break;



		*dst ^= *(src - 1);

		src -= 1;

		dst -= 1;

	}



done:

	*dst ^= *(u64 *)walk->iv;

	*(u64 *)walk->iv = last_iv;



	return nbytes;

}



static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,

		       struct scatterlist *src, unsigned int nbytes)

{

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);



	while ((nbytes = walk.nbytes)) {

		nbytes = __cbc_decrypt(desc, &walk);

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}



	return err;

}



static struct crypto_alg blk_cbc_alg = {

	.cra_name		= "cbc(blowfish)",

	.cra_driver_name	= "cbc-blowfish-asm",

	.cra_priority		= 300,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= BF_BLOCK_SIZE,

	.cra_ctxsize		= sizeof(struct bf_ctx),

	.cra_alignmask		= 0,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_list		= LIST_HEAD_INIT(blk_cbc_alg.cra_list),

	.cra_u = {

		.blkcipher = {

			.min_keysize	= BF_MIN_KEY_SIZE,

			.max_keysize	= BF_MAX_KEY_SIZE,

			.ivsize		= BF_BLOCK_SIZE,

			.setkey		= blowfish_setkey,

			.encrypt	= cbc_encrypt,

			.decrypt	= cbc_decrypt,

		},

	},

};



static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)

{

	u8 *ctrblk = walk->iv;

	u8 keystream[BF_BLOCK_SIZE];

	u8 *src = walk->src.virt.addr;

	u8 *dst = walk->dst.virt.addr;

	unsigned int nbytes = walk->nbytes;



	blowfish_enc_blk(ctx, keystream, ctrblk);

	crypto_xor(keystream, src, nbytes);

	memcpy(dst, keystream, nbytes);



	crypto_inc(ctrblk, BF_BLOCK_SIZE);

}



static unsigned int __ctr_crypt(struct blkcipher_desc *desc,

				struct blkcipher_walk *walk)

{

	struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	unsigned int bsize = BF_BLOCK_SIZE;

	unsigned int nbytes = walk->nbytes;

	u64 *src = (u64 *)walk->src.virt.addr;

	u64 *dst = (u64 *)walk->dst.virt.addr;

	u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);

	__be64 ctrblocks[4];



	/* Process four block batch */

	if (nbytes >= bsize * 4) {

		do {

			if (dst != src) {

				dst[0] = src[0];

				dst[1] = src[1];

				dst[2] = src[2];

				dst[3] = src[3];

			}



			/* create ctrblks for parallel encrypt */

			ctrblocks[0] = cpu_to_be64(ctrblk++);

			ctrblocks[1] = cpu_to_be64(ctrblk++);

			ctrblocks[2] = cpu_to_be64(ctrblk++);

			ctrblocks[3] = cpu_to_be64(ctrblk++);



			blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,

						  (u8 *)ctrblocks);



			src += 4;

			dst += 4;

		} while ((nbytes -= bsize * 4) >= bsize * 4);



		if (nbytes < bsize)

			goto done;

	}



	/* Handle leftovers */

	do {

		if (dst != src)

			*dst = *src;



		ctrblocks[0] = cpu_to_be64(ctrblk++);



		blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);



		src += 1;

		dst += 1;

	} while ((nbytes -= bsize) >= bsize);



done:

	*(__be64 *)walk->iv = cpu_to_be64(ctrblk);

	return nbytes;

}



static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,

		     struct scatterlist *src, unsigned int nbytes)

{

	struct blkcipher_walk walk;

	int err;



	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);



	while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {

		nbytes = __ctr_crypt(desc, &walk);

		err = blkcipher_walk_done(desc, &walk, nbytes);

	}



	if (walk.nbytes) {

		ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);

		err = blkcipher_walk_done(desc, &walk, 0);

	}



	return err;

}



static struct crypto_alg blk_ctr_alg = {

	.cra_name		= "ctr(blowfish)",

	.cra_driver_name	= "ctr-blowfish-asm",

	.cra_priority		= 300,

	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,

	.cra_blocksize		= 1,

	.cra_ctxsize		= sizeof(struct bf_ctx),

	.cra_alignmask		= 0,

	.cra_type		= &crypto_blkcipher_type,

	.cra_module		= THIS_MODULE,

	.cra_list		= LIST_HEAD_INIT(blk_ctr_alg.cra_list),

	.cra_u = {

		.blkcipher = {

			.min_keysize	= BF_MIN_KEY_SIZE,

			.max_keysize	= BF_MAX_KEY_SIZE,

			.ivsize		= BF_BLOCK_SIZE,

			.setkey		= blowfish_setkey,

			.encrypt	= ctr_crypt,

			.decrypt	= ctr_crypt,

		},

	},

};



static int __init init(void)

{

	int err;



	err = crypto_register_alg(&bf_alg);

	if (err)

		goto bf_err;

	err = crypto_register_alg(&blk_ecb_alg);

	if (err)

		goto ecb_err;

	err = crypto_register_alg(&blk_cbc_alg);

	if (err)

		goto cbc_err;

	err = crypto_register_alg(&blk_ctr_alg);

	if (err)

		goto ctr_err;



	return 0;



ctr_err:

	crypto_unregister_alg(&blk_cbc_alg);

cbc_err:

	crypto_unregister_alg(&blk_ecb_alg);

ecb_err:

	crypto_unregister_alg(&bf_alg);

bf_err:

	return err;

}



static void __exit fini(void)

{

	crypto_unregister_alg(&blk_ctr_alg);

	crypto_unregister_alg(&blk_cbc_alg);

	crypto_unregister_alg(&blk_ecb_alg);

	crypto_unregister_alg(&bf_alg);

}



module_init(init);

module_exit(fini);



MODULE_LICENSE("GPL");

MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");

MODULE_ALIAS("blowfish");

MODULE_ALIAS("blowfish-asm");