Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b2c31107 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6 

Pull crypto update from Herbert Xu:
 - Do not idle omap device between crypto operations in one session.
 - Added sha224/sha384 shims for SSSE3.
 - More optimisations for camellia-aesni-avx2.
 - Removed defunct blowfish/twofish AVX2 implementations.
 - Added unaligned buffer self-tests.
 - Added PCLMULQDQ optimisation for CRCT10DIF.
 - Added support for Freescale's DCP co-processor
 - Misc fixes.

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (44 commits)
  crypto: testmgr - test hash implementations with unaligned buffers
  crypto: testmgr - test AEADs with unaligned buffers
  crypto: testmgr - test skciphers with unaligned buffers
  crypto: testmgr - check that entries in alg_test_descs are in correct order
  Revert "crypto: twofish - add AVX2/x86_64 assembler implementation of twofish cipher"
  Revert "crypto: blowfish - add AVX2/x86_64 implementation of blowfish cipher"
  crypto: camellia-aesni-avx2 - tune assembly code for more performance
  hwrng: bcm2835 - fix MODULE_LICENSE tag
  hwrng: nomadik - use clk_prepare_enable()
  crypto: picoxcell - replace strict_strtoul() with kstrtoul()
  crypto: dcp - Staticize local symbols
  crypto: dcp - Use NULL instead of 0
  crypto: dcp - Use devm_* APIs
  crypto: dcp - Remove redundant platform_set_drvdata()
  hwrng: use platform_{get,set}_drvdata()
  crypto: omap-aes - Don't idle/start AES device between Encrypt operations
  crypto: crct10dif - Use PTR_RET
  crypto: ux500 - Cocci spatch "resource_size.spatch"
  crypto: sha256_ssse3 - add sha224 support
  crypto: sha512_ssse3 - add sha384 support
  ...
parents 45175476 02c0241b

arch/arm/boot/dts/imx28.dtsi

+1 −1
Original line number Diff line number Diff line
@@ -736,7 +736,7 @@ 
			dcp@80028000 {

				reg = <0x80028000 0x2000>;

				interrupts = <52 53 54>;

				status = "disabled";

				compatible = "fsl-dcp";

			};



			pxp@8002a000 {

arch/x86/crypto/Makefile

+2 −6
Original line number Diff line number Diff line
@@ -3,8 +3,6 @@ 
#



avx_supported := $(call as-instr,vpxor %xmm0$(comma)%xmm0$(comma)%xmm0,yes,no)

avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\

					$(comma)4)$(comma)%ymm2,yes,no)



obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o

obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o
@@ -29,6 +27,7 @@ obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o 
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o

obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o

obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o

obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o



# These modules require assembler to support AVX.

ifeq ($(avx_supported),yes)
@@ -42,10 +41,8 @@ endif 


# These modules require assembler to support AVX2.

ifeq ($(avx2_supported),yes)

	obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o

	obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o

	obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o

	obj-$(CONFIG_CRYPTO_TWOFISH_AVX2_X86_64) += twofish-avx2.o

endif



aes-i586-y := aes-i586-asm_32.o aes_glue.o
@@ -73,10 +70,8 @@ ifeq ($(avx_supported),yes) 
endif



ifeq ($(avx2_supported),yes)

	blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o

	camellia-aesni-avx2-y := camellia-aesni-avx2-asm_64.o camellia_aesni_avx2_glue.o

	serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o

	twofish-avx2-y := twofish-avx2-asm_64.o twofish_avx2_glue.o

endif



aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
@@ -87,3 +82,4 @@ crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o 
crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o

sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o

sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o

crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o

arch/x86/crypto/blowfish-avx2-asm_64.S

deleted100644 → 0
+0 −449
Original line number Diff line number Diff line 
/*

 * x86_64/AVX2 assembler optimized version of Blowfish

 *

 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 */



#include <linux/linkage.h>



.file "blowfish-avx2-asm_64.S"



.data

.align 32



.Lprefetch_mask:

.long 0*64

.long 1*64

.long 2*64

.long 3*64

.long 4*64

.long 5*64

.long 6*64

.long 7*64



.Lbswap32_mask:

.long 0x00010203

.long 0x04050607

.long 0x08090a0b

.long 0x0c0d0e0f



.Lbswap128_mask:

	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0

.Lbswap_iv_mask:

	.byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0



.text

/* structure of crypto context */

#define p	0

#define s0	((16 + 2) * 4)

#define s1	((16 + 2 + (1 * 256)) * 4)

#define s2	((16 + 2 + (2 * 256)) * 4)

#define s3	((16 + 2 + (3 * 256)) * 4)



/* register macros */

#define CTX	%rdi

#define RIO	 %rdx



#define RS0	%rax

#define RS1	%r8

#define RS2	%r9

#define RS3	%r10



#define RLOOP	%r11

#define RLOOPd	%r11d



#define RXr0	%ymm8

#define RXr1	%ymm9

#define RXr2	%ymm10

#define RXr3	%ymm11

#define RXl0	%ymm12

#define RXl1	%ymm13

#define RXl2	%ymm14

#define RXl3	%ymm15



/* temp regs */

#define RT0	%ymm0

#define RT0x	%xmm0

#define RT1	%ymm1

#define RT1x	%xmm1

#define RIDX0	%ymm2

#define RIDX1	%ymm3

#define RIDX1x	%xmm3

#define RIDX2	%ymm4

#define RIDX3	%ymm5



/* vpgatherdd mask and '-1' */

#define RNOT	%ymm6



/* byte mask, (-1 >> 24) */

#define RBYTE	%ymm7



/***********************************************************************

 * 32-way AVX2 blowfish

 ***********************************************************************/

#define F(xl, xr) \

	vpsrld $24, xl, RIDX0; \

	vpsrld $16, xl, RIDX1; \

	vpsrld $8, xl, RIDX2; \

	vpand RBYTE, RIDX1, RIDX1; \

	vpand RBYTE, RIDX2, RIDX2; \

	vpand RBYTE, xl, RIDX3; \

	\

	vpgatherdd RNOT, (RS0, RIDX0, 4), RT0; \

	vpcmpeqd RNOT, RNOT, RNOT; \

	vpcmpeqd RIDX0, RIDX0, RIDX0; \

	\

	vpgatherdd RNOT, (RS1, RIDX1, 4), RT1; \

	vpcmpeqd RIDX1, RIDX1, RIDX1; \

	vpaddd RT0, RT1, RT0; \

	\

	vpgatherdd RIDX0, (RS2, RIDX2, 4), RT1; \

	vpxor RT0, RT1, RT0; \

	\

	vpgatherdd RIDX1, (RS3, RIDX3, 4), RT1; \

	vpcmpeqd RNOT, RNOT, RNOT; \

	vpaddd RT0, RT1, RT0; \

	\

	vpxor RT0, xr, xr;



#define add_roundkey(xl, nmem) \

	vpbroadcastd nmem, RT0; \

	vpxor RT0, xl ## 0, xl ## 0; \

	vpxor RT0, xl ## 1, xl ## 1; \

	vpxor RT0, xl ## 2, xl ## 2; \

	vpxor RT0, xl ## 3, xl ## 3;



#define round_enc() \

	add_roundkey(RXr, p(CTX,RLOOP,4)); \

	F(RXl0, RXr0); \

	F(RXl1, RXr1); \

	F(RXl2, RXr2); \

	F(RXl3, RXr3); \

	\

	add_roundkey(RXl, p+4(CTX,RLOOP,4)); \

	F(RXr0, RXl0); \

	F(RXr1, RXl1); \

	F(RXr2, RXl2); \

	F(RXr3, RXl3);



#define round_dec() \

	add_roundkey(RXr, p+4*2(CTX,RLOOP,4)); \

	F(RXl0, RXr0); \

	F(RXl1, RXr1); \

	F(RXl2, RXr2); \

	F(RXl3, RXr3); \

	\

	add_roundkey(RXl, p+4(CTX,RLOOP,4)); \

	F(RXr0, RXl0); \

	F(RXr1, RXl1); \

	F(RXr2, RXl2); \

	F(RXr3, RXl3);



#define init_round_constants() \

	vpcmpeqd RNOT, RNOT, RNOT; \

	leaq s0(CTX), RS0; \

	leaq s1(CTX), RS1; \

	leaq s2(CTX), RS2; \

	leaq s3(CTX), RS3; \

	vpsrld $24, RNOT, RBYTE;



#define transpose_2x2(x0, x1, t0) \

	vpunpckldq x0, x1, t0; \

	vpunpckhdq x0, x1, x1; \

	\

	vpunpcklqdq t0, x1, x0; \

	vpunpckhqdq t0, x1, x1;



#define read_block(xl, xr) \

	vbroadcasti128 .Lbswap32_mask, RT1; \

	\

	vpshufb RT1, xl ## 0, xl ## 0; \

	vpshufb RT1, xr ## 0, xr ## 0; \

	vpshufb RT1, xl ## 1, xl ## 1; \

	vpshufb RT1, xr ## 1, xr ## 1; \

	vpshufb RT1, xl ## 2, xl ## 2; \

	vpshufb RT1, xr ## 2, xr ## 2; \

	vpshufb RT1, xl ## 3, xl ## 3; \

	vpshufb RT1, xr ## 3, xr ## 3; \

	\

	transpose_2x2(xl ## 0, xr ## 0, RT0); \

	transpose_2x2(xl ## 1, xr ## 1, RT0); \

	transpose_2x2(xl ## 2, xr ## 2, RT0); \

	transpose_2x2(xl ## 3, xr ## 3, RT0);



#define write_block(xl, xr) \

	vbroadcasti128 .Lbswap32_mask, RT1; \

	\

	transpose_2x2(xl ## 0, xr ## 0, RT0); \

	transpose_2x2(xl ## 1, xr ## 1, RT0); \

	transpose_2x2(xl ## 2, xr ## 2, RT0); \

	transpose_2x2(xl ## 3, xr ## 3, RT0); \

	\

	vpshufb RT1, xl ## 0, xl ## 0; \

	vpshufb RT1, xr ## 0, xr ## 0; \

	vpshufb RT1, xl ## 1, xl ## 1; \

	vpshufb RT1, xr ## 1, xr ## 1; \

	vpshufb RT1, xl ## 2, xl ## 2; \

	vpshufb RT1, xr ## 2, xr ## 2; \

	vpshufb RT1, xl ## 3, xl ## 3; \

	vpshufb RT1, xr ## 3, xr ## 3;



.align 8

__blowfish_enc_blk32:

	/* input:

	 *	%rdi: ctx, CTX

	 *	RXl0..4, RXr0..4: plaintext

	 * output:

	 *	RXl0..4, RXr0..4: ciphertext (RXl <=> RXr swapped)

	 */

	init_round_constants();



	read_block(RXl, RXr);



	movl $1, RLOOPd;

	add_roundkey(RXl, p+4*(0)(CTX));



.align 4

.L__enc_loop:

	round_enc();



	leal 2(RLOOPd), RLOOPd;

	cmpl $17, RLOOPd;

	jne .L__enc_loop;



	add_roundkey(RXr, p+4*(17)(CTX));



	write_block(RXl, RXr);



	ret;

ENDPROC(__blowfish_enc_blk32)



.align 8

__blowfish_dec_blk32:

	/* input:

	 *	%rdi: ctx, CTX

	 *	RXl0..4, RXr0..4: ciphertext

	 * output:

	 *	RXl0..4, RXr0..4: plaintext (RXl <=> RXr swapped)

	 */

	init_round_constants();



	read_block(RXl, RXr);



	movl $14, RLOOPd;

	add_roundkey(RXl, p+4*(17)(CTX));



.align 4

.L__dec_loop:

	round_dec();



	addl $-2, RLOOPd;

	jns .L__dec_loop;



	add_roundkey(RXr, p+4*(0)(CTX));



	write_block(RXl, RXr);



	ret;

ENDPROC(__blowfish_dec_blk32)



ENTRY(blowfish_ecb_enc_32way)

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 */



	vzeroupper;



	vmovdqu 0*32(%rdx), RXl0;

	vmovdqu 1*32(%rdx), RXr0;

	vmovdqu 2*32(%rdx), RXl1;

	vmovdqu 3*32(%rdx), RXr1;

	vmovdqu 4*32(%rdx), RXl2;

	vmovdqu 5*32(%rdx), RXr2;

	vmovdqu 6*32(%rdx), RXl3;

	vmovdqu 7*32(%rdx), RXr3;



	call __blowfish_enc_blk32;



	vmovdqu RXr0, 0*32(%rsi);

	vmovdqu RXl0, 1*32(%rsi);

	vmovdqu RXr1, 2*32(%rsi);

	vmovdqu RXl1, 3*32(%rsi);

	vmovdqu RXr2, 4*32(%rsi);

	vmovdqu RXl2, 5*32(%rsi);

	vmovdqu RXr3, 6*32(%rsi);

	vmovdqu RXl3, 7*32(%rsi);



	vzeroupper;



	ret;

ENDPROC(blowfish_ecb_enc_32way)



ENTRY(blowfish_ecb_dec_32way)

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 */



	vzeroupper;



	vmovdqu 0*32(%rdx), RXl0;

	vmovdqu 1*32(%rdx), RXr0;

	vmovdqu 2*32(%rdx), RXl1;

	vmovdqu 3*32(%rdx), RXr1;

	vmovdqu 4*32(%rdx), RXl2;

	vmovdqu 5*32(%rdx), RXr2;

	vmovdqu 6*32(%rdx), RXl3;

	vmovdqu 7*32(%rdx), RXr3;



	call __blowfish_dec_blk32;



	vmovdqu RXr0, 0*32(%rsi);

	vmovdqu RXl0, 1*32(%rsi);

	vmovdqu RXr1, 2*32(%rsi);

	vmovdqu RXl1, 3*32(%rsi);

	vmovdqu RXr2, 4*32(%rsi);

	vmovdqu RXl2, 5*32(%rsi);

	vmovdqu RXr3, 6*32(%rsi);

	vmovdqu RXl3, 7*32(%rsi);



	vzeroupper;



	ret;

ENDPROC(blowfish_ecb_dec_32way)



ENTRY(blowfish_cbc_dec_32way)

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 */



	vzeroupper;



	vmovdqu 0*32(%rdx), RXl0;

	vmovdqu 1*32(%rdx), RXr0;

	vmovdqu 2*32(%rdx), RXl1;

	vmovdqu 3*32(%rdx), RXr1;

	vmovdqu 4*32(%rdx), RXl2;

	vmovdqu 5*32(%rdx), RXr2;

	vmovdqu 6*32(%rdx), RXl3;

	vmovdqu 7*32(%rdx), RXr3;



	call __blowfish_dec_blk32;



	/* xor with src */

	vmovq (%rdx), RT0x;

	vpshufd $0x4f, RT0x, RT0x;

	vinserti128 $1, 8(%rdx), RT0, RT0;

	vpxor RT0, RXr0, RXr0;

	vpxor 0*32+24(%rdx), RXl0, RXl0;

	vpxor 1*32+24(%rdx), RXr1, RXr1;

	vpxor 2*32+24(%rdx), RXl1, RXl1;

	vpxor 3*32+24(%rdx), RXr2, RXr2;

	vpxor 4*32+24(%rdx), RXl2, RXl2;

	vpxor 5*32+24(%rdx), RXr3, RXr3;

	vpxor 6*32+24(%rdx), RXl3, RXl3;



	vmovdqu RXr0, (0*32)(%rsi);

	vmovdqu RXl0, (1*32)(%rsi);

	vmovdqu RXr1, (2*32)(%rsi);

	vmovdqu RXl1, (3*32)(%rsi);

	vmovdqu RXr2, (4*32)(%rsi);

	vmovdqu RXl2, (5*32)(%rsi);

	vmovdqu RXr3, (6*32)(%rsi);

	vmovdqu RXl3, (7*32)(%rsi);



	vzeroupper;



	ret;

ENDPROC(blowfish_cbc_dec_32way)



ENTRY(blowfish_ctr_32way)

	/* input:

	 *	%rdi: ctx, CTX

	 *	%rsi: dst

	 *	%rdx: src

	 *	%rcx: iv (big endian, 64bit)

	 */



	vzeroupper;



	vpcmpeqd RT0, RT0, RT0;

	vpsrldq $8, RT0, RT0; /* a: -1, b: 0, c: -1, d: 0 */



	vpcmpeqd RT1x, RT1x, RT1x;

	vpaddq RT1x, RT1x, RT1x; /* a: -2, b: -2 */

	vpxor RIDX0, RIDX0, RIDX0;

	vinserti128 $1, RT1x, RIDX0, RIDX0; /* a: 0, b: 0, c: -2, d: -2 */



	vpaddq RIDX0, RT0, RT0; /* a: -1, b: 0, c: -3, d: -2 */



	vpcmpeqd RT1, RT1, RT1;

	vpaddq RT1, RT1, RT1; /* a: -2, b: -2, c: -2, d: -2 */

	vpaddq RT1, RT1, RIDX2; /* a: -4, b: -4, c: -4, d: -4 */



	vbroadcasti128 .Lbswap_iv_mask, RIDX0;

	vbroadcasti128 .Lbswap128_mask, RIDX1;



	/* load IV and byteswap */

	vmovq (%rcx), RT1x;

	vinserti128 $1, RT1x, RT1, RT1; /* a: BE, b: 0, c: BE, d: 0 */

	vpshufb RIDX0, RT1, RT1; /* a: LE, b: LE, c: LE, d: LE */



	/* construct IVs */

	vpsubq RT0, RT1, RT1;		/* a: le1, b: le0, c: le3, d: le2 */

	vpshufb RIDX1, RT1, RXl0;	/* a: be0, b: be1, c: be2, d: be3 */

	vpsubq RIDX2, RT1, RT1;		/* le5, le4, le7, le6 */

	vpshufb RIDX1, RT1, RXr0;	/* be4, be5, be6, be7 */

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXl1;

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXr1;

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXl2;

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXr2;

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXl3;

	vpsubq RIDX2, RT1, RT1;

	vpshufb RIDX1, RT1, RXr3;



	/* store last IV */

	vpsubq RIDX2, RT1, RT1; /* a: le33, b: le32, ... */

	vpshufb RIDX1x, RT1x, RT1x; /* a: be32, ... */

	vmovq RT1x, (%rcx);



	call __blowfish_enc_blk32;



	/* dst = src ^ iv */

	vpxor 0*32(%rdx), RXr0, RXr0;

	vpxor 1*32(%rdx), RXl0, RXl0;

	vpxor 2*32(%rdx), RXr1, RXr1;

	vpxor 3*32(%rdx), RXl1, RXl1;

	vpxor 4*32(%rdx), RXr2, RXr2;

	vpxor 5*32(%rdx), RXl2, RXl2;

	vpxor 6*32(%rdx), RXr3, RXr3;

	vpxor 7*32(%rdx), RXl3, RXl3;

	vmovdqu RXr0, (0*32)(%rsi);

	vmovdqu RXl0, (1*32)(%rsi);

	vmovdqu RXr1, (2*32)(%rsi);

	vmovdqu RXl1, (3*32)(%rsi);

	vmovdqu RXr2, (4*32)(%rsi);

	vmovdqu RXl2, (5*32)(%rsi);

	vmovdqu RXr3, (6*32)(%rsi);

	vmovdqu RXl3, (7*32)(%rsi);



	vzeroupper;



	ret;

ENDPROC(blowfish_ctr_32way)

arch/x86/crypto/blowfish_avx2_glue.c

deleted100644 → 0
+0 −585

File deleted.

Preview size limit exceeded, changes collapsed.

arch/x86/crypto/blowfish_glue.c

+24 −8
Original line number Diff line number Diff line 
/*

 * Glue Code for assembler optimized version of Blowfish

 *

 * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>

 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>

 *

 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:

 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
@@ -32,24 +32,40 @@ 
#include <linux/module.h>

#include <linux/types.h>

#include <crypto/algapi.h>

#include <asm/crypto/blowfish.h>



/* regular block cipher functions */

asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,

				   bool xor);

EXPORT_SYMBOL_GPL(__blowfish_enc_blk);



asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);

EXPORT_SYMBOL_GPL(blowfish_dec_blk);



/* 4-way parallel cipher functions */

asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,

					const u8 *src, bool xor);

EXPORT_SYMBOL_GPL(__blowfish_enc_blk_4way);



asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,

				      const u8 *src);

EXPORT_SYMBOL_GPL(blowfish_dec_blk_4way);



static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)

{

	__blowfish_enc_blk(ctx, dst, src, false);

}



static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,

					const u8 *src)

{

	__blowfish_enc_blk(ctx, dst, src, true);

}



static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,

					 const u8 *src)

{

	__blowfish_enc_blk_4way(ctx, dst, src, false);

}



static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,

				      const u8 *src)

{

	__blowfish_enc_blk_4way(ctx, dst, src, true);

}



static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{