Loading arch/arm64/crypto/crc32-arm64.c +19 −3 Original line number Original line Diff line number Diff line Loading @@ -147,13 +147,21 @@ static int chksum_final(struct shash_desc *desc, u8 *out) { { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); put_unaligned_le32(ctx->crc, out); return 0; } static int chksumc_final(struct shash_desc *desc, u8 *out) { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); put_unaligned_le32(~ctx->crc, out); put_unaligned_le32(~ctx->crc, out); return 0; return 0; } } static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) { { put_unaligned_le32(~crc32_arm64_le_hw(crc, data, len), out); put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out); return 0; return 0; } } Loading Loading @@ -199,6 +207,14 @@ static int crc32_cra_init(struct crypto_tfm *tfm) { { struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); mctx->key = 0; return 0; } static int crc32c_cra_init(struct crypto_tfm *tfm) { struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); mctx->key = ~0; mctx->key = ~0; return 0; return 0; } } Loading Loading @@ -229,7 +245,7 @@ static struct shash_alg crc32c_alg = { .setkey = chksum_setkey, .setkey = chksum_setkey, .init = chksum_init, .init = chksum_init, .update = chksumc_update, .update = chksumc_update, .final = chksum_final, .final = chksumc_final, .finup = chksumc_finup, .finup = chksumc_finup, .digest = chksumc_digest, .digest = chksumc_digest, .descsize = sizeof(struct chksum_desc_ctx), .descsize = sizeof(struct chksum_desc_ctx), Loading @@ -241,7 +257,7 @@ static struct shash_alg crc32c_alg = { .cra_alignmask = 0, .cra_alignmask = 0, .cra_ctxsize = sizeof(struct chksum_ctx), .cra_ctxsize = sizeof(struct chksum_ctx), .cra_module = THIS_MODULE, .cra_module = THIS_MODULE, .cra_init = crc32_cra_init, .cra_init = crc32c_cra_init, } } }; }; Loading arch/arm64/crypto/sha1-ce-glue.c +3 −0 Original line number Original line Diff line number Diff line Loading @@ -74,6 +74,9 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, static int sha1_ce_final(struct shash_desc *desc, u8 *out) static int sha1_ce_final(struct shash_desc *desc, u8 *out) { { struct sha1_ce_state *sctx = shash_desc_ctx(desc); sctx->finalize = 0; kernel_neon_begin_partial(16); kernel_neon_begin_partial(16); sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); kernel_neon_end(); kernel_neon_end(); Loading arch/arm64/crypto/sha2-ce-glue.c +3 −0 Original line number Original line Diff line number Diff line Loading @@ -75,6 +75,9 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, static int sha256_ce_final(struct shash_desc *desc, u8 *out) static int sha256_ce_final(struct shash_desc *desc, u8 *out) { { struct sha256_ce_state *sctx = shash_desc_ctx(desc); sctx->finalize = 0; kernel_neon_begin_partial(28); kernel_neon_begin_partial(28); sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); kernel_neon_end(); kernel_neon_end(); Loading arch/s390/crypto/ghash_s390.c +13 −12 Original line number Original line Diff line number Diff line Loading @@ -16,11 +16,12 @@ #define GHASH_DIGEST_SIZE 16 #define GHASH_DIGEST_SIZE 16 struct ghash_ctx { struct ghash_ctx { u8 icv[16]; u8 key[GHASH_BLOCK_SIZE]; u8 key[16]; }; }; struct ghash_desc_ctx { struct ghash_desc_ctx { u8 icv[GHASH_BLOCK_SIZE]; u8 key[GHASH_BLOCK_SIZE]; u8 buffer[GHASH_BLOCK_SIZE]; u8 buffer[GHASH_BLOCK_SIZE]; u32 bytes; u32 bytes; }; }; Loading @@ -28,8 +29,10 @@ struct ghash_desc_ctx { static int ghash_init(struct shash_desc *desc) static int ghash_init(struct shash_desc *desc) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); memset(dctx, 0, sizeof(*dctx)); memset(dctx, 0, sizeof(*dctx)); memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); return 0; return 0; } } Loading @@ -45,7 +48,6 @@ static int ghash_setkey(struct crypto_shash *tfm, } } memcpy(ctx->key, key, GHASH_BLOCK_SIZE); memcpy(ctx->key, key, GHASH_BLOCK_SIZE); memset(ctx->icv, 0, GHASH_BLOCK_SIZE); return 0; return 0; } } Loading @@ -54,7 +56,6 @@ static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) const u8 *src, unsigned int srclen) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); unsigned int n; unsigned int n; u8 *buf = dctx->buffer; u8 *buf = dctx->buffer; int ret; int ret; Loading @@ -70,7 +71,7 @@ static int ghash_update(struct shash_desc *desc, src += n; src += n; if (!dctx->bytes) { if (!dctx->bytes) { ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) if (ret != GHASH_BLOCK_SIZE) return -EIO; return -EIO; Loading @@ -79,7 +80,7 @@ static int ghash_update(struct shash_desc *desc, n = srclen & ~(GHASH_BLOCK_SIZE - 1); n = srclen & ~(GHASH_BLOCK_SIZE - 1); if (n) { if (n) { ret = crypt_s390_kimd(KIMD_GHASH, ctx, src, n); ret = crypt_s390_kimd(KIMD_GHASH, dctx, src, n); if (ret != n) if (ret != n) return -EIO; return -EIO; src += n; src += n; Loading @@ -94,7 +95,7 @@ static int ghash_update(struct shash_desc *desc, return 0; return 0; } } static int ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) static int ghash_flush(struct ghash_desc_ctx *dctx) { { u8 *buf = dctx->buffer; u8 *buf = dctx->buffer; int ret; int ret; Loading @@ -104,24 +105,24 @@ static int ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) memset(pos, 0, dctx->bytes); memset(pos, 0, dctx->bytes); ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, GHASH_BLOCK_SIZE); ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) if (ret != GHASH_BLOCK_SIZE) return -EIO; return -EIO; } dctx->bytes = 0; dctx->bytes = 0; } return 0; return 0; } } static int ghash_final(struct shash_desc *desc, u8 *dst) static int ghash_final(struct shash_desc *desc, u8 *dst) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); int ret; int ret; ret = ghash_flush(ctx, dctx); ret = ghash_flush(dctx); if (!ret) if (!ret) memcpy(dst, ctx->icv, GHASH_BLOCK_SIZE); memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); return ret; return ret; } } Loading arch/x86/crypto/sha512-avx2-asm.S +1 −1 Original line number Original line Diff line number Diff line Loading @@ -79,7 +79,7 @@ NUM_BLKS = %rdx c = %rcx c = %rcx d = %r8 d = %r8 e = %rdx e = %rdx y3 = %rdi y3 = %rsi TBL = %rbp TBL = %rbp Loading Loading
arch/arm64/crypto/crc32-arm64.c +19 −3 Original line number Original line Diff line number Diff line Loading @@ -147,13 +147,21 @@ static int chksum_final(struct shash_desc *desc, u8 *out) { { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); put_unaligned_le32(ctx->crc, out); return 0; } static int chksumc_final(struct shash_desc *desc, u8 *out) { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); put_unaligned_le32(~ctx->crc, out); put_unaligned_le32(~ctx->crc, out); return 0; return 0; } } static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) { { put_unaligned_le32(~crc32_arm64_le_hw(crc, data, len), out); put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out); return 0; return 0; } } Loading Loading @@ -199,6 +207,14 @@ static int crc32_cra_init(struct crypto_tfm *tfm) { { struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); mctx->key = 0; return 0; } static int crc32c_cra_init(struct crypto_tfm *tfm) { struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); mctx->key = ~0; mctx->key = ~0; return 0; return 0; } } Loading Loading @@ -229,7 +245,7 @@ static struct shash_alg crc32c_alg = { .setkey = chksum_setkey, .setkey = chksum_setkey, .init = chksum_init, .init = chksum_init, .update = chksumc_update, .update = chksumc_update, .final = chksum_final, .final = chksumc_final, .finup = chksumc_finup, .finup = chksumc_finup, .digest = chksumc_digest, .digest = chksumc_digest, .descsize = sizeof(struct chksum_desc_ctx), .descsize = sizeof(struct chksum_desc_ctx), Loading @@ -241,7 +257,7 @@ static struct shash_alg crc32c_alg = { .cra_alignmask = 0, .cra_alignmask = 0, .cra_ctxsize = sizeof(struct chksum_ctx), .cra_ctxsize = sizeof(struct chksum_ctx), .cra_module = THIS_MODULE, .cra_module = THIS_MODULE, .cra_init = crc32_cra_init, .cra_init = crc32c_cra_init, } } }; }; Loading
arch/arm64/crypto/sha1-ce-glue.c +3 −0 Original line number Original line Diff line number Diff line Loading @@ -74,6 +74,9 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, static int sha1_ce_final(struct shash_desc *desc, u8 *out) static int sha1_ce_final(struct shash_desc *desc, u8 *out) { { struct sha1_ce_state *sctx = shash_desc_ctx(desc); sctx->finalize = 0; kernel_neon_begin_partial(16); kernel_neon_begin_partial(16); sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); kernel_neon_end(); kernel_neon_end(); Loading
arch/arm64/crypto/sha2-ce-glue.c +3 −0 Original line number Original line Diff line number Diff line Loading @@ -75,6 +75,9 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, static int sha256_ce_final(struct shash_desc *desc, u8 *out) static int sha256_ce_final(struct shash_desc *desc, u8 *out) { { struct sha256_ce_state *sctx = shash_desc_ctx(desc); sctx->finalize = 0; kernel_neon_begin_partial(28); kernel_neon_begin_partial(28); sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); kernel_neon_end(); kernel_neon_end(); Loading
arch/s390/crypto/ghash_s390.c +13 −12 Original line number Original line Diff line number Diff line Loading @@ -16,11 +16,12 @@ #define GHASH_DIGEST_SIZE 16 #define GHASH_DIGEST_SIZE 16 struct ghash_ctx { struct ghash_ctx { u8 icv[16]; u8 key[GHASH_BLOCK_SIZE]; u8 key[16]; }; }; struct ghash_desc_ctx { struct ghash_desc_ctx { u8 icv[GHASH_BLOCK_SIZE]; u8 key[GHASH_BLOCK_SIZE]; u8 buffer[GHASH_BLOCK_SIZE]; u8 buffer[GHASH_BLOCK_SIZE]; u32 bytes; u32 bytes; }; }; Loading @@ -28,8 +29,10 @@ struct ghash_desc_ctx { static int ghash_init(struct shash_desc *desc) static int ghash_init(struct shash_desc *desc) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); memset(dctx, 0, sizeof(*dctx)); memset(dctx, 0, sizeof(*dctx)); memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); return 0; return 0; } } Loading @@ -45,7 +48,6 @@ static int ghash_setkey(struct crypto_shash *tfm, } } memcpy(ctx->key, key, GHASH_BLOCK_SIZE); memcpy(ctx->key, key, GHASH_BLOCK_SIZE); memset(ctx->icv, 0, GHASH_BLOCK_SIZE); return 0; return 0; } } Loading @@ -54,7 +56,6 @@ static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) const u8 *src, unsigned int srclen) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); unsigned int n; unsigned int n; u8 *buf = dctx->buffer; u8 *buf = dctx->buffer; int ret; int ret; Loading @@ -70,7 +71,7 @@ static int ghash_update(struct shash_desc *desc, src += n; src += n; if (!dctx->bytes) { if (!dctx->bytes) { ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) if (ret != GHASH_BLOCK_SIZE) return -EIO; return -EIO; Loading @@ -79,7 +80,7 @@ static int ghash_update(struct shash_desc *desc, n = srclen & ~(GHASH_BLOCK_SIZE - 1); n = srclen & ~(GHASH_BLOCK_SIZE - 1); if (n) { if (n) { ret = crypt_s390_kimd(KIMD_GHASH, ctx, src, n); ret = crypt_s390_kimd(KIMD_GHASH, dctx, src, n); if (ret != n) if (ret != n) return -EIO; return -EIO; src += n; src += n; Loading @@ -94,7 +95,7 @@ static int ghash_update(struct shash_desc *desc, return 0; return 0; } } static int ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) static int ghash_flush(struct ghash_desc_ctx *dctx) { { u8 *buf = dctx->buffer; u8 *buf = dctx->buffer; int ret; int ret; Loading @@ -104,24 +105,24 @@ static int ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) memset(pos, 0, dctx->bytes); memset(pos, 0, dctx->bytes); ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, GHASH_BLOCK_SIZE); ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) if (ret != GHASH_BLOCK_SIZE) return -EIO; return -EIO; } dctx->bytes = 0; dctx->bytes = 0; } return 0; return 0; } } static int ghash_final(struct shash_desc *desc, u8 *dst) static int ghash_final(struct shash_desc *desc, u8 *dst) { { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); int ret; int ret; ret = ghash_flush(ctx, dctx); ret = ghash_flush(dctx); if (!ret) if (!ret) memcpy(dst, ctx->icv, GHASH_BLOCK_SIZE); memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); return ret; return ret; } } Loading
arch/x86/crypto/sha512-avx2-asm.S +1 −1 Original line number Original line Diff line number Diff line Loading @@ -79,7 +79,7 @@ NUM_BLKS = %rdx c = %rcx c = %rcx d = %r8 d = %r8 e = %rdx e = %rdx y3 = %rdi y3 = %rsi TBL = %rbp TBL = %rbp Loading