Verified Commit c81e05c4 authored Sep 24, 2025 by Alexander Martinz

Merge branch 'wireguard' into lineage-20

Changes are cherry-picked from kernel/common @ android12-5.4
into a branch called "wireguard" and then merged into lineage-20
via: `git merge --log=99999999 --no-ff wireguard`.

-----

By Jason A. Donenfeld (92) and others
* wireguard:
UPSTREAM: wireguard: send: annotate intentional data race in checking empty queue
UPSTREAM: wireguard: queueing: annotate intentional data race in cpu round robin
UPSTREAM: wireguard: allowedips: avoid unaligned 64-bit memory accesses
UPSTREAM: wireguard: netlink: access device through ctx instead of peer
UPSTREAM: wireguard: netlink: check for dangling peer via is_dead instead of empty list
UPSTREAM: wireguard: receive: annotate data-race around receiving_counter.counter
UPSTREAM: wireguard: allowedips: expand maximum node depth
UPSTREAM: wireguard: netlink: send staged packets when setting initial private key
UPSTREAM: wireguard: queueing: use saner cpu selection wrapping
UPSTREAM: wireguard: netlink: avoid variable-sized memcpy on sockaddr
UPSTREAM: wireguard: ratelimiter: disable timings test by default
UPSTREAM: wireguard: allowedips: don't corrupt stack when detecting overflow
UPSTREAM: wireguard: ratelimiter: use hrtimer in selftest
UPSTREAM: crypto: arm64/poly1305 - fix a read out-of-bound
UPSTREAM: wireguard: device: check for metadata_dst with skb_valid_dst()
UPSTREAM: wireguard: socket: ignore v6 endpoints when ipv6 is disabled
UPSTREAM: wireguard: socket: free skb in send6 when ipv6 is disabled
UPSTREAM: wireguard: queueing: use CFI-safe ptr_ring cleanup function
UPSTREAM: wireguard: allowedips: add missing __rcu annotation to satisfy sparse
UPSTREAM: wireguard: ratelimiter: use kvcalloc() instead of kvzalloc()
UPSTREAM: wireguard: receive: drop handshakes if queue lock is contended
ANDROID: properly copy the scm_io_uring field in struct sk_buff
UPSTREAM: wireguard: receive: use ring buffer for incoming handshakes
UPSTREAM: wireguard: device: reset peer src endpoint when netns exits
UPSTREAM: wireguard: selftests: actually test for routing loops
UPSTREAM: wireguard: selftests: increase default dmesg log size
UPSTREAM: crypto: x86/curve25519 - fix cpu feature checking logic in mod_exit
BACKPORT: wireguard: peer: allocate in kmem_cache
UPSTREAM: crypto: mips: add poly1305-core.S to .gitignore
UPSTREAM: crypto: poly1305 - fix poly1305_core_setkey() declaration
UPSTREAM: crypto: mips/poly1305 - enable for all MIPS processors
UPSTREAM: wireguard: kconfig: use arm chacha even with no neon
UPSTREAM: wireguard: queueing: get rid of per-peer ring buffers
UPSTREAM: wireguard: device: do not generate ICMP for non-IP packets
UPSTREAM: wireguard: selftests: test multiple parallel streams
UPSTREAM: crypto: Kconfig - CRYPTO_MANAGER_EXTRA_TESTS requires the manager
UPSTREAM: wireguard: allowedips: free empty intermediate nodes when removing single node
UPSTREAM: wireguard: allowedips: allocate nodes in kmem_cache
UPSTREAM: wireguard: selftests: remove old conntrack kconfig value
UPSTREAM: wireguard: allowedips: remove nodes in O(1)
UPSTREAM: wireguard: allowedips: initialize list head in selftest
UPSTREAM: wireguard: selftests: make sure rp_filter is disabled on vethc
UPSTREAM: wireguard: use synchronize_net rather than synchronize_rcu
UPSTREAM: wireguard: do not use -O3
UPSTREAM: crypto: arm/curve25519 - Move '.fpu' after '.arch'
UPSTREAM: crypto: jitter - SP800-90B compliance
UPSTREAM: crypto: jitter - add header to fix buildwarnings
UPSTREAM: crypto: jitter - fix comments
UPSTREAM: wireguard: Kconfig: select CRYPTO_BLAKE2S_ARM
FROMLIST: crypto: arm64/poly1305-neon - reorder PAC authentication with SP update
crypto: lib/sha256 - add sha256() function
crypto: lib/sha256 - return void
UPSTREAM: wireguard: peerlookup: take lock before checking hash in replace operation
UPSTREAM: wireguard: noise: take lock when removing handshake entry from table
UPSTREAM: netlink: consistently use NLA_POLICY_MIN_LEN()
UPSTREAM: netlink: consistently use NLA_POLICY_EXACT_LEN()
UPSTREAM: wireguard: queueing: make use of ip_tunnel_parse_protocol
UPSTREAM: wireguard: implement header_ops->parse_protocol for AF_PACKET
UPSTREAM: net: ip_tunnel: add header_ops for layer 3 devices
UPSTREAM: wireguard: receive: account for napi_gro_receive never returning GRO_DROP
UPSTREAM: wireguard: device: avoid circular netns references
UPSTREAM: wireguard: noise: do not assign initiation time in if condition
UPSTREAM: wireguard: noise: separate receive counter from send counter
UPSTREAM: wireguard: queueing: preserve flow hash across packet scrubbing
UPSTREAM: wireguard: noise: read preshared key while taking lock
UPSTREAM: wireguard: selftests: use newer iproute2 for gcc-10
UPSTREAM: wireguard: send/receive: use explicit unlikely branch instead of implicit coalescing
UPSTREAM: wireguard: selftests: initalize ipv6 members to NULL to squelch clang warning
UPSTREAM: wireguard: send/receive: cond_resched() when processing worker ringbuffers
UPSTREAM: wireguard: socket: remove errant restriction on looping to self
UPSTREAM: wireguard: selftests: use normal kernel stack size on ppc64
UPSTREAM: wireguard: receive: use tunnel helpers for decapsulating ECN markings
UPSTREAM: wireguard: queueing: cleanup ptr_ring in error path of packet_queue_init
UPSTREAM: wireguard: send: remove errant newline from packet_encrypt_worker
UPSTREAM: wireguard: noise: error out precomputed DH during handshake rather than config
UPSTREAM: wireguard: receive: remove dead code from default packet type case
UPSTREAM: wireguard: queueing: account for skb->protocol==0
UPSTREAM: wireguard: selftests: remove duplicated include <sys/types.h>
UPSTREAM: wireguard: socket: remove extra call to synchronize_net
UPSTREAM: wireguard: send: account for mtu=0 devices
UPSTREAM: wireguard: receive: reset last_under_load to zero
UPSTREAM: wireguard: selftests: reduce complexity and fix make races
UPSTREAM: wireguard: device: use icmp_ndo_send helper
UPSTREAM: wireguard: selftests: tie socket waiting to target pid
UPSTREAM: wireguard: selftests: ensure non-addition of peers with failed precomputation
UPSTREAM: wireguard: noise: reject peers with low order public keys
UPSTREAM: wireguard: allowedips: fix use-after-free in root_remove_peer_lists
UPSTREAM: wireguard: socket: mark skbs as not on list when receiving via gro
UPSTREAM: wireguard: queueing: do not account for pfmemalloc when clearing skb header
UPSTREAM: wireguard: selftests: remove ancient kernel compatibility code
UPSTREAM: wireguard: allowedips: use kfree_rcu() instead of call_rcu()
UPSTREAM: wireguard: main: remove unused include <linux/version.h>
ANDROID: GKI: enable CONFIG_WIREGUARD
UPSTREAM: wireguard: global: fix spelling mistakes in comments
UPSTREAM: wireguard: Kconfig: select parent dependency for crypto
UPSTREAM: wireguard: selftests: import harness makefile for test suite
lib/crypto: blake2s: move hmac construction into wireguard
UPSTREAM: net: introduce skb_list_walk_safe for skb segment walking
UPSTREAM: net: WireGuard secure network tunnel
UPSTREAM: crypto: poly1305-x86_64 - Use XORL r32,32
UPSTREAM: crypto: curve25519-x86_64 - Use XORL r32,32
UPSTREAM: crypto: arm/poly1305 - Add prototype for poly1305_blocks_neon
UPSTREAM: crypto: arm/curve25519 - include <linux/scatterlist.h>
UPSTREAM: crypto: x86/curve25519 - Remove unused carry variables
UPSTREAM: crypto: x86/chacha-sse3 - use unaligned loads for state array
UPSTREAM: crypto: lib/chacha20poly1305 - Add missing function declaration
UPSTREAM: crypto: arch/lib - limit simd usage to 4k chunks
UPSTREAM: crypto: arm[64]/poly1305 - add artifact to .gitignore files
UPSTREAM: crypto: x86/curve25519 - leave r12 as spare register
UPSTREAM: crypto: x86/curve25519 - replace with formally verified implementation
UPSTREAM: crypto: arm64/chacha - correctly walk through blocks
UPSTREAM: crypto: x86/curve25519 - support assemblers with no adx support
UPSTREAM: crypto: chacha20poly1305 - prevent integer overflow on large input
UPSTREAM: crypto: Kconfig - allow tests to be disabled when manager is disabled
UPSTREAM: crypto: arm/chacha - fix build failured when kernel mode NEON is disabled
UPSTREAM: crypto: x86/poly1305 - emit does base conversion itself
UPSTREAM: crypto: chacha20poly1305 - add back missing test vectors and test chunking
UPSTREAM: crypto: x86/poly1305 - fix .gitignore typo
UPSTREAM: crypto: curve25519 - Fix selftest build error
UPSTREAM: crypto: {arm,arm64,mips}/poly1305 - remove redundant non-reduction from emit
UPSTREAM: crypto: x86/poly1305 - wire up faster implementations for kernel
UPSTREAM: crypto: x86/poly1305 - import unmodified cryptogams implementation
UPSTREAM: crypto: poly1305 - add new 32 and 64-bit generic versions
UPSTREAM: crypto: lib/curve25519 - re-add selftests
UPSTREAM: crypto: arm/curve25519 - add arch-specific key generation function
UPSTREAM: crypto: chacha - fix warning message in header file
BACKPORT: crypto: arch - conditionalize crypto api in arch glue for lib code
UPSTREAM: crypto: lib/chacha20poly1305 - use chacha20_crypt()
UPSTREAM: crypto: x86/chacha - only unregister algorithms if registered
UPSTREAM: crypto: chacha_generic - remove unnecessary setkey() functions
UPSTREAM: crypto: lib/chacha20poly1305 - reimplement crypt_from_sg() routine
UPSTREAM: crypto: chacha20poly1305 - import construction and selftest from Zinc
UPSTREAM: crypto: arm/curve25519 - wire up NEON implementation
UPSTREAM: crypto: arm/curve25519 - import Bernstein and Schwabe's Curve25519 ARM implementation
UPSTREAM: crypto: curve25519 - x86_64 library and KPP implementations
UPSTREAM: crypto: lib/curve25519 - work around Clang stack spilling issue
UPSTREAM: crypto: curve25519 - implement generic KPP driver
UPSTREAM: crypto: curve25519 - add kpp selftest
UPSTREAM: crypto: curve25519 - generic C library implementations
UPSTREAM: crypto: mips/poly1305 - incorporate OpenSSL/CRYPTOGAMS optimized implementation
UPSTREAM: crypto: arm/poly1305 - incorporate OpenSSL/CRYPTOGAMS NEON implementation
UPSTREAM: crypto: arm64/poly1305 - incorporate OpenSSL/CRYPTOGAMS NEON implementation
UPSTREAM: crypto: x86/poly1305 - expose existing driver as poly1305 library
UPSTREAM: crypto: x86/poly1305 - depend on generic library not generic shash
UPSTREAM: crypto: poly1305 - expose init/update/final library interface
UPSTREAM: crypto: x86/poly1305 - unify Poly1305 state struct with generic code
UPSTREAM: crypto: poly1305 - move core routines into a separate library
UPSTREAM: crypto: chacha - unexport chacha_generic routines
UPSTREAM: crypto: mips/chacha - wire up accelerated 32r2 code from Zinc
UPSTREAM: crypto: mips/chacha - import 32r2 ChaCha code from Zinc
UPSTREAM: crypto: arm/chacha - expose ARM ChaCha routine as library function
UPSTREAM: crypto: arm/chacha - remove dependency on generic ChaCha driver
UPSTREAM: crypto: arm/chacha - import Eric Biggers's scalar accelerated ChaCha code
UPSTREAM: crypto: arm64/chacha - expose arm64 ChaCha routine as library function
UPSTREAM: crypto: arm64/chacha - depend on generic chacha library instead of crypto driver
UPSTREAM: crypto: x86/chacha - expose SIMD ChaCha routine as library function
UPSTREAM: crypto: x86/chacha - depend on generic chacha library instead of crypto driver
UPSTREAM: crypto: chacha - move existing library code into lib/crypto
Revert "BACKPORT: crypto: arch - conditionalize crypto api in arch glue for lib code"

-----

Change-Id: I15633145724b6409932a3a440ca8e55e186d12c1
Signed-off-by: Alexander Martinz <amartinz@shiftphones.com>

parents dae3d3ea 4e4e4524

MAINTAINERS

+8 −0

Original line number	Diff line number	Diff line
		@@ -17595,6 +17595,14 @@ L: linux-gpio@vger.kernel.org
		S: Maintained
		F: drivers/gpio/gpio-ws16c48.c

		WIREGUARD SECURE NETWORK TUNNEL
		M: Jason A. Donenfeld <Jason@zx2c4.com>
		S: Maintained
		F: drivers/net/wireguard/
		F: tools/testing/selftests/wireguard/
		L: wireguard@lists.zx2c4.com
		L: netdev@vger.kernel.org

		WISTRON LAPTOP BUTTON DRIVER
		M: Miloslav Trmac <mitr@volny.cz>
		S: Maintained

arch/arm/crypto/.gitignore

+1 −0

Original line number	Diff line number	Diff line
		aesbs-core.S
		sha256-core.S
		sha512-core.S
		poly1305-core.S

arch/arm/crypto/Kconfig

+13 −3

Original line number	Diff line number	Diff line
		@@ -148,14 +148,24 @@ config CRYPTO_CRC32_ARM_CE
		select CRYPTO_HASH

		config CRYPTO_CHACHA20_NEON
		tristate "NEON accelerated ChaCha stream cipher algorithms"
		depends on KERNEL_MODE_NEON
		tristate "NEON and scalar accelerated ChaCha stream cipher algorithms"
		select CRYPTO_BLKCIPHER
		select CRYPTO_CHACHA20
		select CRYPTO_ARCH_HAVE_LIB_CHACHA

		config CRYPTO_POLY1305_ARM
		tristate "Accelerated scalar and SIMD Poly1305 hash implementations"
		select CRYPTO_HASH
		select CRYPTO_ARCH_HAVE_LIB_POLY1305

		config CRYPTO_NHPOLY1305_NEON
		tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)"
		depends on KERNEL_MODE_NEON
		select CRYPTO_NHPOLY1305

		config CRYPTO_CURVE25519_NEON
		tristate "NEON accelerated Curve25519 scalar multiplication library"
		depends on KERNEL_MODE_NEON
		select CRYPTO_LIB_CURVE25519_GENERIC
		select CRYPTO_ARCH_HAVE_LIB_CURVE25519

		endif

arch/arm/crypto/Makefile

+15 −2

Original line number	Diff line number	Diff line
		@@ -13,7 +13,9 @@ obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += blake2s-arm.o
		obj-$(if $(CONFIG_CRYPTO_BLAKE2S_ARM),y) += libblake2s-arm.o
		obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o
		obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o
		obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o
		obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
		obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o

		obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
		obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
		@@ -39,13 +41,19 @@ aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
		ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
		crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
		crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
		chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
		chacha-neon-y := chacha-scalar-core.o chacha-glue.o
		chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
		poly1305-arm-y := poly1305-core.o poly1305-glue.o
		nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
		curve25519-neon-y := curve25519-core.o curve25519-glue.o

		ifdef REGENERATE_ARM_CRYPTO
		quiet_cmd_perl = PERL $@
		cmd_perl = $(PERL) $(<) > $(@)

		$(src)/poly1305-core.S_shipped: $(src)/poly1305-armv4.pl
		$(call cmd,perl)

		$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
		$(call cmd,perl)

		@@ -53,4 +61,9 @@ $(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl
		$(call cmd,perl)
		endif

		clean-files += sha256-core.S sha512-core.S
		clean-files += poly1305-core.S sha256-core.S sha512-core.S

		# massage the perlasm code a bit so we only get the NEON routine if we need it
		poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
		poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
		AFLAGS_poly1305-core.o += $(poly1305-aflags-y)

arch/arm/crypto/chacha-neon-glue.c→arch/arm/crypto/chacha-glue.c

+357 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0
		/*
		* ARM NEON accelerated ChaCha and XChaCha stream ciphers,
		* including ChaCha20 (RFC7539)
		*
		* Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*
		* Based on:
		* ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
		*
		* Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
		* Copyright (C) 2015 Martin Willi
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation; either version 2 of the License, or
		* (at your option) any later version.
		*/

		#include <crypto/algapi.h>
		#include <crypto/chacha.h>
		#include <crypto/internal/chacha.h>
		#include <crypto/internal/simd.h>
		#include <crypto/internal/skcipher.h>
		#include <linux/jump_label.h>
		#include <linux/kernel.h>
		#include <linux/module.h>

		#include <asm/cputype.h>
		#include <asm/hwcap.h>
		#include <asm/neon.h>
		#include <asm/simd.h>
		@@ -34,8 +24,19 @@ asmlinkage void chacha_block_xor_neon(const u32 state, u8 dst, const u8 *src,
		int nrounds);
		asmlinkage void chacha_4block_xor_neon(const u32 state, u8 dst, const u8 *src,
		int nrounds);
		asmlinkage void hchacha_block_arm(const u32 state, u32 out, int nrounds);
		asmlinkage void hchacha_block_neon(const u32 state, u32 out, int nrounds);

		asmlinkage void chacha_doarm(u8 dst, const u8 src, unsigned int bytes,
		const u32 *state, int nrounds);

		static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);

		static inline bool neon_usable(void)
		{
		return static_branch_likely(&use_neon) && crypto_simd_usable();
		}

		static void chacha_doneon(u32 state, u8 dst, const u8 *src,
		unsigned int bytes, int nrounds)
		{
		@@ -62,8 +63,51 @@ static void chacha_doneon(u32 state, u8 dst, const u8 *src,
		}
		}

		static int chacha_neon_stream_xor(struct skcipher_request *req,
		const struct chacha_ctx ctx, const u8 iv)
		void hchacha_block_arch(const u32 state, u32 stream, int nrounds)
		{
		if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) \|\| !neon_usable()) {
		hchacha_block_arm(state, stream, nrounds);
		} else {
		kernel_neon_begin();
		hchacha_block_neon(state, stream, nrounds);
		kernel_neon_end();
		}
		}
		EXPORT_SYMBOL(hchacha_block_arch);

		void chacha_init_arch(u32 state, const u32 key, const u8 *iv)
		{
		chacha_init_generic(state, key, iv);
		}
		EXPORT_SYMBOL(chacha_init_arch);

		void chacha_crypt_arch(u32 state, u8 dst, const u8 *src, unsigned int bytes,
		int nrounds)
		{
		if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) \|\| !neon_usable() \|\|
		bytes <= CHACHA_BLOCK_SIZE) {
		chacha_doarm(dst, src, bytes, state, nrounds);
		state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
		return;
		}

		do {
		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);

		kernel_neon_begin();
		chacha_doneon(state, dst, src, todo, nrounds);
		kernel_neon_end();

		bytes -= todo;
		src += todo;
		dst += todo;
		} while (bytes);
		}
		EXPORT_SYMBOL(chacha_crypt_arch);

		static int chacha_stream_xor(struct skcipher_request *req,
		const struct chacha_ctx ctx, const u8 iv,
		bool neon)
		{
		struct skcipher_walk walk;
		u32 state[16];
		@@ -71,7 +115,7 @@ static int chacha_neon_stream_xor(struct skcipher_request *req,

		err = skcipher_walk_virt(&walk, req, false);

		crypto_chacha_init(state, ctx, iv);
		chacha_init_generic(state, ctx->key, iv);

		while (walk.nbytes > 0) {
		unsigned int nbytes = walk.nbytes;
		@@ -79,28 +123,41 @@ static int chacha_neon_stream_xor(struct skcipher_request *req,
		if (nbytes < walk.total)
		nbytes = round_down(nbytes, walk.stride);

		if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) \|\| !neon) {
		chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
		nbytes, state, ctx->nrounds);
		state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
		} else {
		kernel_neon_begin();
		chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
		nbytes, ctx->nrounds);
		chacha_doneon(state, walk.dst.virt.addr,
		walk.src.virt.addr, nbytes, ctx->nrounds);
		kernel_neon_end();
		}
		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
		}

		return err;
		}

		static int chacha_neon(struct skcipher_request *req)
		static int do_chacha(struct skcipher_request *req, bool neon)
		{
		struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
		struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);

		if (req->cryptlen <= CHACHA_BLOCK_SIZE \|\| !crypto_simd_usable())
		return crypto_chacha_crypt(req);
		return chacha_stream_xor(req, ctx, req->iv, neon);
		}

		static int chacha_arm(struct skcipher_request *req)
		{
		return do_chacha(req, false);
		}

		return chacha_neon_stream_xor(req, ctx, req->iv);
		static int chacha_neon(struct skcipher_request *req)
		{
		return do_chacha(req, neon_usable());
		}

		static int xchacha_neon(struct skcipher_request *req)
		static int do_xchacha(struct skcipher_request *req, bool neon)
		{
		struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
		struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
		@@ -108,22 +165,82 @@ static int xchacha_neon(struct skcipher_request *req)
		u32 state[16];
		u8 real_iv[16];

		if (req->cryptlen <= CHACHA_BLOCK_SIZE \|\| !crypto_simd_usable())
		return crypto_xchacha_crypt(req);

		crypto_chacha_init(state, ctx, req->iv);
		chacha_init_generic(state, ctx->key, req->iv);

		if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) \|\| !neon) {
		hchacha_block_arm(state, subctx.key, ctx->nrounds);
		} else {
		kernel_neon_begin();
		hchacha_block_neon(state, subctx.key, ctx->nrounds);
		kernel_neon_end();
		}
		subctx.nrounds = ctx->nrounds;

		memcpy(&real_iv[0], req->iv + 24, 8);
		memcpy(&real_iv[8], req->iv + 16, 8);
		return chacha_neon_stream_xor(req, &subctx, real_iv);
		return chacha_stream_xor(req, &subctx, real_iv, neon);
		}

		static int xchacha_arm(struct skcipher_request *req)
		{
		return do_xchacha(req, false);
		}

		static struct skcipher_alg algs[] = {
		static int xchacha_neon(struct skcipher_request *req)
		{
		return do_xchacha(req, neon_usable());
		}

		static struct skcipher_alg arm_algs[] = {
		{
		.base.cra_name = "chacha20",
		.base.cra_driver_name = "chacha20-arm",
		.base.cra_priority = 200,
		.base.cra_blocksize = 1,
		.base.cra_ctxsize = sizeof(struct chacha_ctx),
		.base.cra_module = THIS_MODULE,

		.min_keysize = CHACHA_KEY_SIZE,
		.max_keysize = CHACHA_KEY_SIZE,
		.ivsize = CHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.setkey = chacha20_setkey,
		.encrypt = chacha_arm,
		.decrypt = chacha_arm,
		}, {
		.base.cra_name = "xchacha20",
		.base.cra_driver_name = "xchacha20-arm",
		.base.cra_priority = 200,
		.base.cra_blocksize = 1,
		.base.cra_ctxsize = sizeof(struct chacha_ctx),
		.base.cra_module = THIS_MODULE,

		.min_keysize = CHACHA_KEY_SIZE,
		.max_keysize = CHACHA_KEY_SIZE,
		.ivsize = XCHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.setkey = chacha20_setkey,
		.encrypt = xchacha_arm,
		.decrypt = xchacha_arm,
		}, {
		.base.cra_name = "xchacha12",
		.base.cra_driver_name = "xchacha12-arm",
		.base.cra_priority = 200,
		.base.cra_blocksize = 1,
		.base.cra_ctxsize = sizeof(struct chacha_ctx),
		.base.cra_module = THIS_MODULE,

		.min_keysize = CHACHA_KEY_SIZE,
		.max_keysize = CHACHA_KEY_SIZE,
		.ivsize = XCHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.setkey = chacha12_setkey,
		.encrypt = xchacha_arm,
		.decrypt = xchacha_arm,
		},
		};

		static struct skcipher_alg neon_algs[] = {
		{
		.base.cra_name = "chacha20",
		.base.cra_driver_name = "chacha20-neon",
		@@ -137,7 +254,7 @@ static struct skcipher_alg algs[] = {
		.ivsize = CHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.walksize = 4 * CHACHA_BLOCK_SIZE,
		.setkey = crypto_chacha20_setkey,
		.setkey = chacha20_setkey,
		.encrypt = chacha_neon,
		.decrypt = chacha_neon,
		}, {
		@@ -153,7 +270,7 @@ static struct skcipher_alg algs[] = {
		.ivsize = XCHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.walksize = 4 * CHACHA_BLOCK_SIZE,
		.setkey = crypto_chacha20_setkey,
		.setkey = chacha20_setkey,
		.encrypt = xchacha_neon,
		.decrypt = xchacha_neon,
		}, {
		@@ -169,7 +286,7 @@ static struct skcipher_alg algs[] = {
		.ivsize = XCHACHA_IV_SIZE,
		.chunksize = CHACHA_BLOCK_SIZE,
		.walksize = 4 * CHACHA_BLOCK_SIZE,
		.setkey = crypto_chacha12_setkey,
		.setkey = chacha12_setkey,
		.encrypt = xchacha_neon,
		.decrypt = xchacha_neon,
		}
		@@ -177,26 +294,64 @@ static struct skcipher_alg algs[] = {

		static int __init chacha_simd_mod_init(void)
		{
		if (!(elf_hwcap & HWCAP_NEON))
		return -ENODEV;
		int err = 0;

		return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
		if (IS_REACHABLE(CONFIG_CRYPTO_BLKCIPHER)) {
		err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		if (err)
		return err;
		}

		if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
		int i;

		switch (read_cpuid_part()) {
		case ARM_CPU_PART_CORTEX_A7:
		case ARM_CPU_PART_CORTEX_A5:
		/*
		* The Cortex-A7 and Cortex-A5 do not perform well with
		* the NEON implementation but do incredibly with the
		* scalar one and use less power.
		*/
		for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
		neon_algs[i].base.cra_priority = 0;
		break;
		default:
		static_branch_enable(&use_neon);
		}

		if (IS_REACHABLE(CONFIG_CRYPTO_BLKCIPHER)) {
		err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
		if (err)
		crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		}
		}
		return err;
		}

		static void __exit chacha_simd_mod_fini(void)
		{
		crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
		if (IS_REACHABLE(CONFIG_CRYPTO_BLKCIPHER)) {
		crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
		crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
		}
		}

		module_init(chacha_simd_mod_init);
		module_exit(chacha_simd_mod_fini);

		MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
		MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
		MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
		MODULE_LICENSE("GPL v2");
		MODULE_ALIAS_CRYPTO("chacha20");
		MODULE_ALIAS_CRYPTO("chacha20-neon");
		MODULE_ALIAS_CRYPTO("chacha20-arm");
		MODULE_ALIAS_CRYPTO("xchacha20");
		MODULE_ALIAS_CRYPTO("xchacha20-neon");
		MODULE_ALIAS_CRYPTO("xchacha20-arm");
		MODULE_ALIAS_CRYPTO("xchacha12");
		MODULE_ALIAS_CRYPTO("xchacha12-arm");
		#ifdef CONFIG_KERNEL_MODE_NEON
		MODULE_ALIAS_CRYPTO("chacha20-neon");
		MODULE_ALIAS_CRYPTO("xchacha20-neon");
		MODULE_ALIAS_CRYPTO("xchacha12-neon");
		#endif