diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj b/builds/msvc/vs2010/libsodium/libsodium.vcxproj
index 77c1dd6d..9d521ad6 100644
--- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj
@@ -162,11 +162,13 @@
+
+
@@ -345,23 +347,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj b/builds/msvc/vs2012/libsodium/libsodium.vcxproj
index 57368b0d..92c277d4 100644
--- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj
@@ -162,11 +162,13 @@
+
+
@@ -345,23 +347,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj b/builds/msvc/vs2013/libsodium/libsodium.vcxproj
index c85ed549..f8a4d2b1 100644
--- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj
@@ -162,11 +162,13 @@
+
+
@@ -345,23 +347,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj b/builds/msvc/vs2015/libsodium/libsodium.vcxproj
index 6a4efbe2..39c644ec 100644
--- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj
@@ -162,11 +162,13 @@
+
+
@@ -345,23 +347,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj b/builds/msvc/vs2017/libsodium/libsodium.vcxproj
index 6fecdf2a..22a1a528 100644
--- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj
@@ -162,11 +162,13 @@
+
+
@@ -345,23 +347,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj b/builds/msvc/vs2019/libsodium/libsodium.vcxproj
index 9c57d2dc..b8966235 100644
--- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj
@@ -186,11 +186,13 @@
+
+
@@ -369,23 +371,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj b/builds/msvc/vs2022/libsodium/libsodium.vcxproj
index b6954440..edc52c6c 100644
--- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj
@@ -186,11 +186,13 @@
+
+
@@ -369,23 +371,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj b/builds/msvc/vs2026/libsodium/libsodium.vcxproj
index 12f7acf7..ae7068b0 100644
--- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj
+++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj
@@ -186,11 +186,13 @@
+
+
@@ -369,23 +371,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters
index a0f80e26..053de321 100644
--- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters
+++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters
@@ -306,6 +306,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -321,6 +324,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -851,6 +857,9 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
@@ -869,9 +878,21 @@
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\chacha20\dolbeau
+
+ crypto_stream\chacha20\dolbeau
+
+
+ crypto_stream\chacha20\dolbeau
+
crypto_stream\salsa20
@@ -881,6 +902,9 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
@@ -896,12 +920,24 @@
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6int
crypto_stream\salsa20\xmm6int
+
+ crypto_stream\salsa20\xmm6int
+
+
+ crypto_stream\salsa20\xmm6int
+
crypto_stream\salsa20\xmm6
diff --git a/ci/appveyor/libsodium.vcxproj b/ci/appveyor/libsodium.vcxproj
index 1dbbae1d..7e9fb601 100644
--- a/ci/appveyor/libsodium.vcxproj
+++ b/ci/appveyor/libsodium.vcxproj
@@ -410,11 +410,13 @@
+
+
@@ -593,23 +595,33 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/ci/appveyor/libsodium.vcxproj.filters b/ci/appveyor/libsodium.vcxproj.filters
index 700b5c5f..84b80835 100644
--- a/ci/appveyor/libsodium.vcxproj.filters
+++ b/ci/appveyor/libsodium.vcxproj.filters
@@ -297,6 +297,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -312,6 +315,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -842,6 +848,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -860,9 +869,21 @@
Header Files
+
+ Header Files
+
+
+ Header Files
+
Header Files
+
+ Header Files
+
+
+ Header Files
+
Header Files
@@ -872,6 +893,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -887,12 +911,24 @@
Header Files
+
+ Header Files
+
+
+ Header Files
+
Header Files
Header Files
+
+ Header Files
+
+
+ Header Files
+
Header Files
diff --git a/src/libsodium/Makefile.am b/src/libsodium/Makefile.am
index 1e4b14f1..59720889 100644
--- a/src/libsodium/Makefile.am
+++ b/src/libsodium/Makefile.am
@@ -269,7 +269,19 @@ libarmcrypto_la_SOURCES = \
crypto_ipcrypt/ipcrypt_armcrypto.c \
crypto_ipcrypt/ipcrypt_armcrypto.h \
crypto_pwhash/argon2/argon2-fill-block-neon.c \
- crypto_pwhash/argon2/blamka-round-neon.h
+ crypto_pwhash/argon2/blamka-round-neon.h \
+ crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c \
+ crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h \
+ crypto_stream/chacha20/dolbeau/u0-neon.h \
+ crypto_stream/chacha20/dolbeau/u1-neon.h \
+ crypto_stream/chacha20/dolbeau/u2-neon.h \
+ crypto_stream/chacha20/dolbeau/u4-neon.h \
+ crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c \
+ crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h \
+ crypto_stream/salsa20/xmm6int/u0-neon.h \
+ crypto_stream/salsa20/xmm6int/u1-neon.h \
+ crypto_stream/salsa20/xmm6int/u2-neon.h \
+ crypto_stream/salsa20/xmm6int/u4-neon.h
libaesni_la_LDFLAGS = $(libsodium_la_LDFLAGS)
libaesni_la_CPPFLAGS = $(libsodium_la_CPPFLAGS) \
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c
new file mode 100644
index 00000000..9d5e6bfa
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c
@@ -0,0 +1,202 @@
+
+#include
+#include
+#include
+
+#include "core.h"
+#include "crypto_stream_chacha20.h"
+#include "private/common.h"
+#include "utils.h"
+
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+
+# include
+
+# include "../ref/chacha20_ref.h"
+# include "../stream_chacha20.h"
+# include "chacha20_dolbeau-neon.h"
+
+# define ROUNDS 20
+
+# define MIN_VECTOR_BYTES 128
+
+typedef struct chacha_ctx {
+ uint32_t input[16];
+} chacha_ctx;
+
+static inline uint32x4_t
+rotl32_16_neon(uint32x4_t x)
+{
+ return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x)));
+}
+
+static inline uint32x4_t
+rotl32_12_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 12), x, 20);
+}
+
+static inline uint32x4_t
+rotl32_8_neon(uint32x4_t x)
+{
+ static const uint8_t rot8_tbl[16] = {
+ 3, 0, 1, 2, 7, 4, 5, 6,
+ 11, 8, 9, 10, 15, 12, 13, 14
+ };
+ const uint8x16_t tbl = vld1q_u8(rot8_tbl);
+
+ return vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(x), tbl));
+}
+
+static inline uint32x4_t
+rotl32_7_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 7), x, 25);
+}
+
+static void
+chacha_keysetup(chacha_ctx *ctx, const uint8_t *k)
+{
+ ctx->input[0] = 0x61707865;
+ ctx->input[1] = 0x3320646e;
+ ctx->input[2] = 0x79622d32;
+ ctx->input[3] = 0x6b206574;
+ ctx->input[4] = LOAD32_LE(k + 0);
+ ctx->input[5] = LOAD32_LE(k + 4);
+ ctx->input[6] = LOAD32_LE(k + 8);
+ ctx->input[7] = LOAD32_LE(k + 12);
+ ctx->input[8] = LOAD32_LE(k + 16);
+ ctx->input[9] = LOAD32_LE(k + 20);
+ ctx->input[10] = LOAD32_LE(k + 24);
+ ctx->input[11] = LOAD32_LE(k + 28);
+}
+
+static void
+chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter)
+{
+ ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter + 0);
+ ctx->input[13] = counter == NULL ? 0 : LOAD32_LE(counter + 4);
+ ctx->input[14] = LOAD32_LE(iv + 0);
+ ctx->input[15] = LOAD32_LE(iv + 4);
+}
+
+static void
+chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter)
+{
+ ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter);
+ ctx->input[13] = LOAD32_LE(iv + 0);
+ ctx->input[14] = LOAD32_LE(iv + 4);
+ ctx->input[15] = LOAD32_LE(iv + 8);
+}
+
+static void
+chacha20_encrypt_bytes(chacha_ctx *ctx, const uint8_t *m, uint8_t *c,
+ unsigned long long bytes)
+{
+ uint32_t * const x = &ctx->input[0];
+
+ if (!bytes) {
+ return; /* LCOV_EXCL_LINE */
+ }
+# include "u4-neon.h"
+# include "u2-neon.h"
+# include "u1-neon.h"
+# include "u0-neon.h"
+}
+
+static int
+stream_neon(unsigned char *c, unsigned long long clen, const unsigned char *n,
+ const unsigned char *k)
+{
+ struct chacha_ctx ctx;
+
+ if (clen < MIN_VECTOR_BYTES) {
+ return crypto_stream_chacha20_ref_implementation.stream(c, clen, n, k);
+ }
+ COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8);
+ chacha_keysetup(&ctx, k);
+ chacha_ivsetup(&ctx, n, NULL);
+ memset(c, 0, clen);
+ chacha20_encrypt_bytes(&ctx, c, c, clen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+static int
+stream_ietf_ext_neon(unsigned char *c, unsigned long long clen,
+ const unsigned char *n, const unsigned char *k)
+{
+ struct chacha_ctx ctx;
+
+ if (clen < MIN_VECTOR_BYTES) {
+ return crypto_stream_chacha20_ref_implementation.stream_ietf_ext(
+ c, clen, n, k);
+ }
+ COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8);
+ chacha_keysetup(&ctx, k);
+ chacha_ietf_ivsetup(&ctx, n, NULL);
+ memset(c, 0, clen);
+ chacha20_encrypt_bytes(&ctx, c, c, clen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+static int
+stream_neon_xor_ic(unsigned char *c, const unsigned char *m,
+ unsigned long long mlen, const unsigned char *n,
+ uint64_t ic, const unsigned char *k)
+{
+ struct chacha_ctx ctx;
+ uint8_t ic_bytes[8];
+ uint32_t ic_high;
+ uint32_t ic_low;
+
+ if (mlen < MIN_VECTOR_BYTES) {
+ return crypto_stream_chacha20_ref_implementation.stream_xor_ic(
+ c, m, mlen, n, ic, k);
+ }
+ ic_high = (uint32_t) (ic >> 32);
+ ic_low = (uint32_t) ic;
+ STORE32_LE(&ic_bytes[0], ic_low);
+ STORE32_LE(&ic_bytes[4], ic_high);
+ chacha_keysetup(&ctx, k);
+ chacha_ivsetup(&ctx, n, ic_bytes);
+ chacha20_encrypt_bytes(&ctx, m, c, mlen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+static int
+stream_ietf_ext_neon_xor_ic(unsigned char *c, const unsigned char *m,
+ unsigned long long mlen, const unsigned char *n,
+ uint32_t ic, const unsigned char *k)
+{
+ struct chacha_ctx ctx;
+ uint8_t ic_bytes[4];
+
+ if (mlen < MIN_VECTOR_BYTES) {
+ return crypto_stream_chacha20_ref_implementation.stream_ietf_ext_xor_ic(
+ c, m, mlen, n, ic, k);
+ }
+ STORE32_LE(ic_bytes, ic);
+ chacha_keysetup(&ctx, k);
+ chacha_ietf_ivsetup(&ctx, n, ic_bytes);
+ chacha20_encrypt_bytes(&ctx, m, c, mlen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+struct crypto_stream_chacha20_implementation
+ crypto_stream_chacha20_dolbeau_neon_implementation = {
+ SODIUM_C99(.stream =) stream_neon,
+ SODIUM_C99(.stream_ietf_ext =) stream_ietf_ext_neon,
+ SODIUM_C99(.stream_xor_ic =) stream_neon_xor_ic,
+ SODIUM_C99(.stream_ietf_ext_xor_ic =) stream_ietf_ext_neon_xor_ic
+ };
+
+#endif
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h
new file mode 100644
index 00000000..18f2ebf4
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h
@@ -0,0 +1,8 @@
+
+#include
+
+#include "../stream_chacha20.h"
+#include "crypto_stream_chacha20.h"
+
+extern struct crypto_stream_chacha20_implementation
+ crypto_stream_chacha20_dolbeau_neon_implementation;
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h
new file mode 100644
index 00000000..ae031533
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h
@@ -0,0 +1,66 @@
+if (bytes > 0) {
+ uint32x4_t x_0, x_1, x_2, x_3;
+ uint8_t partialblock[64];
+
+ unsigned int i;
+
+ x_0 = vld1q_u32(x + 0);
+ x_1 = vld1q_u32(x + 4);
+ x_2 = vld1q_u32(x + 8);
+ x_3 = vld1q_u32(x + 12);
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_3 = rotl32_16_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_1 = veorq_u32(x_1, x_2);
+ x_1 = rotl32_12_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_0 = vextq_u32(x_0, x_0, 3);
+ x_3 = rotl32_8_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_3 = vextq_u32(x_3, x_3, 2);
+ x_1 = veorq_u32(x_1, x_2);
+ x_2 = vextq_u32(x_2, x_2, 1);
+ x_1 = rotl32_7_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_3 = rotl32_16_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_1 = veorq_u32(x_1, x_2);
+ x_1 = rotl32_12_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_0 = vextq_u32(x_0, x_0, 1);
+ x_3 = rotl32_8_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_3 = vextq_u32(x_3, x_3, 2);
+ x_1 = veorq_u32(x_1, x_2);
+ x_2 = vextq_u32(x_2, x_2, 3);
+ x_1 = rotl32_7_neon(x_1);
+ }
+ x_0 = vaddq_u32(x_0, vld1q_u32(x + 0));
+ x_1 = vaddq_u32(x_1, vld1q_u32(x + 4));
+ x_2 = vaddq_u32(x_2, vld1q_u32(x + 8));
+ x_3 = vaddq_u32(x_3, vld1q_u32(x + 12));
+
+ vst1q_u8(partialblock + 0, vreinterpretq_u8_u32(x_0));
+ vst1q_u8(partialblock + 16, vreinterpretq_u8_u32(x_1));
+ vst1q_u8(partialblock + 32, vreinterpretq_u8_u32(x_2));
+ vst1q_u8(partialblock + 48, vreinterpretq_u8_u32(x_3));
+
+ for (i = 0; i < bytes; i++) {
+ c[i] = m[i] ^ partialblock[i];
+ }
+
+ sodium_memzero(partialblock, sizeof partialblock);
+}
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h
new file mode 100644
index 00000000..275668bc
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h
@@ -0,0 +1,79 @@
+while (bytes >= 64) {
+ uint32x4_t x_0, x_1, x_2, x_3;
+
+ uint32_t in12;
+ uint32_t in13;
+ int i;
+
+ x_0 = vld1q_u32(x + 0);
+ x_1 = vld1q_u32(x + 4);
+ x_2 = vld1q_u32(x + 8);
+ x_3 = vld1q_u32(x + 12);
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_3 = rotl32_16_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_1 = veorq_u32(x_1, x_2);
+ x_1 = rotl32_12_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_0 = vextq_u32(x_0, x_0, 3);
+ x_3 = rotl32_8_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_3 = vextq_u32(x_3, x_3, 2);
+ x_1 = veorq_u32(x_1, x_2);
+ x_2 = vextq_u32(x_2, x_2, 1);
+ x_1 = rotl32_7_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_3 = rotl32_16_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_1 = veorq_u32(x_1, x_2);
+ x_1 = rotl32_12_neon(x_1);
+
+ x_0 = vaddq_u32(x_0, x_1);
+ x_3 = veorq_u32(x_3, x_0);
+ x_0 = vextq_u32(x_0, x_0, 1);
+ x_3 = rotl32_8_neon(x_3);
+
+ x_2 = vaddq_u32(x_2, x_3);
+ x_3 = vextq_u32(x_3, x_3, 2);
+ x_1 = veorq_u32(x_1, x_2);
+ x_2 = vextq_u32(x_2, x_2, 3);
+ x_1 = rotl32_7_neon(x_1);
+ }
+ x_0 = vaddq_u32(x_0, vld1q_u32(x + 0));
+ x_1 = vaddq_u32(x_1, vld1q_u32(x + 4));
+ x_2 = vaddq_u32(x_2, vld1q_u32(x + 8));
+ x_3 = vaddq_u32(x_3, vld1q_u32(x + 12));
+
+ x_0 = veorq_u32(x_0, vreinterpretq_u32_u8(vld1q_u8(m + 0)));
+ x_1 = veorq_u32(x_1, vreinterpretq_u32_u8(vld1q_u8(m + 16)));
+ x_2 = veorq_u32(x_2, vreinterpretq_u32_u8(vld1q_u8(m + 32)));
+ x_3 = veorq_u32(x_3, vreinterpretq_u32_u8(vld1q_u8(m + 48)));
+
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(x_0));
+ vst1q_u8(c + 16, vreinterpretq_u8_u32(x_1));
+ vst1q_u8(c + 32, vreinterpretq_u8_u32(x_2));
+ vst1q_u8(c + 48, vreinterpretq_u8_u32(x_3));
+
+ in12 = x[12];
+ in13 = x[13];
+ in12++;
+ if (in12 == 0) {
+ in13++;
+ }
+ x[12] = in12;
+ x[13] = in13;
+
+ bytes -= 64;
+ c += 64;
+ m += 64;
+}
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h
new file mode 100644
index 00000000..2986396c
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h
@@ -0,0 +1,86 @@
+
+#define VEC2_HALFROUND(P, E0, E2) \
+ P##_0 = vaddq_u32(P##_0, P##_1); \
+ P##_3 = veorq_u32(P##_3, P##_0); \
+ P##_3 = rotl32_16_neon(P##_3); \
+ P##_2 = vaddq_u32(P##_2, P##_3); \
+ P##_1 = veorq_u32(P##_1, P##_2); \
+ P##_1 = rotl32_12_neon(P##_1); \
+ P##_0 = vaddq_u32(P##_0, P##_1); \
+ P##_3 = veorq_u32(P##_3, P##_0); \
+ P##_0 = vextq_u32(P##_0, P##_0, E0); \
+ P##_3 = rotl32_8_neon(P##_3); \
+ P##_2 = vaddq_u32(P##_2, P##_3); \
+ P##_3 = vextq_u32(P##_3, P##_3, 2); \
+ P##_1 = veorq_u32(P##_1, P##_2); \
+ P##_2 = vextq_u32(P##_2, P##_2, E2); \
+ P##_1 = rotl32_7_neon(P##_1)
+
+while (bytes >= 128) {
+ static const uint64_t addv_tbl[2] = { 1, 0 };
+
+ uint32x4_t orig0 = vld1q_u32(x + 0);
+ uint32x4_t orig1 = vld1q_u32(x + 4);
+ uint32x4_t orig2 = vld1q_u32(x + 8);
+ uint32x4_t orig3 = vld1q_u32(x + 12);
+ uint32x4_t orig3b = vreinterpretq_u32_u64(vaddq_u64(
+ vreinterpretq_u64_u32(orig3), vld1q_u64(addv_tbl)));
+ uint32x4_t x_0, x_1, x_2, x_3;
+ uint32x4_t y_0, y_1, y_2, y_3;
+
+ uint64_t in1213;
+ int i;
+
+ x_0 = orig0;
+ x_1 = orig1;
+ x_2 = orig2;
+ x_3 = orig3;
+ y_0 = orig0;
+ y_1 = orig1;
+ y_2 = orig2;
+ y_3 = orig3b;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ VEC2_HALFROUND(x, 3, 1);
+ VEC2_HALFROUND(y, 3, 1);
+ VEC2_HALFROUND(x, 1, 3);
+ VEC2_HALFROUND(y, 1, 3);
+ }
+ x_0 = vaddq_u32(x_0, orig0);
+ x_1 = vaddq_u32(x_1, orig1);
+ x_2 = vaddq_u32(x_2, orig2);
+ x_3 = vaddq_u32(x_3, orig3);
+ y_0 = vaddq_u32(y_0, orig0);
+ y_1 = vaddq_u32(y_1, orig1);
+ y_2 = vaddq_u32(y_2, orig2);
+ y_3 = vaddq_u32(y_3, orig3b);
+
+ x_0 = veorq_u32(x_0, vreinterpretq_u32_u8(vld1q_u8(m + 0)));
+ x_1 = veorq_u32(x_1, vreinterpretq_u32_u8(vld1q_u8(m + 16)));
+ x_2 = veorq_u32(x_2, vreinterpretq_u32_u8(vld1q_u8(m + 32)));
+ x_3 = veorq_u32(x_3, vreinterpretq_u32_u8(vld1q_u8(m + 48)));
+ y_0 = veorq_u32(y_0, vreinterpretq_u32_u8(vld1q_u8(m + 64)));
+ y_1 = veorq_u32(y_1, vreinterpretq_u32_u8(vld1q_u8(m + 80)));
+ y_2 = veorq_u32(y_2, vreinterpretq_u32_u8(vld1q_u8(m + 96)));
+ y_3 = veorq_u32(y_3, vreinterpretq_u32_u8(vld1q_u8(m + 112)));
+
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(x_0));
+ vst1q_u8(c + 16, vreinterpretq_u8_u32(x_1));
+ vst1q_u8(c + 32, vreinterpretq_u8_u32(x_2));
+ vst1q_u8(c + 48, vreinterpretq_u8_u32(x_3));
+ vst1q_u8(c + 64, vreinterpretq_u8_u32(y_0));
+ vst1q_u8(c + 80, vreinterpretq_u8_u32(y_1));
+ vst1q_u8(c + 96, vreinterpretq_u8_u32(y_2));
+ vst1q_u8(c + 112, vreinterpretq_u8_u32(y_3));
+
+ in1213 = ((uint64_t) x[12]) | (((uint64_t) x[13]) << 32);
+ in1213 += 2;
+
+ x[12] = (uint32_t) (in1213 & 0xFFFFFFFF);
+ x[13] = (uint32_t) (in1213 >> 32);
+
+ bytes -= 128;
+ c += 128;
+ m += 128;
+}
+#undef VEC2_HALFROUND
diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h
new file mode 100644
index 00000000..7a5a6936
--- /dev/null
+++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h
@@ -0,0 +1,164 @@
+
+#define VEC4_QUARTERROUND(A, B, C, D) \
+ x_##A = vaddq_u32(x_##A, x_##B); \
+ t_##A = veorq_u32(x_##D, x_##A); \
+ x_##D = rotl32_16_neon(t_##A); \
+ x_##C = vaddq_u32(x_##C, x_##D); \
+ t_##C = veorq_u32(x_##B, x_##C); \
+ x_##B = rotl32_12_neon(t_##C); \
+ x_##A = vaddq_u32(x_##A, x_##B); \
+ t_##A = veorq_u32(x_##D, x_##A); \
+ x_##D = rotl32_8_neon(t_##A); \
+ x_##C = vaddq_u32(x_##C, x_##D); \
+ t_##C = veorq_u32(x_##B, x_##C); \
+ x_##B = rotl32_7_neon(t_##C)
+
+if (bytes >= 256) {
+ uint32x4_t x_0 = vdupq_n_u32(x[0]);
+ uint32x4_t x_1 = vdupq_n_u32(x[1]);
+ uint32x4_t x_2 = vdupq_n_u32(x[2]);
+ uint32x4_t x_3 = vdupq_n_u32(x[3]);
+ uint32x4_t x_4 = vdupq_n_u32(x[4]);
+ uint32x4_t x_5 = vdupq_n_u32(x[5]);
+ uint32x4_t x_6 = vdupq_n_u32(x[6]);
+ uint32x4_t x_7 = vdupq_n_u32(x[7]);
+ uint32x4_t x_8 = vdupq_n_u32(x[8]);
+ uint32x4_t x_9 = vdupq_n_u32(x[9]);
+ uint32x4_t x_10 = vdupq_n_u32(x[10]);
+ uint32x4_t x_11 = vdupq_n_u32(x[11]);
+ uint32x4_t x_12;
+ uint32x4_t x_13;
+ uint32x4_t x_14 = vdupq_n_u32(x[14]);
+ uint32x4_t x_15 = vdupq_n_u32(x[15]);
+ uint32x4_t orig0 = x_0;
+ uint32x4_t orig1 = x_1;
+ uint32x4_t orig2 = x_2;
+ uint32x4_t orig3 = x_3;
+ uint32x4_t orig4 = x_4;
+ uint32x4_t orig5 = x_5;
+ uint32x4_t orig6 = x_6;
+ uint32x4_t orig7 = x_7;
+ uint32x4_t orig8 = x_8;
+ uint32x4_t orig9 = x_9;
+ uint32x4_t orig10 = x_10;
+ uint32x4_t orig11 = x_11;
+ uint32x4_t orig12;
+ uint32x4_t orig13;
+ uint32x4_t orig14 = x_14;
+ uint32x4_t orig15 = x_15;
+ uint32x4_t t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11,
+ t_12, t_13, t_14, t_15;
+
+ uint32_t in12, in13;
+ int i;
+
+ while (bytes >= 256) {
+ static const uint64_t addv12_tbl[2] = { 0, 1 };
+ static const uint64_t addv13_tbl[2] = { 2, 3 };
+ uint64x2_t t12, t13;
+ uint64_t in1213;
+
+ x_0 = orig0;
+ x_1 = orig1;
+ x_2 = orig2;
+ x_3 = orig3;
+ x_4 = orig4;
+ x_5 = orig5;
+ x_6 = orig6;
+ x_7 = orig7;
+ x_8 = orig8;
+ x_9 = orig9;
+ x_10 = orig10;
+ x_11 = orig11;
+ x_14 = orig14;
+ x_15 = orig15;
+
+ in12 = x[12];
+ in13 = x[13];
+ in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32);
+
+ t12 = vaddq_u64(vld1q_u64(addv12_tbl), vdupq_n_u64(in1213));
+ t13 = vaddq_u64(vld1q_u64(addv13_tbl), vdupq_n_u64(in1213));
+
+ x_12 = vuzp1q_u32(vreinterpretq_u32_u64(t12),
+ vreinterpretq_u32_u64(t13));
+ x_13 = vuzp2q_u32(vreinterpretq_u32_u64(t12),
+ vreinterpretq_u32_u64(t13));
+
+ orig12 = x_12;
+ orig13 = x_13;
+
+ in1213 += 4;
+
+ x[12] = in1213 & 0xFFFFFFFF;
+ x[13] = (in1213 >> 32) & 0xFFFFFFFF;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ VEC4_QUARTERROUND(0, 4, 8, 12);
+ VEC4_QUARTERROUND(1, 5, 9, 13);
+ VEC4_QUARTERROUND(2, 6, 10, 14);
+ VEC4_QUARTERROUND(3, 7, 11, 15);
+ VEC4_QUARTERROUND(0, 5, 10, 15);
+ VEC4_QUARTERROUND(1, 6, 11, 12);
+ VEC4_QUARTERROUND(2, 7, 8, 13);
+ VEC4_QUARTERROUND(3, 4, 9, 14);
+ }
+
+#define ONEQUAD_TRANSPOSE(A, B, C, D) \
+ { \
+ uint32x4_t t0, t1, t2, t3; \
+ \
+ x_##A = vaddq_u32(x_##A, orig##A); \
+ x_##B = vaddq_u32(x_##B, orig##B); \
+ x_##C = vaddq_u32(x_##C, orig##C); \
+ x_##D = vaddq_u32(x_##D, orig##D); \
+ t_##A = vtrn1q_u32(x_##A, x_##B); \
+ t_##B = vtrn2q_u32(x_##A, x_##B); \
+ t_##C = vtrn1q_u32(x_##C, x_##D); \
+ t_##D = vtrn2q_u32(x_##C, x_##D); \
+ x_##A = vreinterpretq_u32_u64( \
+ vtrn1q_u64(vreinterpretq_u64_u32(t_##A), \
+ vreinterpretq_u64_u32(t_##C))); \
+ x_##B = vreinterpretq_u32_u64( \
+ vtrn1q_u64(vreinterpretq_u64_u32(t_##B), \
+ vreinterpretq_u64_u32(t_##D))); \
+ x_##C = vreinterpretq_u32_u64( \
+ vtrn2q_u64(vreinterpretq_u64_u32(t_##A), \
+ vreinterpretq_u64_u32(t_##C))); \
+ x_##D = vreinterpretq_u32_u64( \
+ vtrn2q_u64(vreinterpretq_u64_u32(t_##B), \
+ vreinterpretq_u64_u32(t_##D))); \
+ t0 = veorq_u32(x_##A, vreinterpretq_u32_u8(vld1q_u8(m + 0))); \
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(t0)); \
+ t1 = veorq_u32(x_##B, vreinterpretq_u32_u8(vld1q_u8(m + 64))); \
+ vst1q_u8(c + 64, vreinterpretq_u8_u32(t1)); \
+ t2 = veorq_u32(x_##C, vreinterpretq_u32_u8(vld1q_u8(m + 128))); \
+ vst1q_u8(c + 128, vreinterpretq_u8_u32(t2)); \
+ t3 = veorq_u32(x_##D, vreinterpretq_u32_u8(vld1q_u8(m + 192))); \
+ vst1q_u8(c + 192, vreinterpretq_u8_u32(t3)); \
+ }
+
+#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
+
+ ONEQUAD(0, 1, 2, 3);
+ m += 16;
+ c += 16;
+ ONEQUAD(4, 5, 6, 7);
+ m += 16;
+ c += 16;
+ ONEQUAD(8, 9, 10, 11);
+ m += 16;
+ c += 16;
+ ONEQUAD(12, 13, 14, 15);
+ m -= 48;
+ c -= 48;
+
+#undef ONEQUAD
+#undef ONEQUAD_TRANSPOSE
+
+ bytes -= 256;
+ c += 256;
+ m += 256;
+ }
+}
+#undef VEC4_QUARTERROUND
diff --git a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c
index 8bd46d2b..aa2232c0 100644
--- a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c
+++ b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c
@@ -20,6 +20,10 @@
#if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H)
# include "dolbeau/chacha20_dolbeau-ssse3.h"
#endif
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+# include "dolbeau/chacha20_dolbeau-neon.h"
+#endif
static const crypto_stream_chacha20_implementation *implementation =
&crypto_stream_chacha20_ref_implementation;
@@ -192,6 +196,13 @@ _crypto_stream_chacha20_pick_best_implementation(void)
implementation = &crypto_stream_chacha20_dolbeau_ssse3_implementation;
return 0;
}
+#endif
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+ if (sodium_runtime_has_neon()) {
+ implementation = &crypto_stream_chacha20_dolbeau_neon_implementation;
+ return 0;
+ }
#endif
return 0;
}
diff --git a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c
index 2979a55d..25e5c780 100644
--- a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c
+++ b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c
@@ -22,6 +22,10 @@
defined(HAVE_SMMINTRIN_H)
# include "xmm6int/salsa20_xmm6int-avx512.h"
#endif
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+# include "xmm6int/salsa20_xmm6int-neon.h"
+#endif
#if HAVE_AMD64_ASM
static const crypto_stream_salsa20_implementation *implementation =
@@ -108,6 +112,13 @@ _crypto_stream_salsa20_pick_best_implementation(void)
implementation = &crypto_stream_salsa20_xmm6int_sse2_implementation;
return 0;
}
+#endif
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+ if (sodium_runtime_has_neon()) {
+ implementation = &crypto_stream_salsa20_xmm6int_neon_implementation;
+ return 0;
+ }
#endif
return 0; /* LCOV_EXCL_LINE */
}
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c
new file mode 100644
index 00000000..6abbf140
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c
@@ -0,0 +1,181 @@
+
+#include
+#include
+#include
+
+#include "crypto_stream_salsa20.h"
+#include "private/common.h"
+#include "utils.h"
+
+#if (defined(__aarch64__) || defined(_M_ARM64)) && \
+ (defined(__ARM_NEON) || defined(__ARM_NEON__))
+
+# include
+
+# include "../ref/salsa20_ref.h"
+# include "../stream_salsa20.h"
+# include "salsa20_xmm6int-neon.h"
+
+# define ROUNDS 20
+
+# define MIN_VECTOR_BYTES 256
+
+typedef struct salsa_ctx {
+ uint32_t input[16];
+} salsa_ctx;
+
+static const int TR[16] = {
+ 0, 5, 10, 15, 12, 1, 6, 11, 8, 13, 2, 7, 4, 9, 14, 3
+};
+
+static inline uint32x4_t
+rotl32_7_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 7), x, 25);
+}
+
+static inline uint32x4_t
+rotl32_9_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 9), x, 23);
+}
+
+static inline uint32x4_t
+rotl32_13_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 13), x, 19);
+}
+
+static inline uint32x4_t
+rotl32_18_neon(uint32x4_t x)
+{
+ return vsriq_n_u32(vshlq_n_u32(x, 18), x, 14);
+}
+
+static inline uint32x4x4_t
+undiagonalize_neon(uint32x4_t diag0, uint32x4_t diag1, uint32x4_t diag2,
+ uint32x4_t diag3)
+{
+ uint32x4_t t0, t1, t2, t3;
+ uint32x4x4_t rows;
+
+ diag1 = vextq_u32(diag1, diag1, 1);
+ diag2 = vextq_u32(diag2, diag2, 2);
+ diag3 = vextq_u32(diag3, diag3, 3);
+
+ t0 = vtrn1q_u32(diag0, diag1);
+ t1 = vtrn2q_u32(diag0, diag1);
+ t2 = vtrn1q_u32(diag2, diag3);
+ t3 = vtrn2q_u32(diag2, diag3);
+
+ diag0 = vreinterpretq_u32_u64(
+ vtrn1q_u64(vreinterpretq_u64_u32(t0), vreinterpretq_u64_u32(t2)));
+ diag1 = vreinterpretq_u32_u64(
+ vtrn1q_u64(vreinterpretq_u64_u32(t1), vreinterpretq_u64_u32(t3)));
+ diag2 = vreinterpretq_u32_u64(
+ vtrn2q_u64(vreinterpretq_u64_u32(t0), vreinterpretq_u64_u32(t2)));
+ diag3 = vreinterpretq_u32_u64(
+ vtrn2q_u64(vreinterpretq_u64_u32(t1), vreinterpretq_u64_u32(t3)));
+
+ rows.val[0] = diag0;
+ rows.val[1] = vextq_u32(diag1, diag1, 3);
+ rows.val[2] = vextq_u32(diag2, diag2, 2);
+ rows.val[3] = vextq_u32(diag3, diag3, 1);
+
+ return rows;
+}
+
+static void
+salsa_keysetup(salsa_ctx *ctx, const uint8_t *k)
+{
+ ctx->input[TR[1]] = LOAD32_LE(k + 0);
+ ctx->input[TR[2]] = LOAD32_LE(k + 4);
+ ctx->input[TR[3]] = LOAD32_LE(k + 8);
+ ctx->input[TR[4]] = LOAD32_LE(k + 12);
+ ctx->input[TR[11]] = LOAD32_LE(k + 16);
+ ctx->input[TR[12]] = LOAD32_LE(k + 20);
+ ctx->input[TR[13]] = LOAD32_LE(k + 24);
+ ctx->input[TR[14]] = LOAD32_LE(k + 28);
+ ctx->input[TR[0]] = 0x61707865;
+ ctx->input[TR[5]] = 0x3320646e;
+ ctx->input[TR[10]] = 0x79622d32;
+ ctx->input[TR[15]] = 0x6b206574;
+}
+
+static void
+salsa_ivsetup(salsa_ctx *ctx, const uint8_t *iv, const uint8_t *counter)
+{
+ ctx->input[TR[6]] = LOAD32_LE(iv + 0);
+ ctx->input[TR[7]] = LOAD32_LE(iv + 4);
+ ctx->input[TR[8]] = counter == NULL ? 0 : LOAD32_LE(counter + 0);
+ ctx->input[TR[9]] = counter == NULL ? 0 : LOAD32_LE(counter + 4);
+}
+
+static void
+salsa20_encrypt_bytes(salsa_ctx *ctx, const uint8_t *m, uint8_t *c,
+ unsigned long long bytes)
+{
+ uint32_t * const x = &ctx->input[0];
+
+ if (!bytes) {
+ return; /* LCOV_EXCL_LINE */
+ }
+
+#include "u4-neon.h"
+#include "u2-neon.h"
+#include "u1-neon.h"
+#include "u0-neon.h"
+}
+
+static int
+stream_neon(unsigned char *c, unsigned long long clen, const unsigned char *n,
+ const unsigned char *k)
+{
+ struct salsa_ctx ctx;
+
+ if (clen < MIN_VECTOR_BYTES) {
+ return crypto_stream_salsa20_ref_implementation.stream(c, clen, n, k);
+ }
+ COMPILER_ASSERT(crypto_stream_salsa20_KEYBYTES == 256 / 8);
+ salsa_keysetup(&ctx, k);
+ salsa_ivsetup(&ctx, n, NULL);
+ memset(c, 0, clen);
+ salsa20_encrypt_bytes(&ctx, c, c, clen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+static int
+stream_neon_xor_ic(unsigned char *c, const unsigned char *m,
+ unsigned long long mlen, const unsigned char *n, uint64_t ic,
+ const unsigned char *k)
+{
+ struct salsa_ctx ctx;
+ uint8_t ic_bytes[8];
+ uint32_t ic_high;
+ uint32_t ic_low;
+
+ if (mlen < MIN_VECTOR_BYTES) {
+ return crypto_stream_salsa20_ref_implementation.stream_xor_ic(
+ c, m, mlen, n, ic, k);
+ }
+ ic_high = (uint32_t) (ic >> 32);
+ ic_low = (uint32_t) (ic);
+ STORE32_LE(&ic_bytes[0], ic_low);
+ STORE32_LE(&ic_bytes[4], ic_high);
+ salsa_keysetup(&ctx, k);
+ salsa_ivsetup(&ctx, n, ic_bytes);
+ salsa20_encrypt_bytes(&ctx, m, c, mlen);
+ sodium_memzero(&ctx, sizeof ctx);
+
+ return 0;
+}
+
+struct crypto_stream_salsa20_implementation
+ crypto_stream_salsa20_xmm6int_neon_implementation = {
+ SODIUM_C99(.stream =) stream_neon,
+ SODIUM_C99(.stream_xor_ic =) stream_neon_xor_ic
+ };
+
+#endif
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h
new file mode 100644
index 00000000..0dbfc569
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h
@@ -0,0 +1,8 @@
+
+#include
+
+#include "../stream_salsa20.h"
+#include "crypto_stream_salsa20.h"
+
+extern struct crypto_stream_salsa20_implementation
+ crypto_stream_salsa20_xmm6int_neon_implementation;
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h
new file mode 100644
index 00000000..519b4077
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h
@@ -0,0 +1,57 @@
+if (bytes > 0) {
+ uint32x4_t diag0 = vld1q_u32(x + 0);
+ uint32x4_t diag1 = vld1q_u32(x + 4);
+ uint32x4_t diag2 = vld1q_u32(x + 8);
+ uint32x4_t diag3 = vld1q_u32(x + 12);
+ uint32x4_t a;
+ uint32x4x4_t rows;
+ uint8_t partialblock[64];
+
+ unsigned int i;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ a = vaddq_u32(diag0, diag1);
+ diag3 = veorq_u32(diag3, rotl32_7_neon(a));
+ a = vaddq_u32(diag3, diag0);
+ diag2 = veorq_u32(diag2, rotl32_9_neon(a));
+ a = vaddq_u32(diag2, diag3);
+ diag1 = veorq_u32(diag1, rotl32_13_neon(a));
+ a = vaddq_u32(diag1, diag2);
+ diag0 = veorq_u32(diag0, rotl32_18_neon(a));
+
+ diag3 = vextq_u32(diag3, diag3, 3);
+ diag2 = vextq_u32(diag2, diag2, 2);
+ diag1 = vextq_u32(diag1, diag1, 1);
+
+ a = vaddq_u32(diag0, diag3);
+ diag1 = veorq_u32(diag1, rotl32_7_neon(a));
+ a = vaddq_u32(diag1, diag0);
+ diag2 = veorq_u32(diag2, rotl32_9_neon(a));
+ a = vaddq_u32(diag2, diag1);
+ diag3 = veorq_u32(diag3, rotl32_13_neon(a));
+ a = vaddq_u32(diag3, diag2);
+ diag0 = veorq_u32(diag0, rotl32_18_neon(a));
+
+ diag1 = vextq_u32(diag1, diag1, 3);
+ diag2 = vextq_u32(diag2, diag2, 2);
+ diag3 = vextq_u32(diag3, diag3, 1);
+ }
+
+ diag0 = vaddq_u32(diag0, vld1q_u32(x + 0));
+ diag1 = vaddq_u32(diag1, vld1q_u32(x + 4));
+ diag2 = vaddq_u32(diag2, vld1q_u32(x + 8));
+ diag3 = vaddq_u32(diag3, vld1q_u32(x + 12));
+
+ rows = undiagonalize_neon(diag0, diag1, diag2, diag3);
+
+ vst1q_u8(partialblock + 0, vreinterpretq_u8_u32(rows.val[0]));
+ vst1q_u8(partialblock + 16, vreinterpretq_u8_u32(rows.val[1]));
+ vst1q_u8(partialblock + 32, vreinterpretq_u8_u32(rows.val[2]));
+ vst1q_u8(partialblock + 48, vreinterpretq_u8_u32(rows.val[3]));
+
+ for (i = 0; i < bytes; i++) {
+ c[i] = m[i] ^ partialblock[i];
+ }
+
+ sodium_memzero(partialblock, sizeof partialblock);
+}
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h
new file mode 100644
index 00000000..b7c84208
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h
@@ -0,0 +1,69 @@
+while (bytes >= 64) {
+ uint32x4_t diag0 = vld1q_u32(x + 0);
+ uint32x4_t diag1 = vld1q_u32(x + 4);
+ uint32x4_t diag2 = vld1q_u32(x + 8);
+ uint32x4_t diag3 = vld1q_u32(x + 12);
+ uint32x4_t a;
+ uint32x4x4_t rows;
+
+ uint32_t in8;
+ uint32_t in9;
+ int i;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ a = vaddq_u32(diag0, diag1);
+ diag3 = veorq_u32(diag3, rotl32_7_neon(a));
+ a = vaddq_u32(diag3, diag0);
+ diag2 = veorq_u32(diag2, rotl32_9_neon(a));
+ a = vaddq_u32(diag2, diag3);
+ diag1 = veorq_u32(diag1, rotl32_13_neon(a));
+ a = vaddq_u32(diag1, diag2);
+ diag0 = veorq_u32(diag0, rotl32_18_neon(a));
+
+ diag3 = vextq_u32(diag3, diag3, 3);
+ diag2 = vextq_u32(diag2, diag2, 2);
+ diag1 = vextq_u32(diag1, diag1, 1);
+
+ a = vaddq_u32(diag0, diag3);
+ diag1 = veorq_u32(diag1, rotl32_7_neon(a));
+ a = vaddq_u32(diag1, diag0);
+ diag2 = veorq_u32(diag2, rotl32_9_neon(a));
+ a = vaddq_u32(diag2, diag1);
+ diag3 = veorq_u32(diag3, rotl32_13_neon(a));
+ a = vaddq_u32(diag3, diag2);
+ diag0 = veorq_u32(diag0, rotl32_18_neon(a));
+
+ diag1 = vextq_u32(diag1, diag1, 3);
+ diag2 = vextq_u32(diag2, diag2, 2);
+ diag3 = vextq_u32(diag3, diag3, 1);
+ }
+
+ diag0 = vaddq_u32(diag0, vld1q_u32(x + 0));
+ diag1 = vaddq_u32(diag1, vld1q_u32(x + 4));
+ diag2 = vaddq_u32(diag2, vld1q_u32(x + 8));
+ diag3 = vaddq_u32(diag3, vld1q_u32(x + 12));
+
+ rows = undiagonalize_neon(diag0, diag1, diag2, diag3);
+
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 0)))));
+ vst1q_u8(c + 16, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 16)))));
+ vst1q_u8(c + 32, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 32)))));
+ vst1q_u8(c + 48, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 48)))));
+
+ in8 = x[8];
+ in9 = x[13];
+ in8++;
+ if (in8 == 0) {
+ in9++;
+ }
+ x[8] = in8;
+ x[13] = in9;
+
+ c += 64;
+ m += 64;
+ bytes -= 64;
+}
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h
new file mode 100644
index 00000000..f67806ca
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h
@@ -0,0 +1,85 @@
+
+#define VEC2_HALFROUND(T, D0, D1, D2, D3) \
+ T = vaddq_u32(D0, D1); \
+ D3 = veorq_u32(D3, rotl32_7_neon(T)); \
+ T = vaddq_u32(D3, D0); \
+ D2 = veorq_u32(D2, rotl32_9_neon(T)); \
+ T = vaddq_u32(D2, D3); \
+ D1 = veorq_u32(D1, rotl32_13_neon(T)); \
+ T = vaddq_u32(D1, D2); \
+ D0 = veorq_u32(D0, rotl32_18_neon(T)); \
+ D3 = vextq_u32(D3, D3, 3); \
+ D2 = vextq_u32(D2, D2, 2); \
+ D1 = vextq_u32(D1, D1, 1)
+
+while (bytes >= 128) {
+ uint32x4_t diag0 = vld1q_u32(x + 0);
+ uint32x4_t diag1 = vld1q_u32(x + 4);
+ uint32x4_t diag2 = vld1q_u32(x + 8);
+ uint32x4_t diag3 = vld1q_u32(x + 12);
+ uint32x4_t diag0b, diag1b, diag2b, diag3b;
+ uint32x4_t orig2b, orig3b;
+ uint32x4_t a, b;
+ uint32x4x4_t rows;
+
+ uint64_t in89;
+ int i;
+
+ in89 = (((uint64_t) x[8]) | (((uint64_t) x[13]) << 32)) + 1;
+
+ orig2b = vsetq_lane_u32((uint32_t) (in89 & 0xFFFFFFFF), diag2, 0);
+ orig3b = vsetq_lane_u32((uint32_t) (in89 >> 32), diag3, 1);
+
+ diag0b = diag0;
+ diag1b = diag1;
+ diag2b = orig2b;
+ diag3b = orig3b;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ VEC2_HALFROUND(a, diag0, diag1, diag2, diag3);
+ VEC2_HALFROUND(b, diag0b, diag1b, diag2b, diag3b);
+ VEC2_HALFROUND(a, diag0, diag3, diag2, diag1);
+ VEC2_HALFROUND(b, diag0b, diag3b, diag2b, diag1b);
+ }
+
+ diag0 = vaddq_u32(diag0, vld1q_u32(x + 0));
+ diag1 = vaddq_u32(diag1, vld1q_u32(x + 4));
+ diag2 = vaddq_u32(diag2, vld1q_u32(x + 8));
+ diag3 = vaddq_u32(diag3, vld1q_u32(x + 12));
+ diag0b = vaddq_u32(diag0b, vld1q_u32(x + 0));
+ diag1b = vaddq_u32(diag1b, vld1q_u32(x + 4));
+ diag2b = vaddq_u32(diag2b, orig2b);
+ diag3b = vaddq_u32(diag3b, orig3b);
+
+ rows = undiagonalize_neon(diag0, diag1, diag2, diag3);
+
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 0)))));
+ vst1q_u8(c + 16, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 16)))));
+ vst1q_u8(c + 32, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 32)))));
+ vst1q_u8(c + 48, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 48)))));
+
+ rows = undiagonalize_neon(diag0b, diag1b, diag2b, diag3b);
+
+ vst1q_u8(c + 64, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 64)))));
+ vst1q_u8(c + 80, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 80)))));
+ vst1q_u8(c + 96, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 96)))));
+ vst1q_u8(c + 112, vreinterpretq_u8_u32(veorq_u32(
+ rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 112)))));
+
+ in89 += 1;
+
+ x[8] = (uint32_t) (in89 & 0xFFFFFFFF);
+ x[13] = (uint32_t) (in89 >> 32);
+
+ c += 128;
+ m += 128;
+ bytes -= 128;
+}
+#undef VEC2_HALFROUND
diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h
new file mode 100644
index 00000000..eadda3bb
--- /dev/null
+++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h
@@ -0,0 +1,170 @@
+
+#define VEC4_QUARTERROUND(A, B, C, D) \
+ do { \
+ uint32x4_t t; \
+ \
+ t = vaddq_u32(z##A, z##D); \
+ z##B = veorq_u32(z##B, rotl32_7_neon(t)); \
+ t = vaddq_u32(z##B, z##A); \
+ z##C = veorq_u32(z##C, rotl32_9_neon(t)); \
+ t = vaddq_u32(z##C, z##B); \
+ z##D = veorq_u32(z##D, rotl32_13_neon(t)); \
+ t = vaddq_u32(z##D, z##C); \
+ z##A = veorq_u32(z##A, rotl32_18_neon(t)); \
+ } while (0)
+
+if (bytes >= 256) {
+ uint32x4_t y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14,
+ y15;
+ uint32x4_t z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14,
+ z15;
+ uint32x4_t orig0, orig1, orig2, orig3, orig4, orig5, orig6, orig7, orig8,
+ orig9, orig10, orig11, orig12, orig13, orig14, orig15;
+
+ uint32_t in8;
+ uint32_t in9;
+ int i;
+
+ z0 = vld1q_u32(x + 0);
+ z5 = vdupq_laneq_u32(z0, 1);
+ z10 = vdupq_laneq_u32(z0, 2);
+ z15 = vdupq_laneq_u32(z0, 3);
+ z0 = vdupq_laneq_u32(z0, 0);
+ z1 = vld1q_u32(x + 4);
+ z6 = vdupq_laneq_u32(z1, 2);
+ z11 = vdupq_laneq_u32(z1, 3);
+ z12 = vdupq_laneq_u32(z1, 0);
+ z1 = vdupq_laneq_u32(z1, 1);
+ z2 = vld1q_u32(x + 8);
+ z7 = vdupq_laneq_u32(z2, 3);
+ z13 = vdupq_laneq_u32(z2, 1);
+ z2 = vdupq_laneq_u32(z2, 2);
+ /* no z8 -> first half of the counter, will fill later */
+ z3 = vld1q_u32(x + 12);
+ z4 = vdupq_laneq_u32(z3, 0);
+ z14 = vdupq_laneq_u32(z3, 2);
+ z3 = vdupq_laneq_u32(z3, 3);
+ /* no z9 -> second half of the counter, will fill later */
+ orig0 = z0;
+ orig1 = z1;
+ orig2 = z2;
+ orig3 = z3;
+ orig4 = z4;
+ orig5 = z5;
+ orig6 = z6;
+ orig7 = z7;
+ orig10 = z10;
+ orig11 = z11;
+ orig12 = z12;
+ orig13 = z13;
+ orig14 = z14;
+ orig15 = z15;
+
+ while (bytes >= 256) {
+ static const uint64_t addv8_tbl[2] = { 0, 1 };
+ static const uint64_t addv9_tbl[2] = { 2, 3 };
+ uint64x2_t t8, t9;
+ uint64_t in89;
+
+ in8 = x[8];
+ in9 = x[13];
+ in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32);
+
+ t8 = vaddq_u64(vld1q_u64(addv8_tbl), vdupq_n_u64(in89));
+ t9 = vaddq_u64(vld1q_u64(addv9_tbl), vdupq_n_u64(in89));
+
+ z8 = vuzp1q_u32(vreinterpretq_u32_u64(t8), vreinterpretq_u32_u64(t9));
+ z9 = vuzp2q_u32(vreinterpretq_u32_u64(t8), vreinterpretq_u32_u64(t9));
+
+ orig8 = z8;
+ orig9 = z9;
+
+ in89 += 4;
+
+ x[8] = in89 & 0xFFFFFFFF;
+ x[13] = (in89 >> 32) & 0xFFFFFFFF;
+
+ z0 = orig0;
+ z1 = orig1;
+ z2 = orig2;
+ z3 = orig3;
+ z4 = orig4;
+ z5 = orig5;
+ z6 = orig6;
+ z7 = orig7;
+ z10 = orig10;
+ z11 = orig11;
+ z12 = orig12;
+ z13 = orig13;
+ z14 = orig14;
+ z15 = orig15;
+
+ for (i = 0; i < ROUNDS; i += 2) {
+ VEC4_QUARTERROUND(0, 4, 8, 12);
+ VEC4_QUARTERROUND(5, 9, 13, 1);
+ VEC4_QUARTERROUND(10, 14, 2, 6);
+ VEC4_QUARTERROUND(15, 3, 7, 11);
+ VEC4_QUARTERROUND(0, 1, 2, 3);
+ VEC4_QUARTERROUND(5, 6, 7, 4);
+ VEC4_QUARTERROUND(10, 11, 8, 9);
+ VEC4_QUARTERROUND(15, 12, 13, 14);
+ }
+
+#define ONEQUAD_TRANSPOSE(A, B, C, D) \
+ { \
+ uint32x4_t t0, t1, t2, t3; \
+ \
+ z##A = vaddq_u32(z##A, orig##A); \
+ z##B = vaddq_u32(z##B, orig##B); \
+ z##C = vaddq_u32(z##C, orig##C); \
+ z##D = vaddq_u32(z##D, orig##D); \
+ y##A = vtrn1q_u32(z##A, z##B); \
+ y##B = vtrn2q_u32(z##A, z##B); \
+ y##C = vtrn1q_u32(z##C, z##D); \
+ y##D = vtrn2q_u32(z##C, z##D); \
+ z##A = vreinterpretq_u32_u64( \
+ vtrn1q_u64(vreinterpretq_u64_u32(y##A), \
+ vreinterpretq_u64_u32(y##C))); \
+ z##B = vreinterpretq_u32_u64( \
+ vtrn1q_u64(vreinterpretq_u64_u32(y##B), \
+ vreinterpretq_u64_u32(y##D))); \
+ z##C = vreinterpretq_u32_u64( \
+ vtrn2q_u64(vreinterpretq_u64_u32(y##A), \
+ vreinterpretq_u64_u32(y##C))); \
+ z##D = vreinterpretq_u32_u64( \
+ vtrn2q_u64(vreinterpretq_u64_u32(y##B), \
+ vreinterpretq_u64_u32(y##D))); \
+ t0 = veorq_u32(z##A, vreinterpretq_u32_u8(vld1q_u8(m + 0))); \
+ vst1q_u8(c + 0, vreinterpretq_u8_u32(t0)); \
+ t1 = veorq_u32(z##B, vreinterpretq_u32_u8(vld1q_u8(m + 64))); \
+ vst1q_u8(c + 64, vreinterpretq_u8_u32(t1)); \
+ t2 = veorq_u32(z##C, vreinterpretq_u32_u8(vld1q_u8(m + 128))); \
+ vst1q_u8(c + 128, vreinterpretq_u8_u32(t2)); \
+ t3 = veorq_u32(z##D, vreinterpretq_u32_u8(vld1q_u8(m + 192))); \
+ vst1q_u8(c + 192, vreinterpretq_u8_u32(t3)); \
+ }
+
+#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
+
+ ONEQUAD(0, 1, 2, 3);
+ m += 16;
+ c += 16;
+ ONEQUAD(4, 5, 6, 7);
+ m += 16;
+ c += 16;
+ ONEQUAD(8, 9, 10, 11);
+ m += 16;
+ c += 16;
+ ONEQUAD(12, 13, 14, 15);
+ m -= 48;
+ c -= 48;
+
+#undef ONEQUAD
+#undef ONEQUAD_TRANSPOSE
+
+ bytes -= 256;
+ c += 256;
+ m += 256;
+ }
+}
+#undef VEC4_QUARTERROUND