From 5e261847ce3b84f0f1f6cd660bf2590f926b28e5 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sun, 12 Jul 2026 15:00:34 +0200 Subject: [PATCH] Add NEON implementations of salsa20 and chacha20 --- .../msvc/vs2010/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2012/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2013/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2015/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2017/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2019/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2022/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ .../msvc/vs2026/libsodium/libsodium.vcxproj | 12 ++ .../libsodium/libsodium.vcxproj.filters | 36 ++++ ci/appveyor/libsodium.vcxproj | 12 ++ ci/appveyor/libsodium.vcxproj.filters | 36 ++++ src/libsodium/Makefile.am | 14 +- .../chacha20/dolbeau/chacha20_dolbeau-neon.c | 202 ++++++++++++++++++ .../chacha20/dolbeau/chacha20_dolbeau-neon.h | 8 + .../crypto_stream/chacha20/dolbeau/u0-neon.h | 66 ++++++ .../crypto_stream/chacha20/dolbeau/u1-neon.h | 79 +++++++ .../crypto_stream/chacha20/dolbeau/u2-neon.h | 86 ++++++++ .../crypto_stream/chacha20/dolbeau/u4-neon.h | 164 ++++++++++++++ .../crypto_stream/chacha20/stream_chacha20.c | 11 + .../crypto_stream/salsa20/stream_salsa20.c | 11 + .../salsa20/xmm6int/salsa20_xmm6int-neon.c | 181 ++++++++++++++++ .../salsa20/xmm6int/salsa20_xmm6int-neon.h | 8 + .../crypto_stream/salsa20/xmm6int/u0-neon.h | 57 +++++ .../crypto_stream/salsa20/xmm6int/u1-neon.h | 69 ++++++ .../crypto_stream/salsa20/xmm6int/u2-neon.h | 85 ++++++++ .../crypto_stream/salsa20/xmm6int/u4-neon.h | 170 +++++++++++++++ 33 files changed, 1642 insertions(+), 1 deletion(-) create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h create mode 100644 src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h create mode 100644 src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj b/builds/msvc/vs2010/libsodium/libsodium.vcxproj index 77c1dd6d..9d521ad6 100644 --- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj @@ -162,11 +162,13 @@ + + @@ -345,23 +347,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj b/builds/msvc/vs2012/libsodium/libsodium.vcxproj index 57368b0d..92c277d4 100644 --- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj @@ -162,11 +162,13 @@ + + @@ -345,23 +347,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj b/builds/msvc/vs2013/libsodium/libsodium.vcxproj index c85ed549..f8a4d2b1 100644 --- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj @@ -162,11 +162,13 @@ + + @@ -345,23 +347,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj b/builds/msvc/vs2015/libsodium/libsodium.vcxproj index 6a4efbe2..39c644ec 100644 --- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj @@ -162,11 +162,13 @@ + + @@ -345,23 +347,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj b/builds/msvc/vs2017/libsodium/libsodium.vcxproj index 6fecdf2a..22a1a528 100644 --- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj @@ -162,11 +162,13 @@ + + @@ -345,23 +347,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj b/builds/msvc/vs2019/libsodium/libsodium.vcxproj index 9c57d2dc..b8966235 100644 --- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj @@ -186,11 +186,13 @@ + + @@ -369,23 +371,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj b/builds/msvc/vs2022/libsodium/libsodium.vcxproj index b6954440..edc52c6c 100644 --- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj @@ -186,11 +186,13 @@ + + @@ -369,23 +371,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj b/builds/msvc/vs2026/libsodium/libsodium.vcxproj index 12f7acf7..ae7068b0 100644 --- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj @@ -186,11 +186,13 @@ + + @@ -369,23 +371,33 @@ + + + + + + + + + + diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters index a0f80e26..053de321 100644 --- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters @@ -306,6 +306,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -321,6 +324,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -851,6 +857,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -869,9 +878,21 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + + + crypto_stream\chacha20\dolbeau + crypto_stream\salsa20 @@ -881,6 +902,9 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -896,12 +920,24 @@ crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6 diff --git a/ci/appveyor/libsodium.vcxproj b/ci/appveyor/libsodium.vcxproj index 1dbbae1d..7e9fb601 100644 --- a/ci/appveyor/libsodium.vcxproj +++ b/ci/appveyor/libsodium.vcxproj @@ -410,11 +410,13 @@ + + @@ -593,23 +595,33 @@ + + + + + + + + + + diff --git a/ci/appveyor/libsodium.vcxproj.filters b/ci/appveyor/libsodium.vcxproj.filters index 700b5c5f..84b80835 100644 --- a/ci/appveyor/libsodium.vcxproj.filters +++ b/ci/appveyor/libsodium.vcxproj.filters @@ -297,6 +297,9 @@ Source Files + + Source Files + Source Files @@ -312,6 +315,9 @@ Source Files + + Source Files + Source Files @@ -842,6 +848,9 @@ Header Files + + Header Files + Header Files @@ -860,9 +869,21 @@ Header Files + + Header Files + + + Header Files + Header Files + + Header Files + + + Header Files + Header Files @@ -872,6 +893,9 @@ Header Files + + Header Files + Header Files @@ -887,12 +911,24 @@ Header Files + + Header Files + + + Header Files + Header Files Header Files + + Header Files + + + Header Files + Header Files diff --git a/src/libsodium/Makefile.am b/src/libsodium/Makefile.am index 1e4b14f1..59720889 100644 --- a/src/libsodium/Makefile.am +++ b/src/libsodium/Makefile.am @@ -269,7 +269,19 @@ libarmcrypto_la_SOURCES = \ crypto_ipcrypt/ipcrypt_armcrypto.c \ crypto_ipcrypt/ipcrypt_armcrypto.h \ crypto_pwhash/argon2/argon2-fill-block-neon.c \ - crypto_pwhash/argon2/blamka-round-neon.h + crypto_pwhash/argon2/blamka-round-neon.h \ + crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c \ + crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h \ + crypto_stream/chacha20/dolbeau/u0-neon.h \ + crypto_stream/chacha20/dolbeau/u1-neon.h \ + crypto_stream/chacha20/dolbeau/u2-neon.h \ + crypto_stream/chacha20/dolbeau/u4-neon.h \ + crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c \ + crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h \ + crypto_stream/salsa20/xmm6int/u0-neon.h \ + crypto_stream/salsa20/xmm6int/u1-neon.h \ + crypto_stream/salsa20/xmm6int/u2-neon.h \ + crypto_stream/salsa20/xmm6int/u4-neon.h libaesni_la_LDFLAGS = $(libsodium_la_LDFLAGS) libaesni_la_CPPFLAGS = $(libsodium_la_CPPFLAGS) \ diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c new file mode 100644 index 00000000..9d5e6bfa --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.c @@ -0,0 +1,202 @@ + +#include +#include +#include + +#include "core.h" +#include "crypto_stream_chacha20.h" +#include "private/common.h" +#include "utils.h" + +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) + +# include + +# include "../ref/chacha20_ref.h" +# include "../stream_chacha20.h" +# include "chacha20_dolbeau-neon.h" + +# define ROUNDS 20 + +# define MIN_VECTOR_BYTES 128 + +typedef struct chacha_ctx { + uint32_t input[16]; +} chacha_ctx; + +static inline uint32x4_t +rotl32_16_neon(uint32x4_t x) +{ + return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); +} + +static inline uint32x4_t +rotl32_12_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 12), x, 20); +} + +static inline uint32x4_t +rotl32_8_neon(uint32x4_t x) +{ + static const uint8_t rot8_tbl[16] = { + 3, 0, 1, 2, 7, 4, 5, 6, + 11, 8, 9, 10, 15, 12, 13, 14 + }; + const uint8x16_t tbl = vld1q_u8(rot8_tbl); + + return vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(x), tbl)); +} + +static inline uint32x4_t +rotl32_7_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 7), x, 25); +} + +static void +chacha_keysetup(chacha_ctx *ctx, const uint8_t *k) +{ + ctx->input[0] = 0x61707865; + ctx->input[1] = 0x3320646e; + ctx->input[2] = 0x79622d32; + ctx->input[3] = 0x6b206574; + ctx->input[4] = LOAD32_LE(k + 0); + ctx->input[5] = LOAD32_LE(k + 4); + ctx->input[6] = LOAD32_LE(k + 8); + ctx->input[7] = LOAD32_LE(k + 12); + ctx->input[8] = LOAD32_LE(k + 16); + ctx->input[9] = LOAD32_LE(k + 20); + ctx->input[10] = LOAD32_LE(k + 24); + ctx->input[11] = LOAD32_LE(k + 28); +} + +static void +chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter + 0); + ctx->input[13] = counter == NULL ? 0 : LOAD32_LE(counter + 4); + ctx->input[14] = LOAD32_LE(iv + 0); + ctx->input[15] = LOAD32_LE(iv + 4); +} + +static void +chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter); + ctx->input[13] = LOAD32_LE(iv + 0); + ctx->input[14] = LOAD32_LE(iv + 4); + ctx->input[15] = LOAD32_LE(iv + 8); +} + +static void +chacha20_encrypt_bytes(chacha_ctx *ctx, const uint8_t *m, uint8_t *c, + unsigned long long bytes) +{ + uint32_t * const x = &ctx->input[0]; + + if (!bytes) { + return; /* LCOV_EXCL_LINE */ + } +# include "u4-neon.h" +# include "u2-neon.h" +# include "u1-neon.h" +# include "u0-neon.h" +} + +static int +stream_neon(unsigned char *c, unsigned long long clen, const unsigned char *n, + const unsigned char *k) +{ + struct chacha_ctx ctx; + + if (clen < MIN_VECTOR_BYTES) { + return crypto_stream_chacha20_ref_implementation.stream(c, clen, n, k); + } + COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); + chacha_keysetup(&ctx, k); + chacha_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + chacha20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_ietf_ext_neon(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) +{ + struct chacha_ctx ctx; + + if (clen < MIN_VECTOR_BYTES) { + return crypto_stream_chacha20_ref_implementation.stream_ietf_ext( + c, clen, n, k); + } + COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); + chacha_keysetup(&ctx, k); + chacha_ietf_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + chacha20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_neon_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + uint64_t ic, const unsigned char *k) +{ + struct chacha_ctx ctx; + uint8_t ic_bytes[8]; + uint32_t ic_high; + uint32_t ic_low; + + if (mlen < MIN_VECTOR_BYTES) { + return crypto_stream_chacha20_ref_implementation.stream_xor_ic( + c, m, mlen, n, ic, k); + } + ic_high = (uint32_t) (ic >> 32); + ic_low = (uint32_t) ic; + STORE32_LE(&ic_bytes[0], ic_low); + STORE32_LE(&ic_bytes[4], ic_high); + chacha_keysetup(&ctx, k); + chacha_ivsetup(&ctx, n, ic_bytes); + chacha20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_ietf_ext_neon_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + uint32_t ic, const unsigned char *k) +{ + struct chacha_ctx ctx; + uint8_t ic_bytes[4]; + + if (mlen < MIN_VECTOR_BYTES) { + return crypto_stream_chacha20_ref_implementation.stream_ietf_ext_xor_ic( + c, m, mlen, n, ic, k); + } + STORE32_LE(ic_bytes, ic); + chacha_keysetup(&ctx, k); + chacha_ietf_ivsetup(&ctx, n, ic_bytes); + chacha20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +struct crypto_stream_chacha20_implementation + crypto_stream_chacha20_dolbeau_neon_implementation = { + SODIUM_C99(.stream =) stream_neon, + SODIUM_C99(.stream_ietf_ext =) stream_ietf_ext_neon, + SODIUM_C99(.stream_xor_ic =) stream_neon_xor_ic, + SODIUM_C99(.stream_ietf_ext_xor_ic =) stream_ietf_ext_neon_xor_ic + }; + +#endif diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h new file mode 100644 index 00000000..18f2ebf4 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-neon.h @@ -0,0 +1,8 @@ + +#include + +#include "../stream_chacha20.h" +#include "crypto_stream_chacha20.h" + +extern struct crypto_stream_chacha20_implementation + crypto_stream_chacha20_dolbeau_neon_implementation; diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h new file mode 100644 index 00000000..ae031533 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u0-neon.h @@ -0,0 +1,66 @@ +if (bytes > 0) { + uint32x4_t x_0, x_1, x_2, x_3; + uint8_t partialblock[64]; + + unsigned int i; + + x_0 = vld1q_u32(x + 0); + x_1 = vld1q_u32(x + 4); + x_2 = vld1q_u32(x + 8); + x_3 = vld1q_u32(x + 12); + + for (i = 0; i < ROUNDS; i += 2) { + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_3 = rotl32_16_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_1 = veorq_u32(x_1, x_2); + x_1 = rotl32_12_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_0 = vextq_u32(x_0, x_0, 3); + x_3 = rotl32_8_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_3 = vextq_u32(x_3, x_3, 2); + x_1 = veorq_u32(x_1, x_2); + x_2 = vextq_u32(x_2, x_2, 1); + x_1 = rotl32_7_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_3 = rotl32_16_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_1 = veorq_u32(x_1, x_2); + x_1 = rotl32_12_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_0 = vextq_u32(x_0, x_0, 1); + x_3 = rotl32_8_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_3 = vextq_u32(x_3, x_3, 2); + x_1 = veorq_u32(x_1, x_2); + x_2 = vextq_u32(x_2, x_2, 3); + x_1 = rotl32_7_neon(x_1); + } + x_0 = vaddq_u32(x_0, vld1q_u32(x + 0)); + x_1 = vaddq_u32(x_1, vld1q_u32(x + 4)); + x_2 = vaddq_u32(x_2, vld1q_u32(x + 8)); + x_3 = vaddq_u32(x_3, vld1q_u32(x + 12)); + + vst1q_u8(partialblock + 0, vreinterpretq_u8_u32(x_0)); + vst1q_u8(partialblock + 16, vreinterpretq_u8_u32(x_1)); + vst1q_u8(partialblock + 32, vreinterpretq_u8_u32(x_2)); + vst1q_u8(partialblock + 48, vreinterpretq_u8_u32(x_3)); + + for (i = 0; i < bytes; i++) { + c[i] = m[i] ^ partialblock[i]; + } + + sodium_memzero(partialblock, sizeof partialblock); +} diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h new file mode 100644 index 00000000..275668bc --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u1-neon.h @@ -0,0 +1,79 @@ +while (bytes >= 64) { + uint32x4_t x_0, x_1, x_2, x_3; + + uint32_t in12; + uint32_t in13; + int i; + + x_0 = vld1q_u32(x + 0); + x_1 = vld1q_u32(x + 4); + x_2 = vld1q_u32(x + 8); + x_3 = vld1q_u32(x + 12); + + for (i = 0; i < ROUNDS; i += 2) { + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_3 = rotl32_16_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_1 = veorq_u32(x_1, x_2); + x_1 = rotl32_12_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_0 = vextq_u32(x_0, x_0, 3); + x_3 = rotl32_8_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_3 = vextq_u32(x_3, x_3, 2); + x_1 = veorq_u32(x_1, x_2); + x_2 = vextq_u32(x_2, x_2, 1); + x_1 = rotl32_7_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_3 = rotl32_16_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_1 = veorq_u32(x_1, x_2); + x_1 = rotl32_12_neon(x_1); + + x_0 = vaddq_u32(x_0, x_1); + x_3 = veorq_u32(x_3, x_0); + x_0 = vextq_u32(x_0, x_0, 1); + x_3 = rotl32_8_neon(x_3); + + x_2 = vaddq_u32(x_2, x_3); + x_3 = vextq_u32(x_3, x_3, 2); + x_1 = veorq_u32(x_1, x_2); + x_2 = vextq_u32(x_2, x_2, 3); + x_1 = rotl32_7_neon(x_1); + } + x_0 = vaddq_u32(x_0, vld1q_u32(x + 0)); + x_1 = vaddq_u32(x_1, vld1q_u32(x + 4)); + x_2 = vaddq_u32(x_2, vld1q_u32(x + 8)); + x_3 = vaddq_u32(x_3, vld1q_u32(x + 12)); + + x_0 = veorq_u32(x_0, vreinterpretq_u32_u8(vld1q_u8(m + 0))); + x_1 = veorq_u32(x_1, vreinterpretq_u32_u8(vld1q_u8(m + 16))); + x_2 = veorq_u32(x_2, vreinterpretq_u32_u8(vld1q_u8(m + 32))); + x_3 = veorq_u32(x_3, vreinterpretq_u32_u8(vld1q_u8(m + 48))); + + vst1q_u8(c + 0, vreinterpretq_u8_u32(x_0)); + vst1q_u8(c + 16, vreinterpretq_u8_u32(x_1)); + vst1q_u8(c + 32, vreinterpretq_u8_u32(x_2)); + vst1q_u8(c + 48, vreinterpretq_u8_u32(x_3)); + + in12 = x[12]; + in13 = x[13]; + in12++; + if (in12 == 0) { + in13++; + } + x[12] = in12; + x[13] = in13; + + bytes -= 64; + c += 64; + m += 64; +} diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h new file mode 100644 index 00000000..2986396c --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u2-neon.h @@ -0,0 +1,86 @@ + +#define VEC2_HALFROUND(P, E0, E2) \ + P##_0 = vaddq_u32(P##_0, P##_1); \ + P##_3 = veorq_u32(P##_3, P##_0); \ + P##_3 = rotl32_16_neon(P##_3); \ + P##_2 = vaddq_u32(P##_2, P##_3); \ + P##_1 = veorq_u32(P##_1, P##_2); \ + P##_1 = rotl32_12_neon(P##_1); \ + P##_0 = vaddq_u32(P##_0, P##_1); \ + P##_3 = veorq_u32(P##_3, P##_0); \ + P##_0 = vextq_u32(P##_0, P##_0, E0); \ + P##_3 = rotl32_8_neon(P##_3); \ + P##_2 = vaddq_u32(P##_2, P##_3); \ + P##_3 = vextq_u32(P##_3, P##_3, 2); \ + P##_1 = veorq_u32(P##_1, P##_2); \ + P##_2 = vextq_u32(P##_2, P##_2, E2); \ + P##_1 = rotl32_7_neon(P##_1) + +while (bytes >= 128) { + static const uint64_t addv_tbl[2] = { 1, 0 }; + + uint32x4_t orig0 = vld1q_u32(x + 0); + uint32x4_t orig1 = vld1q_u32(x + 4); + uint32x4_t orig2 = vld1q_u32(x + 8); + uint32x4_t orig3 = vld1q_u32(x + 12); + uint32x4_t orig3b = vreinterpretq_u32_u64(vaddq_u64( + vreinterpretq_u64_u32(orig3), vld1q_u64(addv_tbl))); + uint32x4_t x_0, x_1, x_2, x_3; + uint32x4_t y_0, y_1, y_2, y_3; + + uint64_t in1213; + int i; + + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + y_0 = orig0; + y_1 = orig1; + y_2 = orig2; + y_3 = orig3b; + + for (i = 0; i < ROUNDS; i += 2) { + VEC2_HALFROUND(x, 3, 1); + VEC2_HALFROUND(y, 3, 1); + VEC2_HALFROUND(x, 1, 3); + VEC2_HALFROUND(y, 1, 3); + } + x_0 = vaddq_u32(x_0, orig0); + x_1 = vaddq_u32(x_1, orig1); + x_2 = vaddq_u32(x_2, orig2); + x_3 = vaddq_u32(x_3, orig3); + y_0 = vaddq_u32(y_0, orig0); + y_1 = vaddq_u32(y_1, orig1); + y_2 = vaddq_u32(y_2, orig2); + y_3 = vaddq_u32(y_3, orig3b); + + x_0 = veorq_u32(x_0, vreinterpretq_u32_u8(vld1q_u8(m + 0))); + x_1 = veorq_u32(x_1, vreinterpretq_u32_u8(vld1q_u8(m + 16))); + x_2 = veorq_u32(x_2, vreinterpretq_u32_u8(vld1q_u8(m + 32))); + x_3 = veorq_u32(x_3, vreinterpretq_u32_u8(vld1q_u8(m + 48))); + y_0 = veorq_u32(y_0, vreinterpretq_u32_u8(vld1q_u8(m + 64))); + y_1 = veorq_u32(y_1, vreinterpretq_u32_u8(vld1q_u8(m + 80))); + y_2 = veorq_u32(y_2, vreinterpretq_u32_u8(vld1q_u8(m + 96))); + y_3 = veorq_u32(y_3, vreinterpretq_u32_u8(vld1q_u8(m + 112))); + + vst1q_u8(c + 0, vreinterpretq_u8_u32(x_0)); + vst1q_u8(c + 16, vreinterpretq_u8_u32(x_1)); + vst1q_u8(c + 32, vreinterpretq_u8_u32(x_2)); + vst1q_u8(c + 48, vreinterpretq_u8_u32(x_3)); + vst1q_u8(c + 64, vreinterpretq_u8_u32(y_0)); + vst1q_u8(c + 80, vreinterpretq_u8_u32(y_1)); + vst1q_u8(c + 96, vreinterpretq_u8_u32(y_2)); + vst1q_u8(c + 112, vreinterpretq_u8_u32(y_3)); + + in1213 = ((uint64_t) x[12]) | (((uint64_t) x[13]) << 32); + in1213 += 2; + + x[12] = (uint32_t) (in1213 & 0xFFFFFFFF); + x[13] = (uint32_t) (in1213 >> 32); + + bytes -= 128; + c += 128; + m += 128; +} +#undef VEC2_HALFROUND diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h new file mode 100644 index 00000000..7a5a6936 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u4-neon.h @@ -0,0 +1,164 @@ + +#define VEC4_QUARTERROUND(A, B, C, D) \ + x_##A = vaddq_u32(x_##A, x_##B); \ + t_##A = veorq_u32(x_##D, x_##A); \ + x_##D = rotl32_16_neon(t_##A); \ + x_##C = vaddq_u32(x_##C, x_##D); \ + t_##C = veorq_u32(x_##B, x_##C); \ + x_##B = rotl32_12_neon(t_##C); \ + x_##A = vaddq_u32(x_##A, x_##B); \ + t_##A = veorq_u32(x_##D, x_##A); \ + x_##D = rotl32_8_neon(t_##A); \ + x_##C = vaddq_u32(x_##C, x_##D); \ + t_##C = veorq_u32(x_##B, x_##C); \ + x_##B = rotl32_7_neon(t_##C) + +if (bytes >= 256) { + uint32x4_t x_0 = vdupq_n_u32(x[0]); + uint32x4_t x_1 = vdupq_n_u32(x[1]); + uint32x4_t x_2 = vdupq_n_u32(x[2]); + uint32x4_t x_3 = vdupq_n_u32(x[3]); + uint32x4_t x_4 = vdupq_n_u32(x[4]); + uint32x4_t x_5 = vdupq_n_u32(x[5]); + uint32x4_t x_6 = vdupq_n_u32(x[6]); + uint32x4_t x_7 = vdupq_n_u32(x[7]); + uint32x4_t x_8 = vdupq_n_u32(x[8]); + uint32x4_t x_9 = vdupq_n_u32(x[9]); + uint32x4_t x_10 = vdupq_n_u32(x[10]); + uint32x4_t x_11 = vdupq_n_u32(x[11]); + uint32x4_t x_12; + uint32x4_t x_13; + uint32x4_t x_14 = vdupq_n_u32(x[14]); + uint32x4_t x_15 = vdupq_n_u32(x[15]); + uint32x4_t orig0 = x_0; + uint32x4_t orig1 = x_1; + uint32x4_t orig2 = x_2; + uint32x4_t orig3 = x_3; + uint32x4_t orig4 = x_4; + uint32x4_t orig5 = x_5; + uint32x4_t orig6 = x_6; + uint32x4_t orig7 = x_7; + uint32x4_t orig8 = x_8; + uint32x4_t orig9 = x_9; + uint32x4_t orig10 = x_10; + uint32x4_t orig11 = x_11; + uint32x4_t orig12; + uint32x4_t orig13; + uint32x4_t orig14 = x_14; + uint32x4_t orig15 = x_15; + uint32x4_t t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, + t_12, t_13, t_14, t_15; + + uint32_t in12, in13; + int i; + + while (bytes >= 256) { + static const uint64_t addv12_tbl[2] = { 0, 1 }; + static const uint64_t addv13_tbl[2] = { 2, 3 }; + uint64x2_t t12, t13; + uint64_t in1213; + + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + x_4 = orig4; + x_5 = orig5; + x_6 = orig6; + x_7 = orig7; + x_8 = orig8; + x_9 = orig9; + x_10 = orig10; + x_11 = orig11; + x_14 = orig14; + x_15 = orig15; + + in12 = x[12]; + in13 = x[13]; + in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32); + + t12 = vaddq_u64(vld1q_u64(addv12_tbl), vdupq_n_u64(in1213)); + t13 = vaddq_u64(vld1q_u64(addv13_tbl), vdupq_n_u64(in1213)); + + x_12 = vuzp1q_u32(vreinterpretq_u32_u64(t12), + vreinterpretq_u32_u64(t13)); + x_13 = vuzp2q_u32(vreinterpretq_u32_u64(t12), + vreinterpretq_u32_u64(t13)); + + orig12 = x_12; + orig13 = x_13; + + in1213 += 4; + + x[12] = in1213 & 0xFFFFFFFF; + x[13] = (in1213 >> 32) & 0xFFFFFFFF; + + for (i = 0; i < ROUNDS; i += 2) { + VEC4_QUARTERROUND(0, 4, 8, 12); + VEC4_QUARTERROUND(1, 5, 9, 13); + VEC4_QUARTERROUND(2, 6, 10, 14); + VEC4_QUARTERROUND(3, 7, 11, 15); + VEC4_QUARTERROUND(0, 5, 10, 15); + VEC4_QUARTERROUND(1, 6, 11, 12); + VEC4_QUARTERROUND(2, 7, 8, 13); + VEC4_QUARTERROUND(3, 4, 9, 14); + } + +#define ONEQUAD_TRANSPOSE(A, B, C, D) \ + { \ + uint32x4_t t0, t1, t2, t3; \ + \ + x_##A = vaddq_u32(x_##A, orig##A); \ + x_##B = vaddq_u32(x_##B, orig##B); \ + x_##C = vaddq_u32(x_##C, orig##C); \ + x_##D = vaddq_u32(x_##D, orig##D); \ + t_##A = vtrn1q_u32(x_##A, x_##B); \ + t_##B = vtrn2q_u32(x_##A, x_##B); \ + t_##C = vtrn1q_u32(x_##C, x_##D); \ + t_##D = vtrn2q_u32(x_##C, x_##D); \ + x_##A = vreinterpretq_u32_u64( \ + vtrn1q_u64(vreinterpretq_u64_u32(t_##A), \ + vreinterpretq_u64_u32(t_##C))); \ + x_##B = vreinterpretq_u32_u64( \ + vtrn1q_u64(vreinterpretq_u64_u32(t_##B), \ + vreinterpretq_u64_u32(t_##D))); \ + x_##C = vreinterpretq_u32_u64( \ + vtrn2q_u64(vreinterpretq_u64_u32(t_##A), \ + vreinterpretq_u64_u32(t_##C))); \ + x_##D = vreinterpretq_u32_u64( \ + vtrn2q_u64(vreinterpretq_u64_u32(t_##B), \ + vreinterpretq_u64_u32(t_##D))); \ + t0 = veorq_u32(x_##A, vreinterpretq_u32_u8(vld1q_u8(m + 0))); \ + vst1q_u8(c + 0, vreinterpretq_u8_u32(t0)); \ + t1 = veorq_u32(x_##B, vreinterpretq_u32_u8(vld1q_u8(m + 64))); \ + vst1q_u8(c + 64, vreinterpretq_u8_u32(t1)); \ + t2 = veorq_u32(x_##C, vreinterpretq_u32_u8(vld1q_u8(m + 128))); \ + vst1q_u8(c + 128, vreinterpretq_u8_u32(t2)); \ + t3 = veorq_u32(x_##D, vreinterpretq_u32_u8(vld1q_u8(m + 192))); \ + vst1q_u8(c + 192, vreinterpretq_u8_u32(t3)); \ + } + +#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) + + ONEQUAD(0, 1, 2, 3); + m += 16; + c += 16; + ONEQUAD(4, 5, 6, 7); + m += 16; + c += 16; + ONEQUAD(8, 9, 10, 11); + m += 16; + c += 16; + ONEQUAD(12, 13, 14, 15); + m -= 48; + c -= 48; + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE + + bytes -= 256; + c += 256; + m += 256; + } +} +#undef VEC4_QUARTERROUND diff --git a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c index 8bd46d2b..aa2232c0 100644 --- a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c +++ b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c @@ -20,6 +20,10 @@ #if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) # include "dolbeau/chacha20_dolbeau-ssse3.h" #endif +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) +# include "dolbeau/chacha20_dolbeau-neon.h" +#endif static const crypto_stream_chacha20_implementation *implementation = &crypto_stream_chacha20_ref_implementation; @@ -192,6 +196,13 @@ _crypto_stream_chacha20_pick_best_implementation(void) implementation = &crypto_stream_chacha20_dolbeau_ssse3_implementation; return 0; } +#endif +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) + if (sodium_runtime_has_neon()) { + implementation = &crypto_stream_chacha20_dolbeau_neon_implementation; + return 0; + } #endif return 0; } diff --git a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c index 2979a55d..25e5c780 100644 --- a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c +++ b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c @@ -22,6 +22,10 @@ defined(HAVE_SMMINTRIN_H) # include "xmm6int/salsa20_xmm6int-avx512.h" #endif +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) +# include "xmm6int/salsa20_xmm6int-neon.h" +#endif #if HAVE_AMD64_ASM static const crypto_stream_salsa20_implementation *implementation = @@ -108,6 +112,13 @@ _crypto_stream_salsa20_pick_best_implementation(void) implementation = &crypto_stream_salsa20_xmm6int_sse2_implementation; return 0; } +#endif +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) + if (sodium_runtime_has_neon()) { + implementation = &crypto_stream_salsa20_xmm6int_neon_implementation; + return 0; + } #endif return 0; /* LCOV_EXCL_LINE */ } diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c new file mode 100644 index 00000000..6abbf140 --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.c @@ -0,0 +1,181 @@ + +#include +#include +#include + +#include "crypto_stream_salsa20.h" +#include "private/common.h" +#include "utils.h" + +#if (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(__ARM_NEON) || defined(__ARM_NEON__)) + +# include + +# include "../ref/salsa20_ref.h" +# include "../stream_salsa20.h" +# include "salsa20_xmm6int-neon.h" + +# define ROUNDS 20 + +# define MIN_VECTOR_BYTES 256 + +typedef struct salsa_ctx { + uint32_t input[16]; +} salsa_ctx; + +static const int TR[16] = { + 0, 5, 10, 15, 12, 1, 6, 11, 8, 13, 2, 7, 4, 9, 14, 3 +}; + +static inline uint32x4_t +rotl32_7_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 7), x, 25); +} + +static inline uint32x4_t +rotl32_9_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 9), x, 23); +} + +static inline uint32x4_t +rotl32_13_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 13), x, 19); +} + +static inline uint32x4_t +rotl32_18_neon(uint32x4_t x) +{ + return vsriq_n_u32(vshlq_n_u32(x, 18), x, 14); +} + +static inline uint32x4x4_t +undiagonalize_neon(uint32x4_t diag0, uint32x4_t diag1, uint32x4_t diag2, + uint32x4_t diag3) +{ + uint32x4_t t0, t1, t2, t3; + uint32x4x4_t rows; + + diag1 = vextq_u32(diag1, diag1, 1); + diag2 = vextq_u32(diag2, diag2, 2); + diag3 = vextq_u32(diag3, diag3, 3); + + t0 = vtrn1q_u32(diag0, diag1); + t1 = vtrn2q_u32(diag0, diag1); + t2 = vtrn1q_u32(diag2, diag3); + t3 = vtrn2q_u32(diag2, diag3); + + diag0 = vreinterpretq_u32_u64( + vtrn1q_u64(vreinterpretq_u64_u32(t0), vreinterpretq_u64_u32(t2))); + diag1 = vreinterpretq_u32_u64( + vtrn1q_u64(vreinterpretq_u64_u32(t1), vreinterpretq_u64_u32(t3))); + diag2 = vreinterpretq_u32_u64( + vtrn2q_u64(vreinterpretq_u64_u32(t0), vreinterpretq_u64_u32(t2))); + diag3 = vreinterpretq_u32_u64( + vtrn2q_u64(vreinterpretq_u64_u32(t1), vreinterpretq_u64_u32(t3))); + + rows.val[0] = diag0; + rows.val[1] = vextq_u32(diag1, diag1, 3); + rows.val[2] = vextq_u32(diag2, diag2, 2); + rows.val[3] = vextq_u32(diag3, diag3, 1); + + return rows; +} + +static void +salsa_keysetup(salsa_ctx *ctx, const uint8_t *k) +{ + ctx->input[TR[1]] = LOAD32_LE(k + 0); + ctx->input[TR[2]] = LOAD32_LE(k + 4); + ctx->input[TR[3]] = LOAD32_LE(k + 8); + ctx->input[TR[4]] = LOAD32_LE(k + 12); + ctx->input[TR[11]] = LOAD32_LE(k + 16); + ctx->input[TR[12]] = LOAD32_LE(k + 20); + ctx->input[TR[13]] = LOAD32_LE(k + 24); + ctx->input[TR[14]] = LOAD32_LE(k + 28); + ctx->input[TR[0]] = 0x61707865; + ctx->input[TR[5]] = 0x3320646e; + ctx->input[TR[10]] = 0x79622d32; + ctx->input[TR[15]] = 0x6b206574; +} + +static void +salsa_ivsetup(salsa_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[TR[6]] = LOAD32_LE(iv + 0); + ctx->input[TR[7]] = LOAD32_LE(iv + 4); + ctx->input[TR[8]] = counter == NULL ? 0 : LOAD32_LE(counter + 0); + ctx->input[TR[9]] = counter == NULL ? 0 : LOAD32_LE(counter + 4); +} + +static void +salsa20_encrypt_bytes(salsa_ctx *ctx, const uint8_t *m, uint8_t *c, + unsigned long long bytes) +{ + uint32_t * const x = &ctx->input[0]; + + if (!bytes) { + return; /* LCOV_EXCL_LINE */ + } + +#include "u4-neon.h" +#include "u2-neon.h" +#include "u1-neon.h" +#include "u0-neon.h" +} + +static int +stream_neon(unsigned char *c, unsigned long long clen, const unsigned char *n, + const unsigned char *k) +{ + struct salsa_ctx ctx; + + if (clen < MIN_VECTOR_BYTES) { + return crypto_stream_salsa20_ref_implementation.stream(c, clen, n, k); + } + COMPILER_ASSERT(crypto_stream_salsa20_KEYBYTES == 256 / 8); + salsa_keysetup(&ctx, k); + salsa_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + salsa20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_neon_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, uint64_t ic, + const unsigned char *k) +{ + struct salsa_ctx ctx; + uint8_t ic_bytes[8]; + uint32_t ic_high; + uint32_t ic_low; + + if (mlen < MIN_VECTOR_BYTES) { + return crypto_stream_salsa20_ref_implementation.stream_xor_ic( + c, m, mlen, n, ic, k); + } + ic_high = (uint32_t) (ic >> 32); + ic_low = (uint32_t) (ic); + STORE32_LE(&ic_bytes[0], ic_low); + STORE32_LE(&ic_bytes[4], ic_high); + salsa_keysetup(&ctx, k); + salsa_ivsetup(&ctx, n, ic_bytes); + salsa20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +struct crypto_stream_salsa20_implementation + crypto_stream_salsa20_xmm6int_neon_implementation = { + SODIUM_C99(.stream =) stream_neon, + SODIUM_C99(.stream_xor_ic =) stream_neon_xor_ic + }; + +#endif diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h new file mode 100644 index 00000000..0dbfc569 --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-neon.h @@ -0,0 +1,8 @@ + +#include + +#include "../stream_salsa20.h" +#include "crypto_stream_salsa20.h" + +extern struct crypto_stream_salsa20_implementation + crypto_stream_salsa20_xmm6int_neon_implementation; diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h new file mode 100644 index 00000000..519b4077 --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u0-neon.h @@ -0,0 +1,57 @@ +if (bytes > 0) { + uint32x4_t diag0 = vld1q_u32(x + 0); + uint32x4_t diag1 = vld1q_u32(x + 4); + uint32x4_t diag2 = vld1q_u32(x + 8); + uint32x4_t diag3 = vld1q_u32(x + 12); + uint32x4_t a; + uint32x4x4_t rows; + uint8_t partialblock[64]; + + unsigned int i; + + for (i = 0; i < ROUNDS; i += 2) { + a = vaddq_u32(diag0, diag1); + diag3 = veorq_u32(diag3, rotl32_7_neon(a)); + a = vaddq_u32(diag3, diag0); + diag2 = veorq_u32(diag2, rotl32_9_neon(a)); + a = vaddq_u32(diag2, diag3); + diag1 = veorq_u32(diag1, rotl32_13_neon(a)); + a = vaddq_u32(diag1, diag2); + diag0 = veorq_u32(diag0, rotl32_18_neon(a)); + + diag3 = vextq_u32(diag3, diag3, 3); + diag2 = vextq_u32(diag2, diag2, 2); + diag1 = vextq_u32(diag1, diag1, 1); + + a = vaddq_u32(diag0, diag3); + diag1 = veorq_u32(diag1, rotl32_7_neon(a)); + a = vaddq_u32(diag1, diag0); + diag2 = veorq_u32(diag2, rotl32_9_neon(a)); + a = vaddq_u32(diag2, diag1); + diag3 = veorq_u32(diag3, rotl32_13_neon(a)); + a = vaddq_u32(diag3, diag2); + diag0 = veorq_u32(diag0, rotl32_18_neon(a)); + + diag1 = vextq_u32(diag1, diag1, 3); + diag2 = vextq_u32(diag2, diag2, 2); + diag3 = vextq_u32(diag3, diag3, 1); + } + + diag0 = vaddq_u32(diag0, vld1q_u32(x + 0)); + diag1 = vaddq_u32(diag1, vld1q_u32(x + 4)); + diag2 = vaddq_u32(diag2, vld1q_u32(x + 8)); + diag3 = vaddq_u32(diag3, vld1q_u32(x + 12)); + + rows = undiagonalize_neon(diag0, diag1, diag2, diag3); + + vst1q_u8(partialblock + 0, vreinterpretq_u8_u32(rows.val[0])); + vst1q_u8(partialblock + 16, vreinterpretq_u8_u32(rows.val[1])); + vst1q_u8(partialblock + 32, vreinterpretq_u8_u32(rows.val[2])); + vst1q_u8(partialblock + 48, vreinterpretq_u8_u32(rows.val[3])); + + for (i = 0; i < bytes; i++) { + c[i] = m[i] ^ partialblock[i]; + } + + sodium_memzero(partialblock, sizeof partialblock); +} diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h new file mode 100644 index 00000000..b7c84208 --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u1-neon.h @@ -0,0 +1,69 @@ +while (bytes >= 64) { + uint32x4_t diag0 = vld1q_u32(x + 0); + uint32x4_t diag1 = vld1q_u32(x + 4); + uint32x4_t diag2 = vld1q_u32(x + 8); + uint32x4_t diag3 = vld1q_u32(x + 12); + uint32x4_t a; + uint32x4x4_t rows; + + uint32_t in8; + uint32_t in9; + int i; + + for (i = 0; i < ROUNDS; i += 2) { + a = vaddq_u32(diag0, diag1); + diag3 = veorq_u32(diag3, rotl32_7_neon(a)); + a = vaddq_u32(diag3, diag0); + diag2 = veorq_u32(diag2, rotl32_9_neon(a)); + a = vaddq_u32(diag2, diag3); + diag1 = veorq_u32(diag1, rotl32_13_neon(a)); + a = vaddq_u32(diag1, diag2); + diag0 = veorq_u32(diag0, rotl32_18_neon(a)); + + diag3 = vextq_u32(diag3, diag3, 3); + diag2 = vextq_u32(diag2, diag2, 2); + diag1 = vextq_u32(diag1, diag1, 1); + + a = vaddq_u32(diag0, diag3); + diag1 = veorq_u32(diag1, rotl32_7_neon(a)); + a = vaddq_u32(diag1, diag0); + diag2 = veorq_u32(diag2, rotl32_9_neon(a)); + a = vaddq_u32(diag2, diag1); + diag3 = veorq_u32(diag3, rotl32_13_neon(a)); + a = vaddq_u32(diag3, diag2); + diag0 = veorq_u32(diag0, rotl32_18_neon(a)); + + diag1 = vextq_u32(diag1, diag1, 3); + diag2 = vextq_u32(diag2, diag2, 2); + diag3 = vextq_u32(diag3, diag3, 1); + } + + diag0 = vaddq_u32(diag0, vld1q_u32(x + 0)); + diag1 = vaddq_u32(diag1, vld1q_u32(x + 4)); + diag2 = vaddq_u32(diag2, vld1q_u32(x + 8)); + diag3 = vaddq_u32(diag3, vld1q_u32(x + 12)); + + rows = undiagonalize_neon(diag0, diag1, diag2, diag3); + + vst1q_u8(c + 0, vreinterpretq_u8_u32(veorq_u32( + rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 0))))); + vst1q_u8(c + 16, vreinterpretq_u8_u32(veorq_u32( + rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 16))))); + vst1q_u8(c + 32, vreinterpretq_u8_u32(veorq_u32( + rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 32))))); + vst1q_u8(c + 48, vreinterpretq_u8_u32(veorq_u32( + rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 48))))); + + in8 = x[8]; + in9 = x[13]; + in8++; + if (in8 == 0) { + in9++; + } + x[8] = in8; + x[13] = in9; + + c += 64; + m += 64; + bytes -= 64; +} diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h new file mode 100644 index 00000000..f67806ca --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u2-neon.h @@ -0,0 +1,85 @@ + +#define VEC2_HALFROUND(T, D0, D1, D2, D3) \ + T = vaddq_u32(D0, D1); \ + D3 = veorq_u32(D3, rotl32_7_neon(T)); \ + T = vaddq_u32(D3, D0); \ + D2 = veorq_u32(D2, rotl32_9_neon(T)); \ + T = vaddq_u32(D2, D3); \ + D1 = veorq_u32(D1, rotl32_13_neon(T)); \ + T = vaddq_u32(D1, D2); \ + D0 = veorq_u32(D0, rotl32_18_neon(T)); \ + D3 = vextq_u32(D3, D3, 3); \ + D2 = vextq_u32(D2, D2, 2); \ + D1 = vextq_u32(D1, D1, 1) + +while (bytes >= 128) { + uint32x4_t diag0 = vld1q_u32(x + 0); + uint32x4_t diag1 = vld1q_u32(x + 4); + uint32x4_t diag2 = vld1q_u32(x + 8); + uint32x4_t diag3 = vld1q_u32(x + 12); + uint32x4_t diag0b, diag1b, diag2b, diag3b; + uint32x4_t orig2b, orig3b; + uint32x4_t a, b; + uint32x4x4_t rows; + + uint64_t in89; + int i; + + in89 = (((uint64_t) x[8]) | (((uint64_t) x[13]) << 32)) + 1; + + orig2b = vsetq_lane_u32((uint32_t) (in89 & 0xFFFFFFFF), diag2, 0); + orig3b = vsetq_lane_u32((uint32_t) (in89 >> 32), diag3, 1); + + diag0b = diag0; + diag1b = diag1; + diag2b = orig2b; + diag3b = orig3b; + + for (i = 0; i < ROUNDS; i += 2) { + VEC2_HALFROUND(a, diag0, diag1, diag2, diag3); + VEC2_HALFROUND(b, diag0b, diag1b, diag2b, diag3b); + VEC2_HALFROUND(a, diag0, diag3, diag2, diag1); + VEC2_HALFROUND(b, diag0b, diag3b, diag2b, diag1b); + } + + diag0 = vaddq_u32(diag0, vld1q_u32(x + 0)); + diag1 = vaddq_u32(diag1, vld1q_u32(x + 4)); + diag2 = vaddq_u32(diag2, vld1q_u32(x + 8)); + diag3 = vaddq_u32(diag3, vld1q_u32(x + 12)); + diag0b = vaddq_u32(diag0b, vld1q_u32(x + 0)); + diag1b = vaddq_u32(diag1b, vld1q_u32(x + 4)); + diag2b = vaddq_u32(diag2b, orig2b); + diag3b = vaddq_u32(diag3b, orig3b); + + rows = undiagonalize_neon(diag0, diag1, diag2, diag3); + + vst1q_u8(c + 0, vreinterpretq_u8_u32(veorq_u32( + rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 0))))); + vst1q_u8(c + 16, vreinterpretq_u8_u32(veorq_u32( + rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 16))))); + vst1q_u8(c + 32, vreinterpretq_u8_u32(veorq_u32( + rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 32))))); + vst1q_u8(c + 48, vreinterpretq_u8_u32(veorq_u32( + rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 48))))); + + rows = undiagonalize_neon(diag0b, diag1b, diag2b, diag3b); + + vst1q_u8(c + 64, vreinterpretq_u8_u32(veorq_u32( + rows.val[0], vreinterpretq_u32_u8(vld1q_u8(m + 64))))); + vst1q_u8(c + 80, vreinterpretq_u8_u32(veorq_u32( + rows.val[1], vreinterpretq_u32_u8(vld1q_u8(m + 80))))); + vst1q_u8(c + 96, vreinterpretq_u8_u32(veorq_u32( + rows.val[2], vreinterpretq_u32_u8(vld1q_u8(m + 96))))); + vst1q_u8(c + 112, vreinterpretq_u8_u32(veorq_u32( + rows.val[3], vreinterpretq_u32_u8(vld1q_u8(m + 112))))); + + in89 += 1; + + x[8] = (uint32_t) (in89 & 0xFFFFFFFF); + x[13] = (uint32_t) (in89 >> 32); + + c += 128; + m += 128; + bytes -= 128; +} +#undef VEC2_HALFROUND diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h new file mode 100644 index 00000000..eadda3bb --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u4-neon.h @@ -0,0 +1,170 @@ + +#define VEC4_QUARTERROUND(A, B, C, D) \ + do { \ + uint32x4_t t; \ + \ + t = vaddq_u32(z##A, z##D); \ + z##B = veorq_u32(z##B, rotl32_7_neon(t)); \ + t = vaddq_u32(z##B, z##A); \ + z##C = veorq_u32(z##C, rotl32_9_neon(t)); \ + t = vaddq_u32(z##C, z##B); \ + z##D = veorq_u32(z##D, rotl32_13_neon(t)); \ + t = vaddq_u32(z##D, z##C); \ + z##A = veorq_u32(z##A, rotl32_18_neon(t)); \ + } while (0) + +if (bytes >= 256) { + uint32x4_t y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + uint32x4_t z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15; + uint32x4_t orig0, orig1, orig2, orig3, orig4, orig5, orig6, orig7, orig8, + orig9, orig10, orig11, orig12, orig13, orig14, orig15; + + uint32_t in8; + uint32_t in9; + int i; + + z0 = vld1q_u32(x + 0); + z5 = vdupq_laneq_u32(z0, 1); + z10 = vdupq_laneq_u32(z0, 2); + z15 = vdupq_laneq_u32(z0, 3); + z0 = vdupq_laneq_u32(z0, 0); + z1 = vld1q_u32(x + 4); + z6 = vdupq_laneq_u32(z1, 2); + z11 = vdupq_laneq_u32(z1, 3); + z12 = vdupq_laneq_u32(z1, 0); + z1 = vdupq_laneq_u32(z1, 1); + z2 = vld1q_u32(x + 8); + z7 = vdupq_laneq_u32(z2, 3); + z13 = vdupq_laneq_u32(z2, 1); + z2 = vdupq_laneq_u32(z2, 2); + /* no z8 -> first half of the counter, will fill later */ + z3 = vld1q_u32(x + 12); + z4 = vdupq_laneq_u32(z3, 0); + z14 = vdupq_laneq_u32(z3, 2); + z3 = vdupq_laneq_u32(z3, 3); + /* no z9 -> second half of the counter, will fill later */ + orig0 = z0; + orig1 = z1; + orig2 = z2; + orig3 = z3; + orig4 = z4; + orig5 = z5; + orig6 = z6; + orig7 = z7; + orig10 = z10; + orig11 = z11; + orig12 = z12; + orig13 = z13; + orig14 = z14; + orig15 = z15; + + while (bytes >= 256) { + static const uint64_t addv8_tbl[2] = { 0, 1 }; + static const uint64_t addv9_tbl[2] = { 2, 3 }; + uint64x2_t t8, t9; + uint64_t in89; + + in8 = x[8]; + in9 = x[13]; + in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32); + + t8 = vaddq_u64(vld1q_u64(addv8_tbl), vdupq_n_u64(in89)); + t9 = vaddq_u64(vld1q_u64(addv9_tbl), vdupq_n_u64(in89)); + + z8 = vuzp1q_u32(vreinterpretq_u32_u64(t8), vreinterpretq_u32_u64(t9)); + z9 = vuzp2q_u32(vreinterpretq_u32_u64(t8), vreinterpretq_u32_u64(t9)); + + orig8 = z8; + orig9 = z9; + + in89 += 4; + + x[8] = in89 & 0xFFFFFFFF; + x[13] = (in89 >> 32) & 0xFFFFFFFF; + + z0 = orig0; + z1 = orig1; + z2 = orig2; + z3 = orig3; + z4 = orig4; + z5 = orig5; + z6 = orig6; + z7 = orig7; + z10 = orig10; + z11 = orig11; + z12 = orig12; + z13 = orig13; + z14 = orig14; + z15 = orig15; + + for (i = 0; i < ROUNDS; i += 2) { + VEC4_QUARTERROUND(0, 4, 8, 12); + VEC4_QUARTERROUND(5, 9, 13, 1); + VEC4_QUARTERROUND(10, 14, 2, 6); + VEC4_QUARTERROUND(15, 3, 7, 11); + VEC4_QUARTERROUND(0, 1, 2, 3); + VEC4_QUARTERROUND(5, 6, 7, 4); + VEC4_QUARTERROUND(10, 11, 8, 9); + VEC4_QUARTERROUND(15, 12, 13, 14); + } + +#define ONEQUAD_TRANSPOSE(A, B, C, D) \ + { \ + uint32x4_t t0, t1, t2, t3; \ + \ + z##A = vaddq_u32(z##A, orig##A); \ + z##B = vaddq_u32(z##B, orig##B); \ + z##C = vaddq_u32(z##C, orig##C); \ + z##D = vaddq_u32(z##D, orig##D); \ + y##A = vtrn1q_u32(z##A, z##B); \ + y##B = vtrn2q_u32(z##A, z##B); \ + y##C = vtrn1q_u32(z##C, z##D); \ + y##D = vtrn2q_u32(z##C, z##D); \ + z##A = vreinterpretq_u32_u64( \ + vtrn1q_u64(vreinterpretq_u64_u32(y##A), \ + vreinterpretq_u64_u32(y##C))); \ + z##B = vreinterpretq_u32_u64( \ + vtrn1q_u64(vreinterpretq_u64_u32(y##B), \ + vreinterpretq_u64_u32(y##D))); \ + z##C = vreinterpretq_u32_u64( \ + vtrn2q_u64(vreinterpretq_u64_u32(y##A), \ + vreinterpretq_u64_u32(y##C))); \ + z##D = vreinterpretq_u32_u64( \ + vtrn2q_u64(vreinterpretq_u64_u32(y##B), \ + vreinterpretq_u64_u32(y##D))); \ + t0 = veorq_u32(z##A, vreinterpretq_u32_u8(vld1q_u8(m + 0))); \ + vst1q_u8(c + 0, vreinterpretq_u8_u32(t0)); \ + t1 = veorq_u32(z##B, vreinterpretq_u32_u8(vld1q_u8(m + 64))); \ + vst1q_u8(c + 64, vreinterpretq_u8_u32(t1)); \ + t2 = veorq_u32(z##C, vreinterpretq_u32_u8(vld1q_u8(m + 128))); \ + vst1q_u8(c + 128, vreinterpretq_u8_u32(t2)); \ + t3 = veorq_u32(z##D, vreinterpretq_u32_u8(vld1q_u8(m + 192))); \ + vst1q_u8(c + 192, vreinterpretq_u8_u32(t3)); \ + } + +#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) + + ONEQUAD(0, 1, 2, 3); + m += 16; + c += 16; + ONEQUAD(4, 5, 6, 7); + m += 16; + c += 16; + ONEQUAD(8, 9, 10, 11); + m += 16; + c += 16; + ONEQUAD(12, 13, 14, 15); + m -= 48; + c -= 48; + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE + + bytes -= 256; + c += 256; + m += 256; + } +} +#undef VEC4_QUARTERROUND