diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj b/builds/msvc/vs2010/libsodium/libsodium.vcxproj index f9816ab6..77c1dd6d 100644 --- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj @@ -161,10 +161,12 @@ + + @@ -342,16 +344,20 @@ + + + + diff --git a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2010/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj b/builds/msvc/vs2012/libsodium/libsodium.vcxproj index 56ec15fa..57368b0d 100644 --- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj @@ -161,10 +161,12 @@ + + @@ -342,16 +344,20 @@ + + + + diff --git a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2012/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj b/builds/msvc/vs2013/libsodium/libsodium.vcxproj index 2787fb75..c85ed549 100644 --- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj @@ -161,10 +161,12 @@ + + @@ -342,16 +344,20 @@ + + + + diff --git a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2013/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj b/builds/msvc/vs2015/libsodium/libsodium.vcxproj index 1c2502cd..6a4efbe2 100644 --- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj @@ -161,10 +161,12 @@ + + @@ -342,16 +344,20 @@ + + + + diff --git a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2015/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj b/builds/msvc/vs2017/libsodium/libsodium.vcxproj index 7ddc2b18..6fecdf2a 100644 --- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj @@ -161,10 +161,12 @@ + + @@ -342,16 +344,20 @@ + + + + diff --git a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2017/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj b/builds/msvc/vs2019/libsodium/libsodium.vcxproj index d964cd8f..9c57d2dc 100644 --- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj @@ -185,10 +185,12 @@ + + @@ -366,16 +368,20 @@ + + + + diff --git a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2019/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj b/builds/msvc/vs2022/libsodium/libsodium.vcxproj index 8f4cf407..b6954440 100644 --- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj @@ -185,10 +185,12 @@ + + @@ -366,16 +368,20 @@ + + + + diff --git a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2022/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj b/builds/msvc/vs2026/libsodium/libsodium.vcxproj index 915f2094..12f7acf7 100644 --- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj +++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj @@ -185,10 +185,12 @@ + + @@ -366,16 +368,20 @@ + + + + diff --git a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters index fc4770d8..a0f80e26 100644 --- a/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters +++ b/builds/msvc/vs2026/libsodium/libsodium.vcxproj.filters @@ -303,6 +303,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -315,6 +318,9 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int @@ -842,6 +848,9 @@ crypto_stream\chacha20\ref + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -851,6 +860,9 @@ crypto_stream\chacha20\dolbeau + + crypto_stream\chacha20\dolbeau + crypto_stream\chacha20\dolbeau @@ -866,12 +878,18 @@ crypto_stream\salsa20\ref + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int crypto_stream\salsa20\xmm6int + + crypto_stream\salsa20\xmm6int + crypto_stream\salsa20\xmm6int diff --git a/ci/appveyor/libsodium.vcxproj b/ci/appveyor/libsodium.vcxproj index 347feeb7..1dbbae1d 100644 --- a/ci/appveyor/libsodium.vcxproj +++ b/ci/appveyor/libsodium.vcxproj @@ -409,10 +409,12 @@ + + @@ -590,16 +592,20 @@ + + + + diff --git a/ci/appveyor/libsodium.vcxproj.filters b/ci/appveyor/libsodium.vcxproj.filters index 469c8d9e..700b5c5f 100644 --- a/ci/appveyor/libsodium.vcxproj.filters +++ b/ci/appveyor/libsodium.vcxproj.filters @@ -294,6 +294,9 @@ Source Files + + Source Files + Source Files @@ -306,6 +309,9 @@ Source Files + + Source Files + Source Files @@ -833,6 +839,9 @@ Header Files + + Header Files + Header Files @@ -842,6 +851,9 @@ Header Files + + Header Files + Header Files @@ -857,12 +869,18 @@ Header Files + + Header Files + Header Files Header Files + + Header Files + Header Files diff --git a/src/libsodium/Makefile.am b/src/libsodium/Makefile.am index 4531a986..1e4b14f1 100644 --- a/src/libsodium/Makefile.am +++ b/src/libsodium/Makefile.am @@ -347,4 +347,10 @@ libavx512f_la_CPPFLAGS = $(libsodium_la_CPPFLAGS) \ @CFLAGS_SSE2@ @CFLAGS_SSSE3@ @CFLAGS_SSE41@ @CFLAGS_AVX@ @CFLAGS_AVX2@ @CFLAGS_AVX512F@ libavx512f_la_SOURCES = \ crypto_pwhash/argon2/argon2-fill-block-avx512f.c \ - crypto_pwhash/argon2/blamka-round-avx512f.h + crypto_pwhash/argon2/blamka-round-avx512f.h \ + crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.c \ + crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.h \ + crypto_stream/chacha20/dolbeau/u16.h \ + crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.c \ + crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.h \ + crypto_stream/salsa20/xmm6int/u16.h diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.c b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.c new file mode 100644 index 00000000..57053569 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.c @@ -0,0 +1,182 @@ + +#include +#include +#include + +#include "core.h" +#include "crypto_stream_chacha20.h" +#include "private/common.h" +#include "utils.h" + +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) + +# ifdef __clang__ +# pragma clang attribute push(__attribute__((target("sse2,ssse3,sse4.1,avx2,avx512f"))), apply_to = function) +# elif defined(__GNUC__) +# pragma GCC target("sse2,ssse3,sse4.1,avx2,avx512f") +# endif + +# include +# include +# include +# include +# include "private/sse2_64_32.h" + +# include "../stream_chacha20.h" +# include "chacha20_dolbeau-avx512.h" + +# define ROUNDS 20 + +typedef struct chacha_ctx { + uint32_t input[16]; +} chacha_ctx; + +static void +chacha_keysetup(chacha_ctx *ctx, const uint8_t *k) +{ + ctx->input[0] = 0x61707865; + ctx->input[1] = 0x3320646e; + ctx->input[2] = 0x79622d32; + ctx->input[3] = 0x6b206574; + ctx->input[4] = LOAD32_LE(k + 0); + ctx->input[5] = LOAD32_LE(k + 4); + ctx->input[6] = LOAD32_LE(k + 8); + ctx->input[7] = LOAD32_LE(k + 12); + ctx->input[8] = LOAD32_LE(k + 16); + ctx->input[9] = LOAD32_LE(k + 20); + ctx->input[10] = LOAD32_LE(k + 24); + ctx->input[11] = LOAD32_LE(k + 28); +} + +static void +chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter + 0); + ctx->input[13] = counter == NULL ? 0 : LOAD32_LE(counter + 4); + ctx->input[14] = LOAD32_LE(iv + 0); + ctx->input[15] = LOAD32_LE(iv + 4); +} + +static void +chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter); + ctx->input[13] = LOAD32_LE(iv + 0); + ctx->input[14] = LOAD32_LE(iv + 4); + ctx->input[15] = LOAD32_LE(iv + 8); +} + +static void +chacha20_encrypt_bytes(chacha_ctx *ctx, const uint8_t *m, uint8_t *c, + unsigned long long bytes) +{ + uint32_t * const x = &ctx->input[0]; + + if (!bytes) { + return; /* LCOV_EXCL_LINE */ + } +# include "u16.h" +# include "u8.h" +# include "u4.h" +# include "u1.h" +# include "u0.h" +} + +static int +stream_ref(unsigned char *c, unsigned long long clen, const unsigned char *n, + const unsigned char *k) +{ + struct chacha_ctx ctx; + + if (!clen) { + return 0; + } + COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); + chacha_keysetup(&ctx, k); + chacha_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + chacha20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_ietf_ext_ref(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) +{ + struct chacha_ctx ctx; + + if (!clen) { + return 0; + } + COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); + chacha_keysetup(&ctx, k); + chacha_ietf_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + chacha20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_ref_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, uint64_t ic, + const unsigned char *k) +{ + struct chacha_ctx ctx; + uint8_t ic_bytes[8]; + uint32_t ic_high; + uint32_t ic_low; + + if (!mlen) { + return 0; + } + ic_high = (uint32_t) (ic >> 32); + ic_low = (uint32_t) ic; + STORE32_LE(&ic_bytes[0], ic_low); + STORE32_LE(&ic_bytes[4], ic_high); + chacha_keysetup(&ctx, k); + chacha_ivsetup(&ctx, n, ic_bytes); + chacha20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_ietf_ext_ref_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + uint32_t ic, const unsigned char *k) +{ + struct chacha_ctx ctx; + uint8_t ic_bytes[4]; + + if (!mlen) { + return 0; + } + STORE32_LE(ic_bytes, ic); + chacha_keysetup(&ctx, k); + chacha_ietf_ivsetup(&ctx, n, ic_bytes); + chacha20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +struct crypto_stream_chacha20_implementation + crypto_stream_chacha20_dolbeau_avx512_implementation = { + SODIUM_C99(.stream =) stream_ref, + SODIUM_C99(.stream_ietf_ext =) stream_ietf_ext_ref, + SODIUM_C99(.stream_xor_ic =) stream_ref_xor_ic, + SODIUM_C99(.stream_ietf_ext_xor_ic =) stream_ietf_ext_ref_xor_ic + }; + +# ifdef __clang__ +# pragma clang attribute pop +# endif + +#endif diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.h b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.h new file mode 100644 index 00000000..b8a7fc27 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx512.h @@ -0,0 +1,8 @@ + +#include + +#include "../stream_chacha20.h" +#include "crypto_stream_chacha20.h" + +extern struct crypto_stream_chacha20_implementation + crypto_stream_chacha20_dolbeau_avx512_implementation; diff --git a/src/libsodium/crypto_stream/chacha20/dolbeau/u16.h b/src/libsodium/crypto_stream/chacha20/dolbeau/u16.h new file mode 100644 index 00000000..77a765b9 --- /dev/null +++ b/src/libsodium/crypto_stream/chacha20/dolbeau/u16.h @@ -0,0 +1,219 @@ + +#define VEC16_LINE1(A, B, C, D) \ + x_##A = _mm512_add_epi32(x_##A, x_##B); \ + x_##D = _mm512_rol_epi32(_mm512_xor_si512(x_##D, x_##A), 16) +#define VEC16_LINE2(A, B, C, D) \ + x_##C = _mm512_add_epi32(x_##C, x_##D); \ + x_##B = _mm512_rol_epi32(_mm512_xor_si512(x_##B, x_##C), 12) +#define VEC16_LINE3(A, B, C, D) \ + x_##A = _mm512_add_epi32(x_##A, x_##B); \ + x_##D = _mm512_rol_epi32(_mm512_xor_si512(x_##D, x_##A), 8) +#define VEC16_LINE4(A, B, C, D) \ + x_##C = _mm512_add_epi32(x_##C, x_##D); \ + x_##B = _mm512_rol_epi32(_mm512_xor_si512(x_##B, x_##C), 7) + +#define VEC16_ROUND(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ + D4) \ + VEC16_LINE1(A1, B1, C1, D1); \ + VEC16_LINE1(A2, B2, C2, D2); \ + VEC16_LINE1(A3, B3, C3, D3); \ + VEC16_LINE1(A4, B4, C4, D4); \ + VEC16_LINE2(A1, B1, C1, D1); \ + VEC16_LINE2(A2, B2, C2, D2); \ + VEC16_LINE2(A3, B3, C3, D3); \ + VEC16_LINE2(A4, B4, C4, D4); \ + VEC16_LINE3(A1, B1, C1, D1); \ + VEC16_LINE3(A2, B2, C2, D2); \ + VEC16_LINE3(A3, B3, C3, D3); \ + VEC16_LINE3(A4, B4, C4, D4); \ + VEC16_LINE4(A1, B1, C1, D1); \ + VEC16_LINE4(A2, B2, C2, D2); \ + VEC16_LINE4(A3, B3, C3, D3); \ + VEC16_LINE4(A4, B4, C4, D4) + +/* store 4 blocks whose 16 words are spread one-per-lane across the 4 source + * registers; a 4x4 transpose of 128-bit lanes finishes the 16x16 transpose that + * the two unpack stages below start. */ +#define ONEQUAD_TRANSPOSE_STORE(A, B, C, D) \ + { \ + __m512i u0 = _mm512_shuffle_i32x4(x_##A, x_##B, 0x88); \ + __m512i u1 = _mm512_shuffle_i32x4(x_##A, x_##B, 0xdd); \ + __m512i u2 = _mm512_shuffle_i32x4(x_##C, x_##D, 0x88); \ + __m512i u3 = _mm512_shuffle_i32x4(x_##C, x_##D, 0xdd); \ + __m512i b0 = _mm512_shuffle_i32x4(u0, u2, 0x88); \ + __m512i b1 = _mm512_shuffle_i32x4(u1, u3, 0x88); \ + __m512i b2 = _mm512_shuffle_i32x4(u0, u2, 0xdd); \ + __m512i b3 = _mm512_shuffle_i32x4(u1, u3, 0xdd); \ + b0 = _mm512_xor_si512( \ + b0, _mm512_loadu_si512((const void *) (m + 64 * (A)))); \ + b1 = _mm512_xor_si512( \ + b1, _mm512_loadu_si512((const void *) (m + 64 * (B)))); \ + b2 = _mm512_xor_si512( \ + b2, _mm512_loadu_si512((const void *) (m + 64 * (C)))); \ + b3 = _mm512_xor_si512( \ + b3, _mm512_loadu_si512((const void *) (m + 64 * (D)))); \ + _mm512_storeu_si512((void *) (c + 64 * (A)), b0); \ + _mm512_storeu_si512((void *) (c + 64 * (B)), b1); \ + _mm512_storeu_si512((void *) (c + 64 * (C)), b2); \ + _mm512_storeu_si512((void *) (c + 64 * (D)), b3); \ + } + +if (bytes >= 1024) { + const __m512i addv = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15); + uint32_t in12, in13; + + __m512i x_0 = _mm512_set1_epi32(x[0]); + __m512i x_1 = _mm512_set1_epi32(x[1]); + __m512i x_2 = _mm512_set1_epi32(x[2]); + __m512i x_3 = _mm512_set1_epi32(x[3]); + __m512i x_4 = _mm512_set1_epi32(x[4]); + __m512i x_5 = _mm512_set1_epi32(x[5]); + __m512i x_6 = _mm512_set1_epi32(x[6]); + __m512i x_7 = _mm512_set1_epi32(x[7]); + __m512i x_8 = _mm512_set1_epi32(x[8]); + __m512i x_9 = _mm512_set1_epi32(x[9]); + __m512i x_10 = _mm512_set1_epi32(x[10]); + __m512i x_11 = _mm512_set1_epi32(x[11]); + __m512i x_12; + __m512i x_13; + __m512i x_14 = _mm512_set1_epi32(x[14]); + __m512i x_15 = _mm512_set1_epi32(x[15]); + + __m512i orig0 = x_0; + __m512i orig1 = x_1; + __m512i orig2 = x_2; + __m512i orig3 = x_3; + __m512i orig4 = x_4; + __m512i orig5 = x_5; + __m512i orig6 = x_6; + __m512i orig7 = x_7; + __m512i orig8 = x_8; + __m512i orig9 = x_9; + __m512i orig10 = x_10; + __m512i orig11 = x_11; + __m512i orig12; + __m512i orig13; + __m512i orig14 = x_14; + __m512i orig15 = x_15; + __m512i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, + t_13, t_14, t_15; + + while (bytes >= 1024) { + __m512i base12, base13; + __mmask16 carry; + uint64_t in1213; + int i; + + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + x_4 = orig4; + x_5 = orig5; + x_6 = orig6; + x_7 = orig7; + x_8 = orig8; + x_9 = orig9; + x_10 = orig10; + x_11 = orig11; + x_14 = orig14; + x_15 = orig15; + + in12 = x[12]; + in13 = x[13]; + + /* per-block 64-bit counter: lane j holds (in12|in13<<32) + j, split back + * into low/high words. carry propagates only where the low add wraps. */ + base12 = _mm512_set1_epi32(in12); + base13 = _mm512_set1_epi32(in13); + x_12 = _mm512_add_epi32(base12, addv); + carry = _mm512_cmplt_epu32_mask(x_12, base12); + x_13 = _mm512_mask_add_epi32(base13, carry, base13, + _mm512_set1_epi32(1)); + + orig12 = x_12; + orig13 = x_13; + + in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32); + in1213 += 16; + + x[12] = in1213 & 0xFFFFFFFF; + x[13] = (in1213 >> 32) & 0xFFFFFFFF; + + for (i = 0; i < ROUNDS; i += 2) { + VEC16_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + VEC16_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); + } + + x_0 = _mm512_add_epi32(x_0, orig0); + x_1 = _mm512_add_epi32(x_1, orig1); + x_2 = _mm512_add_epi32(x_2, orig2); + x_3 = _mm512_add_epi32(x_3, orig3); + x_4 = _mm512_add_epi32(x_4, orig4); + x_5 = _mm512_add_epi32(x_5, orig5); + x_6 = _mm512_add_epi32(x_6, orig6); + x_7 = _mm512_add_epi32(x_7, orig7); + x_8 = _mm512_add_epi32(x_8, orig8); + x_9 = _mm512_add_epi32(x_9, orig9); + x_10 = _mm512_add_epi32(x_10, orig10); + x_11 = _mm512_add_epi32(x_11, orig11); + x_12 = _mm512_add_epi32(x_12, orig12); + x_13 = _mm512_add_epi32(x_13, orig13); + x_14 = _mm512_add_epi32(x_14, orig14); + x_15 = _mm512_add_epi32(x_15, orig15); + + /* 16x16 transpose, stage 1: interleave 32-bit words of adjacent rows */ + t_0 = _mm512_unpacklo_epi32(x_0, x_1); + t_1 = _mm512_unpackhi_epi32(x_0, x_1); + t_2 = _mm512_unpacklo_epi32(x_2, x_3); + t_3 = _mm512_unpackhi_epi32(x_2, x_3); + t_4 = _mm512_unpacklo_epi32(x_4, x_5); + t_5 = _mm512_unpackhi_epi32(x_4, x_5); + t_6 = _mm512_unpacklo_epi32(x_6, x_7); + t_7 = _mm512_unpackhi_epi32(x_6, x_7); + t_8 = _mm512_unpacklo_epi32(x_8, x_9); + t_9 = _mm512_unpackhi_epi32(x_8, x_9); + t_10 = _mm512_unpacklo_epi32(x_10, x_11); + t_11 = _mm512_unpackhi_epi32(x_10, x_11); + t_12 = _mm512_unpacklo_epi32(x_12, x_13); + t_13 = _mm512_unpackhi_epi32(x_12, x_13); + t_14 = _mm512_unpacklo_epi32(x_14, x_15); + t_15 = _mm512_unpackhi_epi32(x_14, x_15); + + /* stage 2: interleave 64-bit words */ + x_0 = _mm512_unpacklo_epi64(t_0, t_2); + x_1 = _mm512_unpackhi_epi64(t_0, t_2); + x_2 = _mm512_unpacklo_epi64(t_1, t_3); + x_3 = _mm512_unpackhi_epi64(t_1, t_3); + x_4 = _mm512_unpacklo_epi64(t_4, t_6); + x_5 = _mm512_unpackhi_epi64(t_4, t_6); + x_6 = _mm512_unpacklo_epi64(t_5, t_7); + x_7 = _mm512_unpackhi_epi64(t_5, t_7); + x_8 = _mm512_unpacklo_epi64(t_8, t_10); + x_9 = _mm512_unpackhi_epi64(t_8, t_10); + x_10 = _mm512_unpacklo_epi64(t_9, t_11); + x_11 = _mm512_unpackhi_epi64(t_9, t_11); + x_12 = _mm512_unpacklo_epi64(t_12, t_14); + x_13 = _mm512_unpackhi_epi64(t_12, t_14); + x_14 = _mm512_unpacklo_epi64(t_13, t_15); + x_15 = _mm512_unpackhi_epi64(t_13, t_15); + + /* stage 3/4: shuffle 128-bit lanes, xor with the message, store */ + ONEQUAD_TRANSPOSE_STORE(0, 4, 8, 12); + ONEQUAD_TRANSPOSE_STORE(1, 5, 9, 13); + ONEQUAD_TRANSPOSE_STORE(2, 6, 10, 14); + ONEQUAD_TRANSPOSE_STORE(3, 7, 11, 15); + + bytes -= 1024; + c += 1024; + m += 1024; + } +} + +#undef VEC16_LINE1 +#undef VEC16_LINE2 +#undef VEC16_LINE3 +#undef VEC16_LINE4 +#undef VEC16_ROUND +#undef ONEQUAD_TRANSPOSE_STORE diff --git a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c index adcab5bf..8bd46d2b 100644 --- a/src/libsodium/crypto_stream/chacha20/stream_chacha20.c +++ b/src/libsodium/crypto_stream/chacha20/stream_chacha20.c @@ -8,6 +8,11 @@ #include "stream_chacha20.h" #include "ref/chacha20_ref.h" +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) +# include "dolbeau/chacha20_dolbeau-avx512.h" +#endif #if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) # include "dolbeau/chacha20_dolbeau-avx2.h" @@ -167,6 +172,14 @@ int _crypto_stream_chacha20_pick_best_implementation(void) { implementation = &crypto_stream_chacha20_ref_implementation; +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) + if (sodium_runtime_has_avx512f()) { + implementation = &crypto_stream_chacha20_dolbeau_avx512_implementation; + return 0; + } +#endif #if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) if (sodium_runtime_has_avx2()) { diff --git a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c index 45298501..2979a55d 100644 --- a/src/libsodium/crypto_stream/salsa20/stream_salsa20.c +++ b/src/libsodium/crypto_stream/salsa20/stream_salsa20.c @@ -17,6 +17,11 @@ defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) # include "xmm6int/salsa20_xmm6int-avx2.h" #endif +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) +# include "xmm6int/salsa20_xmm6int-avx512.h" +#endif #if HAVE_AMD64_ASM static const crypto_stream_salsa20_implementation *implementation = @@ -83,6 +88,14 @@ _crypto_stream_salsa20_pick_best_implementation(void) implementation = &crypto_stream_salsa20_ref_implementation; #endif +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) + if (sodium_runtime_has_avx512f()) { + implementation = &crypto_stream_salsa20_xmm6int_avx512_implementation; + return 0; + } +#endif #if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) if (sodium_runtime_has_avx2()) { diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.c b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.c new file mode 100644 index 00000000..e41e8d1e --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.c @@ -0,0 +1,136 @@ + +#include +#include +#include + +#include "crypto_stream_salsa20.h" +#include "private/common.h" +#include "utils.h" + +#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ + defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ + defined(HAVE_SMMINTRIN_H) + +# ifdef __clang__ +# pragma clang attribute push(__attribute__((target("sse2,ssse3,sse4.1,avx2,avx512f"))), apply_to = function) +# elif defined(__GNUC__) +# pragma GCC target("sse2,ssse3,sse4.1,avx2,avx512f") +# endif + +# include +# include +# include +# include +# include "private/sse2_64_32.h" + +# include "../stream_salsa20.h" +# include "salsa20_xmm6int-avx512.h" + +# define ROUNDS 20 + +typedef struct salsa_ctx { + uint32_t input[16]; +} salsa_ctx; + +static const int TR[16] = { + 0, 5, 10, 15, 12, 1, 6, 11, 8, 13, 2, 7, 4, 9, 14, 3 +}; + +static void +salsa_keysetup(salsa_ctx *ctx, const uint8_t *k) +{ + ctx->input[TR[1]] = LOAD32_LE(k + 0); + ctx->input[TR[2]] = LOAD32_LE(k + 4); + ctx->input[TR[3]] = LOAD32_LE(k + 8); + ctx->input[TR[4]] = LOAD32_LE(k + 12); + ctx->input[TR[11]] = LOAD32_LE(k + 16); + ctx->input[TR[12]] = LOAD32_LE(k + 20); + ctx->input[TR[13]] = LOAD32_LE(k + 24); + ctx->input[TR[14]] = LOAD32_LE(k + 28); + ctx->input[TR[0]] = 0x61707865; + ctx->input[TR[5]] = 0x3320646e; + ctx->input[TR[10]] = 0x79622d32; + ctx->input[TR[15]] = 0x6b206574; +} + +static void +salsa_ivsetup(salsa_ctx *ctx, const uint8_t *iv, const uint8_t *counter) +{ + ctx->input[TR[6]] = LOAD32_LE(iv + 0); + ctx->input[TR[7]] = LOAD32_LE(iv + 4); + ctx->input[TR[8]] = counter == NULL ? 0 : LOAD32_LE(counter + 0); + ctx->input[TR[9]] = counter == NULL ? 0 : LOAD32_LE(counter + 4); +} + +static void +salsa20_encrypt_bytes(salsa_ctx *ctx, const uint8_t *m, uint8_t *c, + unsigned long long bytes) +{ + uint32_t * const x = &ctx->input[0]; + + if (!bytes) { + return; /* LCOV_EXCL_LINE */ + } + +#include "u16.h" +#include "u8.h" +#include "u4.h" +#include "u1.h" +#include "u0.h" +} + +static int +stream_avx512(unsigned char *c, unsigned long long clen, const unsigned char *n, + const unsigned char *k) +{ + struct salsa_ctx ctx; + + if (!clen) { + return 0; + } + COMPILER_ASSERT(crypto_stream_salsa20_KEYBYTES == 256 / 8); + salsa_keysetup(&ctx, k); + salsa_ivsetup(&ctx, n, NULL); + memset(c, 0, clen); + salsa20_encrypt_bytes(&ctx, c, c, clen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +static int +stream_avx512_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + uint64_t ic, const unsigned char *k) +{ + struct salsa_ctx ctx; + uint8_t ic_bytes[8]; + uint32_t ic_high; + uint32_t ic_low; + + if (!mlen) { + return 0; + } + ic_high = (uint32_t) (ic >> 32); + ic_low = (uint32_t) ic; + STORE32_LE(&ic_bytes[0], ic_low); + STORE32_LE(&ic_bytes[4], ic_high); + salsa_keysetup(&ctx, k); + salsa_ivsetup(&ctx, n, ic_bytes); + salsa20_encrypt_bytes(&ctx, m, c, mlen); + sodium_memzero(&ctx, sizeof ctx); + + return 0; +} + +struct crypto_stream_salsa20_implementation + crypto_stream_salsa20_xmm6int_avx512_implementation = { + SODIUM_C99(.stream =) stream_avx512, + SODIUM_C99(.stream_xor_ic =) stream_avx512_xor_ic + }; + +#ifdef __clang__ +# pragma clang attribute pop +#endif + +#endif diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.h b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.h new file mode 100644 index 00000000..3a4b02d2 --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx512.h @@ -0,0 +1,8 @@ + +#include + +#include "../stream_salsa20.h" +#include "crypto_stream_salsa20.h" + +extern struct crypto_stream_salsa20_implementation + crypto_stream_salsa20_xmm6int_avx512_implementation; diff --git a/src/libsodium/crypto_stream/salsa20/xmm6int/u16.h b/src/libsodium/crypto_stream/salsa20/xmm6int/u16.h new file mode 100644 index 00000000..d791332b --- /dev/null +++ b/src/libsodium/crypto_stream/salsa20/xmm6int/u16.h @@ -0,0 +1,241 @@ +#define ONEQUAD_TRANSPOSE_STORE(A, B, C, D) \ + { \ + __m512i u0 = _mm512_shuffle_i32x4(z##A, z##B, 0x88); \ + __m512i u1 = _mm512_shuffle_i32x4(z##A, z##B, 0xdd); \ + __m512i u2 = _mm512_shuffle_i32x4(z##C, z##D, 0x88); \ + __m512i u3 = _mm512_shuffle_i32x4(z##C, z##D, 0xdd); \ + __m512i b0 = _mm512_shuffle_i32x4(u0, u2, 0x88); \ + __m512i b1 = _mm512_shuffle_i32x4(u1, u3, 0x88); \ + __m512i b2 = _mm512_shuffle_i32x4(u0, u2, 0xdd); \ + __m512i b3 = _mm512_shuffle_i32x4(u1, u3, 0xdd); \ + b0 = _mm512_xor_si512( \ + b0, _mm512_loadu_si512((const void *) (m + 64 * (A)))); \ + b1 = _mm512_xor_si512( \ + b1, _mm512_loadu_si512((const void *) (m + 64 * (B)))); \ + b2 = _mm512_xor_si512( \ + b2, _mm512_loadu_si512((const void *) (m + 64 * (C)))); \ + b3 = _mm512_xor_si512( \ + b3, _mm512_loadu_si512((const void *) (m + 64 * (D)))); \ + _mm512_storeu_si512((void *) (c + 64 * (A)), b0); \ + _mm512_storeu_si512((void *) (c + 64 * (B)), b1); \ + _mm512_storeu_si512((void *) (c + 64 * (C)), b2); \ + _mm512_storeu_si512((void *) (c + 64 * (D)), b3); \ + } + +if (bytes >= 1024) { + const __m512i addv = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15); + __m512i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15; + + __m512i z0 = _mm512_set1_epi32(x[0]); + __m512i z5 = _mm512_set1_epi32(x[1]); + __m512i z10 = _mm512_set1_epi32(x[2]); + __m512i z15 = _mm512_set1_epi32(x[3]); + __m512i z12 = _mm512_set1_epi32(x[4]); + __m512i z1 = _mm512_set1_epi32(x[5]); + __m512i z6 = _mm512_set1_epi32(x[6]); + __m512i z11 = _mm512_set1_epi32(x[7]); + __m512i z8; + __m512i z13 = _mm512_set1_epi32(x[9]); + __m512i z2 = _mm512_set1_epi32(x[10]); + __m512i z7 = _mm512_set1_epi32(x[11]); + __m512i z4 = _mm512_set1_epi32(x[12]); + __m512i z9; + __m512i z14 = _mm512_set1_epi32(x[14]); + __m512i z3 = _mm512_set1_epi32(x[15]); + + __m512i orig0 = z0; + __m512i orig1 = z1; + __m512i orig2 = z2; + __m512i orig3 = z3; + __m512i orig4 = z4; + __m512i orig5 = z5; + __m512i orig6 = z6; + __m512i orig7 = z7; + __m512i orig8; + __m512i orig9; + __m512i orig10 = z10; + __m512i orig11 = z11; + __m512i orig12 = z12; + __m512i orig13 = z13; + __m512i orig14 = z14; + __m512i orig15 = z15; + + uint32_t in8; + uint32_t in9; + int i; + + while (bytes >= 1024) { + __m512i base8, base9; + __mmask16 carry; + uint64_t in89; + + /* per-block 64-bit counter (matrix words 8 and 9, stored at x[8]/x[13]): + * lane j holds base + j split into low/high; the high word takes a carry + * only where the low add wraps. */ + in8 = x[8]; + in9 = x[13]; + base8 = _mm512_set1_epi32(in8); + base9 = _mm512_set1_epi32(in9); + z8 = _mm512_add_epi32(base8, addv); + carry = _mm512_cmplt_epu32_mask(z8, base8); + z9 = _mm512_mask_add_epi32(base9, carry, base9, + _mm512_set1_epi32(1)); + + orig8 = z8; + orig9 = z9; + + in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32); + in89 += 16; + + x[8] = in89 & 0xFFFFFFFF; + x[13] = (in89 >> 32) & 0xFFFFFFFF; + + z0 = orig0; + z1 = orig1; + z2 = orig2; + z3 = orig3; + z4 = orig4; + z5 = orig5; + z6 = orig6; + z7 = orig7; + z10 = orig10; + z11 = orig11; + z12 = orig12; + z13 = orig13; + z14 = orig14; + z15 = orig15; + + for (i = 0; i < ROUNDS; i += 2) { + z4 = _mm512_xor_si512( + z4, _mm512_rol_epi32(_mm512_add_epi32(z12, z0), 7)); + z9 = _mm512_xor_si512( + z9, _mm512_rol_epi32(_mm512_add_epi32(z1, z5), 7)); + z8 = _mm512_xor_si512( + z8, _mm512_rol_epi32(_mm512_add_epi32(z0, z4), 9)); + z13 = _mm512_xor_si512( + z13, _mm512_rol_epi32(_mm512_add_epi32(z5, z9), 9)); + z12 = _mm512_xor_si512( + z12, _mm512_rol_epi32(_mm512_add_epi32(z4, z8), 13)); + z1 = _mm512_xor_si512( + z1, _mm512_rol_epi32(_mm512_add_epi32(z9, z13), 13)); + z0 = _mm512_xor_si512( + z0, _mm512_rol_epi32(_mm512_add_epi32(z8, z12), 18)); + z5 = _mm512_xor_si512( + z5, _mm512_rol_epi32(_mm512_add_epi32(z13, z1), 18)); + z14 = _mm512_xor_si512( + z14, _mm512_rol_epi32(_mm512_add_epi32(z6, z10), 7)); + z3 = _mm512_xor_si512( + z3, _mm512_rol_epi32(_mm512_add_epi32(z11, z15), 7)); + z2 = _mm512_xor_si512( + z2, _mm512_rol_epi32(_mm512_add_epi32(z10, z14), 9)); + z7 = _mm512_xor_si512( + z7, _mm512_rol_epi32(_mm512_add_epi32(z15, z3), 9)); + z6 = _mm512_xor_si512( + z6, _mm512_rol_epi32(_mm512_add_epi32(z14, z2), 13)); + z11 = _mm512_xor_si512( + z11, _mm512_rol_epi32(_mm512_add_epi32(z3, z7), 13)); + z10 = _mm512_xor_si512( + z10, _mm512_rol_epi32(_mm512_add_epi32(z2, z6), 18)); + z1 = _mm512_xor_si512( + z1, _mm512_rol_epi32(_mm512_add_epi32(z3, z0), 7)); + z15 = _mm512_xor_si512( + z15, _mm512_rol_epi32(_mm512_add_epi32(z7, z11), 18)); + z6 = _mm512_xor_si512( + z6, _mm512_rol_epi32(_mm512_add_epi32(z4, z5), 7)); + z2 = _mm512_xor_si512( + z2, _mm512_rol_epi32(_mm512_add_epi32(z0, z1), 9)); + z7 = _mm512_xor_si512( + z7, _mm512_rol_epi32(_mm512_add_epi32(z5, z6), 9)); + z3 = _mm512_xor_si512( + z3, _mm512_rol_epi32(_mm512_add_epi32(z1, z2), 13)); + z4 = _mm512_xor_si512( + z4, _mm512_rol_epi32(_mm512_add_epi32(z6, z7), 13)); + z0 = _mm512_xor_si512( + z0, _mm512_rol_epi32(_mm512_add_epi32(z2, z3), 18)); + z5 = _mm512_xor_si512( + z5, _mm512_rol_epi32(_mm512_add_epi32(z7, z4), 18)); + z11 = _mm512_xor_si512( + z11, _mm512_rol_epi32(_mm512_add_epi32(z9, z10), 7)); + z12 = _mm512_xor_si512( + z12, _mm512_rol_epi32(_mm512_add_epi32(z14, z15), 7)); + z8 = _mm512_xor_si512( + z8, _mm512_rol_epi32(_mm512_add_epi32(z10, z11), 9)); + z13 = _mm512_xor_si512( + z13, _mm512_rol_epi32(_mm512_add_epi32(z15, z12), 9)); + z9 = _mm512_xor_si512( + z9, _mm512_rol_epi32(_mm512_add_epi32(z11, z8), 13)); + z14 = _mm512_xor_si512( + z14, _mm512_rol_epi32(_mm512_add_epi32(z12, z13), 13)); + z10 = _mm512_xor_si512( + z10, _mm512_rol_epi32(_mm512_add_epi32(z8, z9), 18)); + z15 = _mm512_xor_si512( + z15, _mm512_rol_epi32(_mm512_add_epi32(z13, z14), 18)); + } + + z0 = _mm512_add_epi32(z0, orig0); + z1 = _mm512_add_epi32(z1, orig1); + z2 = _mm512_add_epi32(z2, orig2); + z3 = _mm512_add_epi32(z3, orig3); + z4 = _mm512_add_epi32(z4, orig4); + z5 = _mm512_add_epi32(z5, orig5); + z6 = _mm512_add_epi32(z6, orig6); + z7 = _mm512_add_epi32(z7, orig7); + z8 = _mm512_add_epi32(z8, orig8); + z9 = _mm512_add_epi32(z9, orig9); + z10 = _mm512_add_epi32(z10, orig10); + z11 = _mm512_add_epi32(z11, orig11); + z12 = _mm512_add_epi32(z12, orig12); + z13 = _mm512_add_epi32(z13, orig13); + z14 = _mm512_add_epi32(z14, orig14); + z15 = _mm512_add_epi32(z15, orig15); + + /* 16x16 transpose, stage 1: interleave 32-bit words of adjacent rows */ + y0 = _mm512_unpacklo_epi32(z0, z1); + y1 = _mm512_unpackhi_epi32(z0, z1); + y2 = _mm512_unpacklo_epi32(z2, z3); + y3 = _mm512_unpackhi_epi32(z2, z3); + y4 = _mm512_unpacklo_epi32(z4, z5); + y5 = _mm512_unpackhi_epi32(z4, z5); + y6 = _mm512_unpacklo_epi32(z6, z7); + y7 = _mm512_unpackhi_epi32(z6, z7); + y8 = _mm512_unpacklo_epi32(z8, z9); + y9 = _mm512_unpackhi_epi32(z8, z9); + y10 = _mm512_unpacklo_epi32(z10, z11); + y11 = _mm512_unpackhi_epi32(z10, z11); + y12 = _mm512_unpacklo_epi32(z12, z13); + y13 = _mm512_unpackhi_epi32(z12, z13); + y14 = _mm512_unpacklo_epi32(z14, z15); + y15 = _mm512_unpackhi_epi32(z14, z15); + + /* stage 2: interleave 64-bit words */ + z0 = _mm512_unpacklo_epi64(y0, y2); + z1 = _mm512_unpackhi_epi64(y0, y2); + z2 = _mm512_unpacklo_epi64(y1, y3); + z3 = _mm512_unpackhi_epi64(y1, y3); + z4 = _mm512_unpacklo_epi64(y4, y6); + z5 = _mm512_unpackhi_epi64(y4, y6); + z6 = _mm512_unpacklo_epi64(y5, y7); + z7 = _mm512_unpackhi_epi64(y5, y7); + z8 = _mm512_unpacklo_epi64(y8, y10); + z9 = _mm512_unpackhi_epi64(y8, y10); + z10 = _mm512_unpacklo_epi64(y9, y11); + z11 = _mm512_unpackhi_epi64(y9, y11); + z12 = _mm512_unpacklo_epi64(y12, y14); + z13 = _mm512_unpackhi_epi64(y12, y14); + z14 = _mm512_unpacklo_epi64(y13, y15); + z15 = _mm512_unpackhi_epi64(y13, y15); + + /* stage 3/4: shuffle 128-bit lanes, xor with the message, store */ + ONEQUAD_TRANSPOSE_STORE(0, 4, 8, 12); + ONEQUAD_TRANSPOSE_STORE(1, 5, 9, 13); + ONEQUAD_TRANSPOSE_STORE(2, 6, 10, 14); + ONEQUAD_TRANSPOSE_STORE(3, 7, 11, 15); + + bytes -= 1024; + c += 1024; + m += 1024; + } +} + +#undef ONEQUAD_TRANSPOSE_STORE