From d4bc6fb5fe33188b88dbf5a531097638cb35c5d5 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Wed, 8 Apr 2026 23:59:53 +0200 Subject: [PATCH] Unify AES key expansion code on ARM --- .../armcrypto/aead_aes256gcm_armcrypto.c | 83 ++++++------------- 1 file changed, 26 insertions(+), 57 deletions(-) diff --git a/src/libsodium/crypto_aead/aes256gcm/armcrypto/aead_aes256gcm_armcrypto.c b/src/libsodium/crypto_aead/aes256gcm/armcrypto/aead_aes256gcm_armcrypto.c index 461c4393..ac73165e 100644 --- a/src/libsodium/crypto_aead/aes256gcm/armcrypto/aead_aes256gcm_armcrypto.c +++ b/src/libsodium/crypto_aead/aes256gcm/armcrypto/aead_aes256gcm_armcrypto.c @@ -67,9 +67,7 @@ typedef uint64x2_t BlockVec; #define CLMULHI128(a, b) \ vreinterpretq_u64_p128(vmull_high_p64(vreinterpretq_p64_u64(a), vreinterpretq_p64_u64(b))) -#ifdef _MSC_VER - -static __forceinline uint64x2_t +static inline uint64x2_t REV128_(uint64x2_t x) { uint8x16_t t = vrev64q_u8(vreinterpretq_u8_u64(x)); @@ -77,69 +75,40 @@ REV128_(uint64x2_t x) } #define REV128(x) REV128_(x) -static __forceinline uint64x2_t -SHUFFLE32x4_(uint64x2_t x, const int a, const int b, const int c, const int d) -{ - uint8_t idx[16]; - - idx[0] = (uint8_t) (a * 4); idx[1] = (uint8_t) (a * 4 + 1); - idx[2] = (uint8_t) (a * 4 + 2); idx[3] = (uint8_t) (a * 4 + 3); - idx[4] = (uint8_t) (b * 4); idx[5] = (uint8_t) (b * 4 + 1); - idx[6] = (uint8_t) (b * 4 + 2); idx[7] = (uint8_t) (b * 4 + 3); - idx[8] = (uint8_t) (c * 4); idx[9] = (uint8_t) (c * 4 + 1); - idx[10] = (uint8_t) (c * 4 + 2); idx[11] = (uint8_t) (c * 4 + 3); - idx[12] = (uint8_t) (d * 4); idx[13] = (uint8_t) (d * 4 + 1); - idx[14] = (uint8_t) (d * 4 + 2); idx[15] = (uint8_t) (d * 4 + 3); - return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(x), vld1q_u8(idx))); -} -#define SHUFFLE32x4(x, a, b, c, d) SHUFFLE32x4_((x), (a), (b), (c), (d)) +#define SHUFFLE32x4_2222(x) \ + vreinterpretq_u64_u32(vdupq_laneq_u32(vreinterpretq_u32_u64(x), 2)) +#define SHUFFLE32x4_3333(x) \ + vreinterpretq_u64_u32(vdupq_laneq_u32(vreinterpretq_u32_u64(x), 3)) +#define SWAP64HALVES(x) \ + vreinterpretq_u64_u32(vextq_u32(vreinterpretq_u32_u64(x), \ + vreinterpretq_u32_u64(x), 2)) #define CLMULLO128(a, b) \ - vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), vget_low_p64(vreinterpretq_p64_u64(b)))) + vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), \ + vget_low_p64(vreinterpretq_p64_u64(b)))) #define CLMULLOHI128(a, b) \ - vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), vget_high_p64(vreinterpretq_p64_u64(b)))) + vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), \ + vget_high_p64(vreinterpretq_p64_u64(b)))) #define CLMULHILO128(a, b) \ - vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(a)), vget_low_p64(vreinterpretq_p64_u64(b)))) - -#define PREFETCH_READ(x) -#define PREFETCH_WRITE(x) - -#else - -#define REV128(x) \ - vreinterpretq_u64_u8(__builtin_shufflevector(vreinterpretq_u8_u64(x), vreinterpretq_u8_u64(x), \ - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, \ - 1, 0)) -#define SHUFFLE32x4(x, a, b, c, d) \ - vreinterpretq_u64_u32(__builtin_shufflevector(vreinterpretq_u32_u64(x), \ - vreinterpretq_u32_u64(x), (a), (b), (c), (d))) - -#define CLMULLO128(a, b) \ - vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_low_u64(a), (poly64_t) vget_low_u64(b))) -#define CLMULLOHI128(a, b) \ - vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_low_u64(a), (poly64_t) vget_high_u64(b))) -#define CLMULHILO128(a, b) \ - vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_high_u64(a), (poly64_t) vget_low_u64(b))) + vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(a)), \ + vget_low_p64(vreinterpretq_p64_u64(b)))) +#if defined(__GNUC__) || defined(__clang__) #define PREFETCH_READ(x) __builtin_prefetch((x), 0, 2) -#define PREFETCH_WRITE(x) __builtin_prefetch((x), 1, 2); - +#define PREFETCH_WRITE(x) __builtin_prefetch((x), 1, 2) +#else +#define PREFETCH_READ(x) ((void) 0) +#define PREFETCH_WRITE(x) ((void) 0) #endif static inline BlockVec AES_KEYGEN(BlockVec block_vec, const int rc) { -#ifdef _MSC_VER - static const uint8_t keygen_shuf[16] = { + static const uint8_t aes_keygen_shuffle[16] = { 4, 1, 14, 11, 1, 14, 11, 4, 12, 9, 6, 3, 9, 6, 3, 12 }; uint8x16_t a = vaeseq_u8(vreinterpretq_u8_u64(block_vec), vdupq_n_u8(0)); - const uint8x16_t b = vqtbl1q_u8(a, vld1q_u8(keygen_shuf)); -#else - uint8x16_t a = vaeseq_u8(vreinterpretq_u8_u64(block_vec), vmovq_n_u8(0)); - const uint8x16_t b = - __builtin_shufflevector(a, a, 4, 1, 14, 11, 1, 14, 11, 4, 12, 9, 6, 3, 9, 6, 3, 12); -#endif + const uint8x16_t b = vqtbl1q_u8(a, vld1q_u8(aes_keygen_shuffle)); const uint64x2_t c = SET64x2((uint64_t) rc << 32, (uint64_t) rc << 32); return XOR128(vreinterpretq_u64_u8(b), c); } @@ -175,14 +144,14 @@ static void __vectorcall expand256(const unsigned char key[KEYBYTES], BlockVec r s = AES_KEYGEN(t2, RC); \ t1 = XOR128(t1, BYTESHL128(t1, 4)); \ t1 = XOR128(t1, BYTESHL128(t1, 8)); \ - t1 = XOR128(t1, SHUFFLE32x4(s, 3, 3, 3, 3)); + t1 = XOR128(t1, SHUFFLE32x4_3333(s)); #define EXPAND_KEY_2(RC) \ rkeys[i++] = t1; \ s = AES_KEYGEN(t1, RC); \ t2 = XOR128(t2, BYTESHL128(t2, 4)); \ t2 = XOR128(t2, BYTESHL128(t2, 8)); \ - t2 = XOR128(t2, SHUFFLE32x4(s, 2, 2, 2, 2)); + t2 = XOR128(t2, SHUFFLE32x4_2222(s)); t1 = LOAD128(&key[0]); t2 = LOAD128(&key[16]); @@ -320,9 +289,9 @@ static inline BlockVec __vectorcall gcm_reduce(const I256 x) const BlockVec p64 = SET64x2(0, 0xc200000000000000); const BlockVec a = CLMULLO128(lo, p64); - const BlockVec b = XOR128(SHUFFLE32x4(lo, 2, 3, 0, 1), a); + const BlockVec b = XOR128(SWAP64HALVES(lo), a); const BlockVec c = CLMULLO128(b, p64); - const BlockVec d = XOR128(SHUFFLE32x4(b, 2, 3, 0, 1), c); + const BlockVec d = XOR128(SWAP64HALVES(b), c); return XOR128(d, hi); } @@ -352,7 +321,7 @@ static void __vectorcall precomp_for_block_count(Precomp hx[PC_COUNT BlockVec h0_shifted; BlockVec h; - mask = SHUFFLE32x4(mask, 3, 3, 3, 3); + mask = SHUFFLE32x4_3333(mask); carry = AND128(carry, mask); h0_shifted = SHL128(h0, 1); h = XOR128(h0_shifted, carry);