QPQ-AG/libsodium
12
0
Fork 0
mirror of https://github.com/jedisct1/libsodium.git synced 2026-08-25 08:37:13 +09:00
Code Issues Packages Projects Releases Wiki Activity

Unify AES key expansion code on ARM

Browse Source
This commit is contained in:
Frank Denis
2026-04-08 23:59:53 +02:00
parent ad61cc6ace
commit 09cb475ee0
1 changed files with 26 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libsodium/crypto_aead/aes256gcm/armcrypto/aead_aes256gcm_armcrypto.c
+26 -57
View File
@@ -67,9 +67,7 @@ typedef uint64x2_t BlockVec;
#define CLMULHI128(a, b) \
vreinterpretq_u64_p128(vmull_high_p64(vreinterpretq_p64_u64(a), vreinterpretq_p64_u64(b)))
#ifdef _MSC_VER
static __forceinline uint64x2_t
static inline uint64x2_t
REV128_(uint64x2_t x)
{
uint8x16_t t = vrev64q_u8(vreinterpretq_u8_u64(x));
@@ -77,69 +75,40 @@ REV128_(uint64x2_t x)
}
#define REV128(x) REV128_(x)
static __forceinline uint64x2_t
SHUFFLE32x4_(uint64x2_t x, const int a, const int b, const int c, const int d)
{
uint8_t idx[16];
idx[0] = (uint8_t) (a * 4); idx[1] = (uint8_t) (a * 4 + 1);
idx[2] = (uint8_t) (a * 4 + 2); idx[3] = (uint8_t) (a * 4 + 3);
idx[4] = (uint8_t) (b * 4); idx[5] = (uint8_t) (b * 4 + 1);
idx[6] = (uint8_t) (b * 4 + 2); idx[7] = (uint8_t) (b * 4 + 3);
idx[8] = (uint8_t) (c * 4); idx[9] = (uint8_t) (c * 4 + 1);
idx[10] = (uint8_t) (c * 4 + 2); idx[11] = (uint8_t) (c * 4 + 3);
idx[12] = (uint8_t) (d * 4); idx[13] = (uint8_t) (d * 4 + 1);
idx[14] = (uint8_t) (d * 4 + 2); idx[15] = (uint8_t) (d * 4 + 3);
return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(x), vld1q_u8(idx)));
}
#define SHUFFLE32x4(x, a, b, c, d) SHUFFLE32x4_((x), (a), (b), (c), (d))
#define SHUFFLE32x4_2222(x) \
vreinterpretq_u64_u32(vdupq_laneq_u32(vreinterpretq_u32_u64(x), 2))
#define SHUFFLE32x4_3333(x) \
vreinterpretq_u64_u32(vdupq_laneq_u32(vreinterpretq_u32_u64(x), 3))
#define SWAP64HALVES(x) \
vreinterpretq_u64_u32(vextq_u32(vreinterpretq_u32_u64(x), \
vreinterpretq_u32_u64(x), 2))
#define CLMULLO128(a, b) \
vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), vget_low_p64(vreinterpretq_p64_u64(b))))
vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), \
vget_low_p64(vreinterpretq_p64_u64(b))))
#define CLMULLOHI128(a, b) \
vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), vget_high_p64(vreinterpretq_p64_u64(b))))
vreinterpretq_u64_p128(vmull_p64(vget_low_p64(vreinterpretq_p64_u64(a)), \
vget_high_p64(vreinterpretq_p64_u64(b))))
#define CLMULHILO128(a, b) \
vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(a)), vget_low_p64(vreinterpretq_p64_u64(b))))
#define PREFETCH_READ(x)
#define PREFETCH_WRITE(x)
#else
#define REV128(x) \
vreinterpretq_u64_u8(__builtin_shufflevector(vreinterpretq_u8_u64(x), vreinterpretq_u8_u64(x), \
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, \
1, 0))
#define SHUFFLE32x4(x, a, b, c, d) \
vreinterpretq_u64_u32(__builtin_shufflevector(vreinterpretq_u32_u64(x), \
vreinterpretq_u32_u64(x), (a), (b), (c), (d)))
#define CLMULLO128(a, b) \
vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_low_u64(a), (poly64_t) vget_low_u64(b)))
#define CLMULLOHI128(a, b) \
vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_low_u64(a), (poly64_t) vget_high_u64(b)))
#define CLMULHILO128(a, b) \
vreinterpretq_u64_p128(vmull_p64((poly64_t) vget_high_u64(a), (poly64_t) vget_low_u64(b)))
vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(a)), \
vget_low_p64(vreinterpretq_p64_u64(b))))
#if defined(__GNUC__) || defined(__clang__)
#define PREFETCH_READ(x) __builtin_prefetch((x), 0, 2)
#define PREFETCH_WRITE(x) __builtin_prefetch((x), 1, 2);
#define PREFETCH_WRITE(x) __builtin_prefetch((x), 1, 2)
#else
#define PREFETCH_READ(x) ((void) 0)
#define PREFETCH_WRITE(x) ((void) 0)
#endif
static inline BlockVec
AES_KEYGEN(BlockVec block_vec, const int rc)
{
#ifdef _MSC_VER
static const uint8_t keygen_shuf[16] = {
static const uint8_t aes_keygen_shuffle[16] = {
4, 1, 14, 11, 1, 14, 11, 4, 12, 9, 6, 3, 9, 6, 3, 12
};
uint8x16_t a = vaeseq_u8(vreinterpretq_u8_u64(block_vec), vdupq_n_u8(0));
const uint8x16_t b = vqtbl1q_u8(a, vld1q_u8(keygen_shuf));
#else
uint8x16_t a = vaeseq_u8(vreinterpretq_u8_u64(block_vec), vmovq_n_u8(0));
const uint8x16_t b =
__builtin_shufflevector(a, a, 4, 1, 14, 11, 1, 14, 11, 4, 12, 9, 6, 3, 9, 6, 3, 12);
#endif
const uint8x16_t b = vqtbl1q_u8(a, vld1q_u8(aes_keygen_shuffle));
const uint64x2_t c = SET64x2((uint64_t) rc << 32, (uint64_t) rc << 32);
return XOR128(vreinterpretq_u64_u8(b), c);
}
@@ -175,14 +144,14 @@ static void __vectorcall expand256(const unsigned char key[KEYBYTES], BlockVec r
s = AES_KEYGEN(t2, RC); \
t1 = XOR128(t1, BYTESHL128(t1, 4)); \
t1 = XOR128(t1, BYTESHL128(t1, 8)); \
t1 = XOR128(t1, SHUFFLE32x4(s, 3, 3, 3, 3));
t1 = XOR128(t1, SHUFFLE32x4_3333(s));
#define EXPAND_KEY_2(RC) \
rkeys[i++] = t1; \
s = AES_KEYGEN(t1, RC); \
t2 = XOR128(t2, BYTESHL128(t2, 4)); \
t2 = XOR128(t2, BYTESHL128(t2, 8)); \
t2 = XOR128(t2, SHUFFLE32x4(s, 2, 2, 2, 2));
t2 = XOR128(t2, SHUFFLE32x4_2222(s));
t1 = LOAD128(&key[0]);
t2 = LOAD128(&key[16]);
@@ -320,9 +289,9 @@ static inline BlockVec __vectorcall gcm_reduce(const I256 x)
const BlockVec p64 = SET64x2(0, 0xc200000000000000);
const BlockVec a = CLMULLO128(lo, p64);
const BlockVec b = XOR128(SHUFFLE32x4(lo, 2, 3, 0, 1), a);
const BlockVec b = XOR128(SWAP64HALVES(lo), a);
const BlockVec c = CLMULLO128(b, p64);
const BlockVec d = XOR128(SHUFFLE32x4(b, 2, 3, 0, 1), c);
const BlockVec d = XOR128(SWAP64HALVES(b), c);
return XOR128(d, hi);
}
@@ -352,7 +321,7 @@ static void __vectorcall precomp_for_block_count(Precomp hx[PC_COUNT
BlockVec h0_shifted;
BlockVec h;
mask = SHUFFLE32x4(mask, 3, 3, 3, 3);
mask = SHUFFLE32x4_3333(mask);
carry = AND128(carry, mask);
h0_shifted = SHL128(h0, 1);
h = XOR128(h0_shifted, carry);