QPQ-AG/libsodium
12
0
Fork 0
mirror of https://github.com/jedisct1/libsodium.git synced 2026-08-25 08:37:13 +09:00
Code Issues Packages Projects Releases Wiki Activity

Make the AEGIS code for ARM and Intel similar

Browse Source
This commit is contained in:
Frank Denis
2022-11-17 22:32:37 +01:00
parent af679cfb66
commit dd5c2bf23c
4 changed files with 336 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libsodium/crypto_aead/aegis128l/aesni/aead_aegis128l_aesni.c
+99 -85
View File
@@ -19,55 +19,68 @@
#if defined(HAVE_TMMINTRIN_H) && defined(HAVE_WMMINTRIN_H)
#ifdef __GNUC__
# pragma GCC target("ssse3")
# pragma GCC target("aes")
#pragma GCC target("ssse3")
#pragma GCC target("aes")
#endif
#include "private/sse2_64_32.h"
#include <tmmintrin.h>
#include <wmmintrin.h>
#include "private/sse2_64_32.h"
typedef __m128i aes_block_t;
#define AES_BLOCK_XOR(A, B) _mm_xor_si128((A), (B))
#define AES_BLOCK_AND(A, B) _mm_and_si128((A), (B))
#define AES_BLOCK_LOAD(A) _mm_loadu_si128((const aes_block_t *) (const void *) (A))
#define AES_BLOCK_LOAD_64x2(A, B) _mm_set_epi64x((A), (B))
#define AES_BLOCK_STORE(A, B) _mm_storeu_si128((aes_block_t *) (void *) (A), (B))
#define AES_ENC(A, B) _mm_aesenc_si128((A), (B))
static inline void
crypto_aead_aegis128l_update(__m128i *const state, const __m128i d1, const __m128i d2)
crypto_aead_aegis128l_update(aes_block_t *const state, const aes_block_t d1, const aes_block_t d2)
{
__m128i tmp;
aes_block_t tmp;
tmp = state[7];
state[7] = _mm_aesenc_si128(state[6], state[7]);
state[6] = _mm_aesenc_si128(state[5], state[6]);
state[5] = _mm_aesenc_si128(state[4], state[5]);
state[4] = _mm_aesenc_si128(state[3], state[4]);
state[3] = _mm_aesenc_si128(state[2], state[3]);
state[2] = _mm_aesenc_si128(state[1], state[2]);
state[1] = _mm_aesenc_si128(state[0], state[1]);
state[0] = _mm_aesenc_si128(tmp, state[0]);
state[7] = AES_ENC(state[6], state[7]);
state[6] = AES_ENC(state[5], state[6]);
state[5] = AES_ENC(state[4], state[5]);
state[4] = AES_ENC(state[3], state[4]);
state[3] = AES_ENC(state[2], state[3]);
state[2] = AES_ENC(state[1], state[2]);
state[1] = AES_ENC(state[0], state[1]);
state[0] = AES_ENC(tmp, state[0]);
state[0] = _mm_xor_si128(state[0], d1);
state[4] = _mm_xor_si128(state[4], d2);
state[0] = AES_BLOCK_XOR(state[0], d1);
state[4] = AES_BLOCK_XOR(state[4], d2);
}
static void
crypto_aead_aegis128l_init(const unsigned char *key, const unsigned char *nonce, __m128i *const state)
crypto_aead_aegis128l_init(const unsigned char *key, const unsigned char *nonce,
aes_block_t *const state)
{
const __m128i c0 = _mm_set_epi8(0xdd, 0x28, 0xb5, 0x73, 0x42, 0x31, 0x11, 0x20, 0xf1, 0x2f, 0xc2, 0x6d,
0x55, 0x18, 0x3d, 0xdb);
const __m128i c1 = _mm_set_epi8(0x62, 0x79, 0xe9, 0x90, 0x59, 0x37, 0x22, 0x15, 0x0d, 0x08, 0x05, 0x03,
0x02, 0x01, 0x01, 0x00);
__m128i k;
__m128i n;
int i;
static CRYPTO_ALIGN(16)
const uint8_t c0_[] = { 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1,
0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd };
static CRYPTO_ALIGN(16)
const uint8_t c1_[] = { 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d,
0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 };
const aes_block_t c0 = AES_BLOCK_LOAD(c0_);
const aes_block_t c1 = AES_BLOCK_LOAD(c1_);
aes_block_t k;
aes_block_t n;
int i;
k = _mm_loadu_si128((const __m128i *) (const void *) key);
n = _mm_loadu_si128((const __m128i *) (const void *) nonce);
k = AES_BLOCK_LOAD((const aes_block_t *) (const void *) key);
n = AES_BLOCK_LOAD((const aes_block_t *) (const void *) nonce);
state[0] = _mm_xor_si128(k, n);
state[0] = AES_BLOCK_XOR(k, n);
state[1] = c0;
state[2] = c1;
state[3] = c0;
state[4] = _mm_xor_si128(k, n);
state[5] = _mm_xor_si128(k, c1);
state[6] = _mm_xor_si128(k, c0);
state[7] = _mm_xor_si128(k, c1);
state[4] = AES_BLOCK_XOR(k, n);
state[5] = AES_BLOCK_XOR(k, c1);
state[6] = AES_BLOCK_XOR(k, c0);
state[7] = AES_BLOCK_XOR(k, c1);
for (i = 0; i < 10; i++) {
crypto_aead_aegis128l_update(state, n, k);
}
@@ -75,65 +88,65 @@ crypto_aead_aegis128l_init(const unsigned char *key, const unsigned char *nonce,
static void
crypto_aead_aegis128l_mac(unsigned char *mac, unsigned long long adlen, unsigned long long mlen,
__m128i *const state)
aes_block_t *const state)
{
__m128i tmp;
int i;
aes_block_t tmp;
int i;
tmp = _mm_set_epi64x(mlen << 3, adlen << 3);
tmp = _mm_xor_si128(tmp, state[2]);
tmp = AES_BLOCK_LOAD_64x2(mlen << 3, adlen << 3);
tmp = AES_BLOCK_XOR(tmp, state[2]);
for (i = 0; i < 7; i++) {
crypto_aead_aegis128l_update(state, tmp, tmp);
}
tmp = _mm_xor_si128(state[6], state[5]);
tmp = _mm_xor_si128(tmp, state[4]);
tmp = _mm_xor_si128(tmp, state[3]);
tmp = _mm_xor_si128(tmp, state[2]);
tmp = _mm_xor_si128(tmp, state[1]);
tmp = _mm_xor_si128(tmp, state[0]);
tmp = AES_BLOCK_XOR(state[6], state[5]);
tmp = AES_BLOCK_XOR(tmp, state[4]);
tmp = AES_BLOCK_XOR(tmp, state[3]);
tmp = AES_BLOCK_XOR(tmp, state[2]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, state[0]);
_mm_storeu_si128((__m128i *) (void *) mac, tmp);
AES_BLOCK_STORE((aes_block_t *) (void *) mac, tmp);
}
static void
crypto_aead_aegis128l_enc(unsigned char *const dst, const unsigned char *const src,
__m128i *const state)
aes_block_t *const state)
{
__m128i msg0, msg1;
__m128i tmp0, tmp1;
aes_block_t msg0, msg1;
aes_block_t tmp0, tmp1;
msg0 = _mm_loadu_si128((const __m128i *) (const void *) src);
msg1 = _mm_loadu_si128((const __m128i *) (const void *) (src + 16));
tmp0 = _mm_xor_si128(msg0, state[6]);
tmp0 = _mm_xor_si128(tmp0, state[1]);
tmp1 = _mm_xor_si128(msg1, state[2]);
tmp1 = _mm_xor_si128(tmp1, state[5]);
tmp0 = _mm_xor_si128(tmp0, _mm_and_si128(state[2], state[3]));
tmp1 = _mm_xor_si128(tmp1, _mm_and_si128(state[6], state[7]));
_mm_storeu_si128((__m128i *) (void *) dst, tmp0);
_mm_storeu_si128((__m128i *) (void *) (dst + 16), tmp1);
msg0 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) src);
msg1 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) (src + 16));
tmp0 = AES_BLOCK_XOR(msg0, state[6]);
tmp0 = AES_BLOCK_XOR(tmp0, state[1]);
tmp1 = AES_BLOCK_XOR(msg1, state[2]);
tmp1 = AES_BLOCK_XOR(tmp1, state[5]);
tmp0 = AES_BLOCK_XOR(tmp0, AES_BLOCK_AND(state[2], state[3]));
tmp1 = AES_BLOCK_XOR(tmp1, AES_BLOCK_AND(state[6], state[7]));
AES_BLOCK_STORE((aes_block_t *) (void *) dst, tmp0);
AES_BLOCK_STORE((aes_block_t *) (void *) (dst + 16), tmp1);
crypto_aead_aegis128l_update(state, msg0, msg1);
}
static void
crypto_aead_aegis128l_dec(unsigned char *const dst, const unsigned char *const src,
__m128i *const state)
aes_block_t *const state)
{
__m128i msg0, msg1;
aes_block_t msg0, msg1;
msg0 = _mm_loadu_si128((const __m128i *) (const void *) src);
msg1 = _mm_loadu_si128((const __m128i *) (const void *) (src + 16));
msg0 = _mm_xor_si128(msg0, state[6]);
msg0 = _mm_xor_si128(msg0, state[1]);
msg1 = _mm_xor_si128(msg1, state[2]);
msg1 = _mm_xor_si128(msg1, state[5]);
msg0 = _mm_xor_si128(msg0, _mm_and_si128(state[2], state[3]));
msg1 = _mm_xor_si128(msg1, _mm_and_si128(state[6], state[7]));
_mm_storeu_si128((__m128i *) (void *) dst, msg0);
_mm_storeu_si128((__m128i *) (void *) (dst + 16), msg1);
msg0 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) src);
msg1 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) (src + 16));
msg0 = AES_BLOCK_XOR(msg0, state[6]);
msg0 = AES_BLOCK_XOR(msg0, state[1]);
msg1 = AES_BLOCK_XOR(msg1, state[2]);
msg1 = AES_BLOCK_XOR(msg1, state[5]);
msg0 = AES_BLOCK_XOR(msg0, AES_BLOCK_AND(state[2], state[3]));
msg1 = AES_BLOCK_XOR(msg1, AES_BLOCK_AND(state[6], state[7]));
AES_BLOCK_STORE((aes_block_t *) (void *) dst, msg0);
AES_BLOCK_STORE((aes_block_t *) (void *) (dst + 16), msg1);
crypto_aead_aegis128l_update(state, msg0, msg1);
}
@@ -145,10 +158,10 @@ crypto_aead_aegis128l_encrypt_detached(unsigned char *c, unsigned char *mac,
unsigned long long adlen, const unsigned char *nsec,
const unsigned char *npub, const unsigned char *k)
{
__m128i state[8];
aes_block_t state[8];
CRYPTO_ALIGN(16) unsigned char src[32];
CRYPTO_ALIGN(16) unsigned char dst[32];
unsigned long long i;
unsigned long long i;
(void) nsec;
crypto_aead_aegis128l_init(k, npub, state);
@@ -194,8 +207,8 @@ crypto_aead_aegis128l_encrypt(unsigned char *c, unsigned long long *clen_p, cons
if (mlen > crypto_aead_aegis128l_MESSAGEBYTES_MAX) {
sodium_misuse();
}
ret = crypto_aead_aegis128l_encrypt_detached(c, c + mlen, NULL, m, mlen,
ad, adlen, nsec, npub, k);
ret = crypto_aead_aegis128l_encrypt_detached(c, c + mlen, NULL, m, mlen, ad, adlen, nsec, npub,
k);
if (clen_p != NULL) {
if (ret == 0) {
clen = mlen + 16ULL;
@@ -206,18 +219,19 @@ crypto_aead_aegis128l_encrypt(unsigned char *c, unsigned long long *clen_p, cons
}
int
crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec, const unsigned char *c,
unsigned long long clen, const unsigned char *mac,
const unsigned char *ad, unsigned long long adlen,
const unsigned char *npub, const unsigned char *k)
crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec,
const unsigned char *c, unsigned long long clen,
const unsigned char *mac, const unsigned char *ad,
unsigned long long adlen, const unsigned char *npub,
const unsigned char *k)
{
__m128i state[8];
aes_block_t state[8];
CRYPTO_ALIGN(16) unsigned char src[32];
CRYPTO_ALIGN(16) unsigned char dst[32];
CRYPTO_ALIGN(16) unsigned char computed_mac[16];
unsigned long long i;
unsigned long long mlen;
int ret;
unsigned long long i;
unsigned long long mlen;
int ret;
(void) nsec;
mlen = clen;
@@ -248,10 +262,10 @@ crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec, co
memcpy(m + i, dst, mlen & 0x1f);
}
memset(dst, 0, mlen & 0x1f);
state[0] = _mm_xor_si128(state[0],
_mm_loadu_si128((const __m128i *) (const void *) dst));
state[4] = _mm_xor_si128(state[4],
_mm_loadu_si128((const __m128i *) (const void *) (dst + 16)));
state[0] =
AES_BLOCK_XOR(state[0], AES_BLOCK_LOAD((const aes_block_t *) (const void *) dst));
state[4] = AES_BLOCK_XOR(state[4],
AES_BLOCK_LOAD((const aes_block_t *) (const void *) (dst + 16)));
}
crypto_aead_aegis128l_mac(computed_mac, adlen, mlen, state);
@@ -280,8 +294,8 @@ crypto_aead_aegis128l_decrypt(unsigned char *m, unsigned long long *mlen_p, unsi
int ret = -1;
if (clen >= 16ULL) {
ret = crypto_aead_aegis128l_decrypt_detached
(m, nsec, c, clen - 16ULL, c + clen - 16ULL, ad, adlen, npub, k);
ret = crypto_aead_aegis128l_decrypt_detached(m, nsec, c, clen - 16ULL, c + clen - 16ULL, ad,
adlen, npub, k);
}
if (mlen_p != NULL) {
if (ret == 0) {
src/libsodium/crypto_aead/aegis128l/armcrypto/aead_aegis128l_armcrypto.c
+97 -94
View File
@@ -14,128 +14,130 @@
#if defined(HAVE_ARMCRYPTO) && defined(NATIVE_LITTLE_ENDIAN)
# include <arm_neon.h>
#include <arm_neon.h>
typedef uint8x16_t aes_block_t;
#define AES_BLOCK_XOR(A, B) veorq_u8((A), (B))
#define AES_BLOCK_AND(A, B) vandq_u8((A), (B))
#define AES_BLOCK_LOAD(A) vld1q_u8(A)
#define AES_BLOCK_LOAD_64x2(A, B) vreinterpretq_u8_u64(vsetq_lane_u64((A), vmovq_n_u64(B), 1))
#define AES_BLOCK_STORE(A, B) vst1q_u8((A), (B))
#define AES_ENC(A, B) veorq_u8(vaesmcq_u8(vaeseq_u8((A), vmovq_n_u8(0))), (B))
static inline void
crypto_aead_aegis128l_update(uint8x16_t *const state,
const uint8x16_t d1, const uint8x16_t d2)
crypto_aead_aegis128l_update(aes_block_t *const state, const aes_block_t d1, const aes_block_t d2)
{
const uint8x16_t zero = vmovq_n_u8(0);
uint8x16_t tmp, tmp2;
aes_block_t tmp;
tmp = state[7];
state[7] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[6], zero)), state[7]);
state[6] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[5], zero)), state[6]);
state[5] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[4], zero)), state[5]);
state[4] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[3], zero)), state[4]);
state[3] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[2], zero)), state[3]);
state[2] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[1], zero)), state[2]);
state[1] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[0], zero)), state[1]);
state[0] = veorq_u8(vaesmcq_u8(vaeseq_u8(tmp, zero)), state[0]);
state[7] = AES_ENC(state[6], state[7]);
state[6] = AES_ENC(state[5], state[6]);
state[5] = AES_ENC(state[4], state[5]);
state[4] = AES_ENC(state[3], state[4]);
state[3] = AES_ENC(state[2], state[3]);
state[2] = AES_ENC(state[1], state[2]);
state[1] = AES_ENC(state[0], state[1]);
state[0] = AES_ENC(tmp, state[0]);
state[0] = veorq_u8(state[0], d1);
state[4] = veorq_u8(state[4], d2);
state[0] = AES_BLOCK_XOR(state[0], d1);
state[4] = AES_BLOCK_XOR(state[4], d2);
}
static void
crypto_aead_aegis128l_init(const unsigned char *key, const unsigned char *nonce,
uint8x16_t *const state)
aes_block_t *const state)
{
static CRYPTO_ALIGN(16) const unsigned char c0_[] = {
0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, 0x20, 0x11, 0x31, 0x42,
0x73, 0xb5, 0x28, 0xdd
};
static CRYPTO_ALIGN(16) const unsigned char c1_[] = {
0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59,
0x90, 0xe9, 0x79, 0x62
};
const uint8x16_t c0 = vld1q_u8(c0_);
const uint8x16_t c1 = vld1q_u8(c1_);
uint8x16_t k;
uint8x16_t n;
int i;
static CRYPTO_ALIGN(16)
const unsigned char c0_[] = { 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1,
0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd };
static CRYPTO_ALIGN(16)
const unsigned char c1_[] = { 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d,
0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 };
const aes_block_t c0 = AES_BLOCK_LOAD(c0_);
const aes_block_t c1 = AES_BLOCK_LOAD(c1_);
aes_block_t k;
aes_block_t n;
int i;
k = vld1q_u8(key);
n = vld1q_u8(nonce);
k = AES_BLOCK_LOAD(key);
n = AES_BLOCK_LOAD(nonce);
state[0] = veorq_u8(k, n);
state[0] = AES_BLOCK_XOR(k, n);
state[1] = c0;
state[2] = c1;
state[3] = c0;
state[4] = veorq_u8(k, n);
state[5] = veorq_u8(k, c1);
state[6] = veorq_u8(k, c0);
state[7] = veorq_u8(k, c1);
state[4] = AES_BLOCK_XOR(k, n);
state[5] = AES_BLOCK_XOR(k, c1);
state[6] = AES_BLOCK_XOR(k, c0);
state[7] = AES_BLOCK_XOR(k, c1);
for (i = 0; i < 10; i++) {
crypto_aead_aegis128l_update(state, n, k);
}
}
static void
crypto_aead_aegis128l_mac(unsigned char *mac, unsigned long long adlen,
unsigned long long mlen, uint8x16_t *const state)
crypto_aead_aegis128l_mac(unsigned char *mac, unsigned long long adlen, unsigned long long mlen,
aes_block_t *const state)
{
uint8x16_t tmp;
int i;
aes_block_t tmp;
int i;
tmp = vreinterpretq_u8_u64(vsetq_lane_u64(mlen << 3,
vmovq_n_u64(adlen << 3), 1));
tmp = veorq_u8(tmp, state[2]);
tmp = AES_BLOCK_LOAD_64x2(mlen << 3, adlen << 3);
tmp = AES_BLOCK_XOR(tmp, state[2]);
for (i = 0; i < 7; i++) {
crypto_aead_aegis128l_update(state, tmp, tmp);
}
tmp = veorq_u8(state[6], state[5]);
tmp = veorq_u8(tmp, state[4]);
tmp = veorq_u8(tmp, state[3]);
tmp = veorq_u8(tmp, state[2]);
tmp = veorq_u8(tmp, state[1]);
tmp = veorq_u8(tmp, state[0]);
tmp = AES_BLOCK_XOR(state[6], state[5]);
tmp = AES_BLOCK_XOR(tmp, state[4]);
tmp = AES_BLOCK_XOR(tmp, state[3]);
tmp = AES_BLOCK_XOR(tmp, state[2]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, state[0]);
vst1q_u8(mac, tmp);
AES_BLOCK_STORE(mac, tmp);
}
static void
crypto_aead_aegis128l_enc(unsigned char *const dst,
crypto_aead_aegis128l_enc(unsigned char *const dst,
const unsigned char *const src,
uint8x16_t *const state)
aes_block_t *const state)
{
uint8x16_t msg0, msg1;
uint8x16_t tmp0, tmp1;
aes_block_t msg0, msg1;
aes_block_t tmp0, tmp1;
msg0 = vld1q_u8(src);
msg1 = vld1q_u8(src + 16);
tmp0 = veorq_u8(msg0, state[6]);
tmp0 = veorq_u8(tmp0, state[1]);
tmp1 = veorq_u8(msg1, state[2]);
tmp1 = veorq_u8(tmp1, state[5]);
tmp0 = veorq_u8(tmp0, vandq_u8(state[2], state[3]));
tmp1 = veorq_u8(tmp1, vandq_u8(state[6], state[7]));
vst1q_u8(dst, tmp0);
vst1q_u8(dst + 16, tmp1);
msg0 = AES_BLOCK_LOAD(src);
msg1 = AES_BLOCK_LOAD(src + 16);
tmp0 = AES_BLOCK_XOR(msg0, state[6]);
tmp0 = AES_BLOCK_XOR(tmp0, state[1]);
tmp1 = AES_BLOCK_XOR(msg1, state[2]);
tmp1 = AES_BLOCK_XOR(tmp1, state[5]);
tmp0 = AES_BLOCK_XOR(tmp0, AES_BLOCK_AND(state[2], state[3]));
tmp1 = AES_BLOCK_XOR(tmp1, AES_BLOCK_AND(state[6], state[7]));
AES_BLOCK_STORE(dst, tmp0);
AES_BLOCK_STORE(dst + 16, tmp1);
crypto_aead_aegis128l_update(state, msg0, msg1);
}
static void
crypto_aead_aegis128l_dec(unsigned char *const dst,
crypto_aead_aegis128l_dec(unsigned char *const dst,
const unsigned char *const src,
uint8x16_t *const state)
aes_block_t *const state)
{
uint8x16_t msg0, msg1;
aes_block_t msg0, msg1;
msg0 = vld1q_u8(src);
msg1 = vld1q_u8(src + 16);
msg0 = veorq_u8(msg0, state[6]);
msg0 = veorq_u8(msg0, state[1]);
msg1 = veorq_u8(msg1, state[2]);
msg1 = veorq_u8(msg1, state[5]);
msg0 = veorq_u8(msg0, vandq_u8(state[2], state[3]));
msg1 = veorq_u8(msg1, vandq_u8(state[6], state[7]));
vst1q_u8(dst, msg0);
vst1q_u8(dst + 16, msg1);
msg0 = AES_BLOCK_LOAD(src);
msg1 = AES_BLOCK_LOAD(src + 16);
msg0 = AES_BLOCK_XOR(msg0, state[6]);
msg0 = AES_BLOCK_XOR(msg0, state[1]);
msg1 = AES_BLOCK_XOR(msg1, state[2]);
msg1 = AES_BLOCK_XOR(msg1, state[5]);
msg0 = AES_BLOCK_XOR(msg0, AES_BLOCK_AND(state[2], state[3]));
msg1 = AES_BLOCK_XOR(msg1, AES_BLOCK_AND(state[6], state[7]));
AES_BLOCK_STORE(dst, msg0);
AES_BLOCK_STORE(dst + 16, msg1);
crypto_aead_aegis128l_update(state, msg0, msg1);
}
@@ -147,10 +149,10 @@ crypto_aead_aegis128l_encrypt_detached(unsigned char *c, unsigned char *mac,
unsigned long long adlen, const unsigned char *nsec,
const unsigned char *npub, const unsigned char *k)
{
uint8x16_t state[8];
aes_block_t state[8];
CRYPTO_ALIGN(16) unsigned char src[32];
CRYPTO_ALIGN(16) unsigned char dst[32];
unsigned long long i;
unsigned long long i;
(void) nsec;
crypto_aead_aegis128l_init(k, npub, state);
@@ -196,8 +198,8 @@ crypto_aead_aegis128l_encrypt(unsigned char *c, unsigned long long *clen_p, cons
if (mlen > crypto_aead_aegis128l_MESSAGEBYTES_MAX) {
sodium_misuse();
}
ret = crypto_aead_aegis128l_encrypt_detached(c, c + mlen, NULL, m, mlen,
ad, adlen, nsec, npub, k);
ret = crypto_aead_aegis128l_encrypt_detached(c, c + mlen, NULL, m, mlen, ad, adlen, nsec, npub,
k);
if (clen_p != NULL) {
if (ret == 0) {
clen = mlen + 16ULL;
@@ -208,18 +210,19 @@ crypto_aead_aegis128l_encrypt(unsigned char *c, unsigned long long *clen_p, cons
}
int
crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec, const unsigned char *c,
unsigned long long clen, const unsigned char *mac,
const unsigned char *ad, unsigned long long adlen,
const unsigned char *npub, const unsigned char *k)
crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec,
const unsigned char *c, unsigned long long clen,
const unsigned char *mac, const unsigned char *ad,
unsigned long long adlen, const unsigned char *npub,
const unsigned char *k)
{
uint8x16_t state[8];
aes_block_t state[8];
CRYPTO_ALIGN(16) unsigned char src[32];
CRYPTO_ALIGN(16) unsigned char dst[32];
CRYPTO_ALIGN(16) unsigned char computed_mac[16];
unsigned long long i;
unsigned long long mlen;
int ret;
unsigned long long i;
unsigned long long mlen;
int ret;
(void) nsec;
mlen = clen;
@@ -250,8 +253,8 @@ crypto_aead_aegis128l_decrypt_detached(unsigned char *m, unsigned char *nsec, co
memcpy(m + i, dst, mlen & 0x1f);
}
memset(dst, 0, mlen & 0x1f);
state[0] = veorq_u8(state[0], vld1q_u8(dst));
state[4] = veorq_u8(state[4], vld1q_u8(dst + 16));
state[0] = AES_BLOCK_XOR(state[0], AES_BLOCK_LOAD(dst));
state[4] = AES_BLOCK_XOR(state[4], AES_BLOCK_LOAD(dst + 16));
}
crypto_aead_aegis128l_mac(computed_mac, adlen, mlen, state);
@@ -280,8 +283,8 @@ crypto_aead_aegis128l_decrypt(unsigned char *m, unsigned long long *mlen_p, unsi
int ret = -1;
if (clen >= 16ULL) {
ret = crypto_aead_aegis128l_decrypt_detached
(m, nsec, c, clen - 16ULL, c + clen - 16ULL, ad, adlen, npub, k);
ret = crypto_aead_aegis128l_decrypt_detached(m, nsec, c, clen - 16ULL, c + clen - 16ULL, ad,
adlen, npub, k);
}
if (mlen_p != NULL) {
if (ret == 0) {
src/libsodium/crypto_aead/aegis256/aesni/aead_aegis256_aesni.c
+77 -64
View File
@@ -19,50 +19,63 @@
#if defined(HAVE_TMMINTRIN_H) && defined(HAVE_WMMINTRIN_H)
#ifdef __GNUC__
# pragma GCC target("ssse3")
# pragma GCC target("aes")
#pragma GCC target("ssse3")
#pragma GCC target("aes")
#endif
#include "private/sse2_64_32.h"
#include <tmmintrin.h>
#include <wmmintrin.h>
#include "private/sse2_64_32.h"
typedef __m128i aes_block_t;
#define AES_BLOCK_XOR(A, B) _mm_xor_si128((A), (B))
#define AES_BLOCK_AND(A, B) _mm_and_si128((A), (B))
#define AES_BLOCK_LOAD(A) _mm_loadu_si128((const aes_block_t *) (const void *) (A))
#define AES_BLOCK_LOAD_64x2(A, B) _mm_set_epi64x((A), (B))
#define AES_BLOCK_STORE(A, B) _mm_storeu_si128((aes_block_t *) (void *) (A), (B))
#define AES_ENC(A, B) _mm_aesenc_si128((A), (B))
static inline void
crypto_aead_aegis256_update(__m128i *const state, const __m128i data)
crypto_aead_aegis256_update(aes_block_t *const state, const aes_block_t data)
{
__m128i tmp;
aes_block_t tmp;
tmp = _mm_aesenc_si128(state[5], state[0]);
state[5] = _mm_aesenc_si128(state[4], state[5]);
state[4] = _mm_aesenc_si128(state[3], state[4]);
state[3] = _mm_aesenc_si128(state[2], state[3]);
state[2] = _mm_aesenc_si128(state[1], state[2]);
state[1] = _mm_aesenc_si128(state[0], state[1]);
state[0] = _mm_xor_si128(tmp, data);
tmp = AES_ENC(state[5], state[0]);
state[5] = AES_ENC(state[4], state[5]);
state[4] = AES_ENC(state[3], state[4]);
state[3] = AES_ENC(state[2], state[3]);
state[2] = AES_ENC(state[1], state[2]);
state[1] = AES_ENC(state[0], state[1]);
state[0] = AES_BLOCK_XOR(tmp, data);
}
static void
crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce, __m128i *const state)
crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce,
aes_block_t *const state)
{
const __m128i c0 = _mm_set_epi8(0xdd, 0x28, 0xb5, 0x73, 0x42, 0x31, 0x11, 0x20, 0xf1, 0x2f, 0xc2, 0x6d,
0x55, 0x18, 0x3d, 0xdb);
const __m128i c1 = _mm_set_epi8(0x62, 0x79, 0xe9, 0x90, 0x59, 0x37, 0x22, 0x15, 0x0d, 0x08, 0x05, 0x03,
0x02, 0x01, 0x01, 0x00);
__m128i k1, k2;
__m128i kxn1, kxn2;
int i;
static CRYPTO_ALIGN(16)
const uint8_t c0_[] = { 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1,
0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd };
static CRYPTO_ALIGN(16)
const uint8_t c1_[] = { 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d,
0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 };
const aes_block_t c0 = AES_BLOCK_LOAD(c0_);
const aes_block_t c1 = AES_BLOCK_LOAD(c1_);
aes_block_t k1, k2;
aes_block_t kxn1, kxn2;
int i;
k1 = _mm_loadu_si128((const __m128i *) (const void *) &key[0]);
k2 = _mm_loadu_si128((const __m128i *) (const void *) &key[16]);
kxn1 = _mm_xor_si128(k1, _mm_loadu_si128((__m128i *) (void *) &nonce[0]));
kxn2 = _mm_xor_si128(k2, _mm_loadu_si128((__m128i *) (void *) &nonce[16]));
k1 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) &key[0]);
k2 = AES_BLOCK_LOAD((const aes_block_t *) (const void *) &key[16]);
kxn1 = AES_BLOCK_XOR(k1, AES_BLOCK_LOAD((aes_block_t *) (void *) &nonce[0]));
kxn2 = AES_BLOCK_XOR(k2, AES_BLOCK_LOAD((aes_block_t *) (void *) &nonce[16]));
state[0] = kxn1;
state[1] = kxn2;
state[2] = c0;
state[3] = c1;
state[4] = _mm_xor_si128(k1, c1);
state[5] = _mm_xor_si128(k2, c0);
state[4] = AES_BLOCK_XOR(k1, c1);
state[5] = AES_BLOCK_XOR(k2, c0);
for (i = 0; i < 4; i++) {
crypto_aead_aegis256_update(state, k1);
@@ -74,56 +87,56 @@ crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce,
static void
crypto_aead_aegis256_mac(unsigned char *mac, unsigned long long adlen, unsigned long long mlen,
__m128i *const state)
aes_block_t *const state)
{
__m128i tmp;
int i;
aes_block_t tmp;
int i;
tmp = _mm_set_epi64x(mlen << 3, adlen << 3);
tmp = _mm_xor_si128(tmp, state[3]);
tmp = AES_BLOCK_LOAD_64x2(mlen << 3, adlen << 3);
tmp = AES_BLOCK_XOR(tmp, state[3]);
for (i = 0; i < 7; i++) {
crypto_aead_aegis256_update(state, tmp);
}
tmp = _mm_xor_si128(state[5], state[4]);
tmp = _mm_xor_si128(tmp, state[3]);
tmp = _mm_xor_si128(tmp, state[2]);
tmp = _mm_xor_si128(tmp, state[1]);
tmp = _mm_xor_si128(tmp, state[0]);
tmp = AES_BLOCK_XOR(state[5], state[4]);
tmp = AES_BLOCK_XOR(tmp, state[3]);
tmp = AES_BLOCK_XOR(tmp, state[2]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, state[0]);
_mm_storeu_si128((__m128i *) (void *) mac, tmp);
AES_BLOCK_STORE((aes_block_t *) (void *) mac, tmp);
}
static void
crypto_aead_aegis256_enc(unsigned char *const dst, const unsigned char *const src,
__m128i *const state)
aes_block_t *const state)
{
__m128i msg;
__m128i tmp;
aes_block_t msg;
aes_block_t tmp;
msg = _mm_loadu_si128((const __m128i *) (const void *) src);
tmp = _mm_xor_si128(msg, state[5]);
tmp = _mm_xor_si128(tmp, state[4]);
tmp = _mm_xor_si128(tmp, state[1]);
tmp = _mm_xor_si128(tmp, _mm_and_si128(state[2], state[3]));
_mm_storeu_si128((__m128i *) (void *) dst, tmp);
msg = AES_BLOCK_LOAD((const aes_block_t *) (const void *) src);
tmp = AES_BLOCK_XOR(msg, state[5]);
tmp = AES_BLOCK_XOR(tmp, state[4]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, AES_BLOCK_AND(state[2], state[3]));
AES_BLOCK_STORE((aes_block_t *) (void *) dst, tmp);
crypto_aead_aegis256_update(state, msg);
}
static void
crypto_aead_aegis256_dec(unsigned char *const dst, const unsigned char *const src,
__m128i *const state)
aes_block_t *const state)
{
__m128i msg;
aes_block_t msg;
msg = _mm_loadu_si128((const __m128i *) (const void *) src);
msg = _mm_xor_si128(msg, state[5]);
msg = _mm_xor_si128(msg, state[4]);
msg = _mm_xor_si128(msg, state[1]);
msg = _mm_xor_si128(msg, _mm_and_si128(state[2], state[3]));
_mm_storeu_si128((__m128i *) (void *) dst, msg);
msg = AES_BLOCK_LOAD((const aes_block_t *) (const void *) src);
msg = AES_BLOCK_XOR(msg, state[5]);
msg = AES_BLOCK_XOR(msg, state[4]);
msg = AES_BLOCK_XOR(msg, state[1]);
msg = AES_BLOCK_XOR(msg, AES_BLOCK_AND(state[2], state[3]));
AES_BLOCK_STORE((aes_block_t *) (void *) dst, msg);
crypto_aead_aegis256_update(state, msg);
}
@@ -135,10 +148,10 @@ crypto_aead_aegis256_encrypt_detached(unsigned char *c, unsigned char *mac,
unsigned long long adlen, const unsigned char *nsec,
const unsigned char *npub, const unsigned char *k)
{
__m128i state[6];
aes_block_t state[6];
CRYPTO_ALIGN(16) unsigned char src[16];
CRYPTO_ALIGN(16) unsigned char dst[16];
unsigned long long i;
unsigned long long i;
(void) nsec;
crypto_aead_aegis256_init(k, npub, state);
@@ -184,8 +197,8 @@ crypto_aead_aegis256_encrypt(unsigned char *c, unsigned long long *clen_p, const
if (mlen > crypto_aead_aegis256_MESSAGEBYTES_MAX) {
sodium_misuse();
}
ret = crypto_aead_aegis256_encrypt_detached(c, c + mlen, NULL, m, mlen,
ad, adlen, nsec, npub, k);
ret =
crypto_aead_aegis256_encrypt_detached(c, c + mlen, NULL, m, mlen, ad, adlen, nsec, npub, k);
if (clen_p != NULL) {
if (ret == 0) {
clen = mlen + 16ULL;
@@ -201,13 +214,13 @@ crypto_aead_aegis256_decrypt_detached(unsigned char *m, unsigned char *nsec, con
const unsigned char *ad, unsigned long long adlen,
const unsigned char *npub, const unsigned char *k)
{
__m128i state[6];
aes_block_t state[6];
CRYPTO_ALIGN(16) unsigned char src[16];
CRYPTO_ALIGN(16) unsigned char dst[16];
CRYPTO_ALIGN(16) unsigned char computed_mac[16];
unsigned long long i;
unsigned long long mlen;
int ret;
unsigned long long i;
unsigned long long mlen;
int ret;
(void) nsec;
mlen = clen;
@@ -238,8 +251,8 @@ crypto_aead_aegis256_decrypt_detached(unsigned char *m, unsigned char *nsec, con
memcpy(m + i, dst, mlen & 0xf);
}
memset(dst, 0, mlen & 0xf);
state[0] = _mm_xor_si128(state[0],
_mm_loadu_si128((const __m128i *) (const void *) dst));
state[0] =
AES_BLOCK_XOR(state[0], AES_BLOCK_LOAD((const aes_block_t *) (const void *) dst));
}
crypto_aead_aegis256_mac(computed_mac, adlen, mlen, state);
src/libsodium/crypto_aead/aegis256/armcrypto/aead_aegis256_armcrypto.c
+63 -58
View File
@@ -16,24 +16,31 @@
# include <arm_neon.h>
static inline void
crypto_aead_aegis256_update(uint8x16_t *const state, const uint8x16_t data)
{
const uint8x16_t zero = vmovq_n_u8(0);
uint8x16_t tmp;
typedef uint8x16_t aes_block_t;
#define AES_BLOCK_XOR(A, B) veorq_u8((A), (B))
#define AES_BLOCK_AND(A, B) vandq_u8((A), (B))
#define AES_BLOCK_LOAD(A) vld1q_u8(A)
#define AES_BLOCK_LOAD_64x2(A, B) vreinterpretq_u8_u64(vsetq_lane_u64((A), vmovq_n_u64(B), 1))
#define AES_BLOCK_STORE(A, B) vst1q_u8((A), (B))
#define AES_ENC(A, B) veorq_u8(vaesmcq_u8(vaeseq_u8((A), vmovq_n_u8(0))), (B))
tmp = veorq_u8(vaesmcq_u8(vaeseq_u8(state[5], zero)), state[0]);
state[5] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[4], zero)), state[5]);
state[4] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[3], zero)), state[4]);
state[3] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[2], zero)), state[3]);
state[2] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[1], zero)), state[2]);
state[1] = veorq_u8(vaesmcq_u8(vaeseq_u8(state[0], zero)), state[1]);
state[0] = veorq_u8(tmp, data);
static inline void
crypto_aead_aegis256_update(aes_block_t *const state, const aes_block_t data)
{
aes_block_t tmp;
tmp = AES_ENC(state[5], state[0]);
state[5] = AES_ENC(state[4], state[5]);
state[4] = AES_ENC(state[3], state[4]);
state[3] = AES_ENC(state[2], state[3]);
state[2] = AES_ENC(state[1], state[2]);
state[1] = AES_ENC(state[0], state[1]);
state[0] = AES_BLOCK_XOR(tmp, data);
}
static void
crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce,
uint8x16_t *const state)
aes_block_t *const state)
{
static CRYPTO_ALIGN(16) const unsigned char c0_[] = {
0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, 0x20, 0x11, 0x31, 0x42,
@@ -43,23 +50,23 @@ crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce,
0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59,
0x90, 0xe9, 0x79, 0x62
};
const uint8x16_t c0 = vld1q_u8(c0_);
const uint8x16_t c1 = vld1q_u8(c1_);
uint8x16_t k1, k2;
uint8x16_t kxn1, kxn2;
const aes_block_t c0 = AES_BLOCK_LOAD(c0_);
const aes_block_t c1 = AES_BLOCK_LOAD(c1_);
aes_block_t k1, k2;
aes_block_t kxn1, kxn2;
int i;
k1 = vld1q_u8(&key[0]);
k2 = vld1q_u8(&key[16]);
kxn1 = veorq_u8(k1, vld1q_u8(&nonce[0]));
kxn2 = veorq_u8(k2, vld1q_u8(&nonce[16]));
k1 = AES_BLOCK_LOAD(&key[0]);
k2 = AES_BLOCK_LOAD(&key[16]);
kxn1 = AES_BLOCK_XOR(k1, AES_BLOCK_LOAD(&nonce[0]));
kxn2 = AES_BLOCK_XOR(k2, AES_BLOCK_LOAD(&nonce[16]));
state[0] = kxn1;
state[1] = kxn2;
state[2] = c0;
state[3] = c1;
state[4] = veorq_u8(k1, c1);
state[5] = veorq_u8(k2, c0);
state[4] = AES_BLOCK_XOR(k1, c1);
state[5] = AES_BLOCK_XOR(k2, c0);
for (i = 0; i < 4; i++) {
crypto_aead_aegis256_update(state, k1);
@@ -70,61 +77,59 @@ crypto_aead_aegis256_init(const unsigned char *key, const unsigned char *nonce,
}
static void
crypto_aead_aegis256_mac(unsigned char *mac, unsigned long long adlen,
unsigned long long mlen, uint8x16_t *const state)
crypto_aead_aegis256_mac(unsigned char *mac, unsigned long long adlen, unsigned long long mlen,
aes_block_t *const state)
{
uint8x16_t tmp;
aes_block_t tmp;
int i;
tmp = vreinterpretq_u8_u64(vsetq_lane_u64(mlen << 3,
vmovq_n_u64(adlen << 3), 1));
tmp = veorq_u8(tmp, state[3]);
tmp = AES_BLOCK_LOAD_64x2(mlen << 3, adlen << 3);
tmp = AES_BLOCK_XOR(tmp, state[3]);
for (i = 0; i < 7; i++) {
crypto_aead_aegis256_update(state, tmp);
}
tmp = veorq_u8(state[5], state[4]);
tmp = veorq_u8(tmp, state[3]);
tmp = veorq_u8(tmp, state[2]);
tmp = veorq_u8(tmp, state[1]);
tmp = veorq_u8(tmp, state[0]);
tmp = AES_BLOCK_XOR(state[5], state[4]);
tmp = AES_BLOCK_XOR(tmp, state[3]);
tmp = AES_BLOCK_XOR(tmp, state[2]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, state[0]);
vst1q_u8(mac, tmp);
AES_BLOCK_STORE(mac, tmp);
}
static void
crypto_aead_aegis256_enc(unsigned char *const dst,
crypto_aead_aegis256_enc(unsigned char *const dst,
const unsigned char *const src,
uint8x16_t *const state)
aes_block_t *const state)
{
uint8x16_t msg;
uint8x16_t tmp;
aes_block_t msg;
aes_block_t tmp;
msg = vld1q_u8(src);
tmp = veorq_u8(msg, state[5]);
tmp = veorq_u8(tmp, state[4]);
tmp = veorq_u8(tmp, state[1]);
tmp = veorq_u8(tmp, vandq_u8(state[2], state[3]));
vst1q_u8(dst, tmp);
msg = AES_BLOCK_LOAD(src);
tmp = AES_BLOCK_XOR(msg, state[5]);
tmp = AES_BLOCK_XOR(tmp, state[4]);
tmp = AES_BLOCK_XOR(tmp, state[1]);
tmp = AES_BLOCK_XOR(tmp, AES_BLOCK_AND(state[2], state[3]));
AES_BLOCK_STORE(dst, tmp);
crypto_aead_aegis256_update(state, msg);
}
static void
crypto_aead_aegis256_dec(unsigned char *const dst,
crypto_aead_aegis256_dec(unsigned char *const dst,
const unsigned char *const src,
uint8x16_t *const state)
aes_block_t *const state)
{
uint8x16_t msg;
aes_block_t msg;
msg = vld1q_u8(src);
msg = veorq_u8(msg, state[5]);
msg = veorq_u8(msg, state[4]);
msg = veorq_u8(msg, state[1]);
msg = veorq_u8(msg, vandq_u8(state[2], state[3]));
vst1q_u8(dst, msg);
msg = AES_BLOCK_LOAD(src);
msg = AES_BLOCK_XOR(msg, state[5]);
msg = AES_BLOCK_XOR(msg, state[4]);
msg = AES_BLOCK_XOR(msg, state[1]);
msg = AES_BLOCK_XOR(msg, AES_BLOCK_AND(state[2], state[3]));
AES_BLOCK_STORE(dst, msg);
crypto_aead_aegis256_update(state, msg);
}
@@ -136,7 +141,7 @@ crypto_aead_aegis256_encrypt_detached(unsigned char *c, unsigned char *mac,
unsigned long long adlen, const unsigned char *nsec,
const unsigned char *npub, const unsigned char *k)
{
uint8x16_t state[6];
aes_block_t state[6];
CRYPTO_ALIGN(16) unsigned char src[16];
CRYPTO_ALIGN(16) unsigned char dst[16];
unsigned long long i;
@@ -202,7 +207,7 @@ crypto_aead_aegis256_decrypt_detached(unsigned char *m, unsigned char *nsec, con
const unsigned char *ad, unsigned long long adlen,
const unsigned char *npub, const unsigned char *k)
{
uint8x16_t state[6];
aes_block_t state[6];
CRYPTO_ALIGN(16) unsigned char src[16];
CRYPTO_ALIGN(16) unsigned char dst[16];
CRYPTO_ALIGN(16) unsigned char computed_mac[16];
@@ -239,7 +244,7 @@ crypto_aead_aegis256_decrypt_detached(unsigned char *m, unsigned char *nsec, con
memcpy(m + i, dst, mlen & 0xf);
}
memset(dst, 0, mlen & 0xf);
state[0] = veorq_u8(state[0], vld1q_u8(dst));
state[0] = AES_BLOCK_XOR(state[0], AES_BLOCK_LOAD(dst));
}
crypto_aead_aegis256_mac(computed_mac, adlen, mlen, state);