Skip to content

X25519/Ed25519: fix CT for field operations for specific compilers #12092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SparkiDev wants to merge 1 commit into
base: master
Choose a base branch
from
Open

X25519/Ed25519: fix CT for field operations for specific compilers #12092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6622,6 +6622,19 @@ then
ENABLED_CURVE25519="yes"
fi

# Cortex-A55/A53: 64-bit multiply latency depends on operand values. Select
# fixed latency multiply code for the Curve25519/Ed25519 field operations when
# building for these cores (-mcpu/-mtune, including big.LITTLE e.g.
# cortex-a76.cortex-a55).
case "$CC $CFLAGS $CPPFLAGS" in
*cortex-a55*)
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_CORTEX_A55"
;;
*cortex-a53*)
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_CORTEX_A53"
;;
esac


# ED25519
AC_ARG_ENABLE([ed25519],
Expand Down
128 changes: 88 additions & 40 deletions wolfcrypt/src/fe_operations.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,52 @@
#include <wolfcrypt/src/misc.c>
#endif

/* Hide a secret-derived mask from the optimizer so that masked selection
* (fe_cswap, fe_cmov) is not converted into a conditional branch or
* conditional loads (seen with GCC -O3 on PPC64).
*/
#if defined(__GNUC__) && !defined(WOLFSSL_NO_ASM_BARRIER)
#define FE_CT_BARRIER(v) __asm__ __volatile__ ("" : "+r" (v))
#else
#define FE_CT_BARRIER(v) (void)(v)
#endif

/* On some CPUs the latency of a 64-bit multiply depends on the operand
* values. Cortex-A55: "64-bit multiplies take two or three cycles, depending
* on whether both operands contain '1's in their top 32 bits" (Software
* Optimization Guide, section 3.4). The Cortex-A53 is the A55's predecessor
* and is treated the same. When enabled, the field code is written so the
* compiler only uses fixed latency multiplies on secret limbs:
* - 32-bit limbs: small multiples are computed in unsigned 32-bit so the
* compiler can't widen them into 64-bit multiplies (Clang does).
* - 128-bit: the x19 and x2 multiples are applied to 64-bit limbs so every
* product is a plain signed 64x64->128 multiply. Otherwise Clang emits
* multiplies with the sign word of a limb as an operand.
* Enable with WOLFSSL_CORTEX_A55, WOLFSSL_CORTEX_A53 or directly with
* WOLFSSL_CURVE25519_FIXED_LATENCY_MUL.
*/
#if (defined(WOLFSSL_CORTEX_A55) || defined(WOLFSSL_CORTEX_A53)) && \
!defined(WOLFSSL_CURVE25519_FIXED_LATENCY_MUL)
#define WOLFSSL_CURVE25519_FIXED_LATENCY_MUL
#endif

/* 128-bit: apply the x19 and x2 multiples to 64-bit limbs before the
* products (fewer 128-bit multiplies). Default as it is faster on x86_64 and
* needed for fixed latency multiplies. NO_CURVED25519_128BIT_FOLDED selects the
* original form, except when fixed latency multiplies are required.
*/
#if (!defined(NO_CURVED25519_128BIT_FOLDED) || \
defined(WOLFSSL_CURVE25519_FIXED_LATENCY_MUL)) && \
!defined(CURVED25519_128BIT_FOLDED)
#define CURVED25519_128BIT_FOLDED
#endif

#ifdef WOLFSSL_CURVE25519_FIXED_LATENCY_MUL
#define FE_MUL32_SMALL(c, v) ((sword32)((word32)(c) * (word32)(v)))
#else
#define FE_MUL32_SMALL(c, v) ((c) * (v))
#endif

#ifdef CURVED25519_X64
/* Assembly code in fe_x25519_asm.* */
#elif defined(WOLFSSL_ARMASM)
Expand Down Expand Up @@ -300,19 +346,19 @@ void fe_sq(fe h,const fe f)
sword32 f7 = f[7];
sword32 f8 = f[8];
sword32 f9 = f[9];
sword32 f0_2 = 2 * f0;
sword32 f1_2 = 2 * f1;
sword32 f2_2 = 2 * f2;
sword32 f3_2 = 2 * f3;
sword32 f4_2 = 2 * f4;
sword32 f5_2 = 2 * f5;
sword32 f6_2 = 2 * f6;
sword32 f7_2 = 2 * f7;
sword32 f5_38 = 38 * f5; /* 1.959375*2^30 */
sword32 f6_19 = 19 * f6; /* 1.959375*2^30 */
sword32 f7_38 = 38 * f7; /* 1.959375*2^30 */
sword32 f8_19 = 19 * f8; /* 1.959375*2^30 */
sword32 f9_38 = 38 * f9; /* 1.959375*2^30 */
sword32 f0_2 = FE_MUL32_SMALL(2, f0);
sword32 f1_2 = FE_MUL32_SMALL(2, f1);
sword32 f2_2 = FE_MUL32_SMALL(2, f2);
sword32 f3_2 = FE_MUL32_SMALL(2, f3);
sword32 f4_2 = FE_MUL32_SMALL(2, f4);
sword32 f5_2 = FE_MUL32_SMALL(2, f5);
sword32 f6_2 = FE_MUL32_SMALL(2, f6);
sword32 f7_2 = FE_MUL32_SMALL(2, f7);
sword32 f5_38 = FE_MUL32_SMALL(38, f5); /* 1.959375*2^30 */
sword32 f6_19 = FE_MUL32_SMALL(19, f6); /* 1.959375*2^30 */
sword32 f7_38 = FE_MUL32_SMALL(38, f7); /* 1.959375*2^30 */
sword32 f8_19 = FE_MUL32_SMALL(19, f8); /* 1.959375*2^30 */
sword32 f9_38 = FE_MUL32_SMALL(38, f9); /* 1.959375*2^30 */
sword64 f0f0 = f0 * (sword64) f0;
sword64 f0f1_2 = f0_2 * (sword64) f1;
sword64 f0f2_2 = f0_2 * (sword64) f2;
Expand Down Expand Up @@ -841,20 +887,20 @@ void fe_mul(fe h,const fe f,const fe g)
sword32 g7 = g[7];
sword32 g8 = g[8];
sword32 g9 = g[9];
sword32 g1_19 = 19 * g1; /* 1.959375*2^29 */
sword32 g2_19 = 19 * g2; /* 1.959375*2^30; still ok */
sword32 g3_19 = 19 * g3;
sword32 g4_19 = 19 * g4;
sword32 g5_19 = 19 * g5;
sword32 g6_19 = 19 * g6;
sword32 g7_19 = 19 * g7;
sword32 g8_19 = 19 * g8;
sword32 g9_19 = 19 * g9;
sword32 f1_2 = 2 * f1;
sword32 f3_2 = 2 * f3;
sword32 f5_2 = 2 * f5;
sword32 f7_2 = 2 * f7;
sword32 f9_2 = 2 * f9;
sword32 g1_19 = FE_MUL32_SMALL(19, g1); /* 1.959375*2^29 */
sword32 g2_19 = FE_MUL32_SMALL(19, g2); /* 1.959375*2^30; still ok */
sword32 g3_19 = FE_MUL32_SMALL(19, g3);
sword32 g4_19 = FE_MUL32_SMALL(19, g4);
sword32 g5_19 = FE_MUL32_SMALL(19, g5);
sword32 g6_19 = FE_MUL32_SMALL(19, g6);
sword32 g7_19 = FE_MUL32_SMALL(19, g7);
sword32 g8_19 = FE_MUL32_SMALL(19, g8);
sword32 g9_19 = FE_MUL32_SMALL(19, g9);
sword32 f1_2 = FE_MUL32_SMALL(2, f1);
sword32 f3_2 = FE_MUL32_SMALL(2, f3);
sword32 f5_2 = FE_MUL32_SMALL(2, f5);
sword32 f7_2 = FE_MUL32_SMALL(2, f7);
sword32 f9_2 = FE_MUL32_SMALL(2, f9);
sword64 f0g0 = f0 * (sword64) g0;
sword64 f0g1 = f0 * (sword64) g1;
sword64 f0g2 = f0 * (sword64) g2;
Expand Down Expand Up @@ -1079,6 +1125,7 @@ void fe_cswap(fe f, fe g, int b)
sword32 x8 = f8 ^ g8;
sword32 x9 = f9 ^ g9;
b = -b;
FE_CT_BARRIER(b);
x0 &= b;
x1 &= b;
x2 &= b;
Expand Down Expand Up @@ -1208,19 +1255,19 @@ void fe_sq2(fe h,const fe f)
sword32 f7 = f[7];
sword32 f8 = f[8];
sword32 f9 = f[9];
sword32 f0_2 = 2 * f0;
sword32 f1_2 = 2 * f1;
sword32 f2_2 = 2 * f2;
sword32 f3_2 = 2 * f3;
sword32 f4_2 = 2 * f4;
sword32 f5_2 = 2 * f5;
sword32 f6_2 = 2 * f6;
sword32 f7_2 = 2 * f7;
sword32 f5_38 = 38 * f5; /* 1.959375*2^30 */
sword32 f6_19 = 19 * f6; /* 1.959375*2^30 */
sword32 f7_38 = 38 * f7; /* 1.959375*2^30 */
sword32 f8_19 = 19 * f8; /* 1.959375*2^30 */
sword32 f9_38 = 38 * f9; /* 1.959375*2^30 */
sword32 f0_2 = FE_MUL32_SMALL(2, f0);
sword32 f1_2 = FE_MUL32_SMALL(2, f1);
sword32 f2_2 = FE_MUL32_SMALL(2, f2);
sword32 f3_2 = FE_MUL32_SMALL(2, f3);
sword32 f4_2 = FE_MUL32_SMALL(2, f4);
sword32 f5_2 = FE_MUL32_SMALL(2, f5);
sword32 f6_2 = FE_MUL32_SMALL(2, f6);
sword32 f7_2 = FE_MUL32_SMALL(2, f7);
sword32 f5_38 = FE_MUL32_SMALL(38, f5); /* 1.959375*2^30 */
sword32 f6_19 = FE_MUL32_SMALL(19, f6); /* 1.959375*2^30 */
sword32 f7_38 = FE_MUL32_SMALL(38, f7); /* 1.959375*2^30 */
sword32 f8_19 = FE_MUL32_SMALL(19, f8); /* 1.959375*2^30 */
sword32 f9_38 = FE_MUL32_SMALL(38, f9); /* 1.959375*2^30 */
sword64 f0f0 = f0 * (sword64) f0;
sword64 f0f1_2 = f0_2 * (sword64) f1;
sword64 f0f2_2 = f0_2 * (sword64) f2;
Expand Down Expand Up @@ -1494,6 +1541,7 @@ void fe_cmov(fe f, const fe g, int b)
sword32 x8 = f8 ^ g8;
sword32 x9 = f9 ^ g9;
b = -b;
FE_CT_BARRIER(b);
x0 &= b;
x1 &= b;
x2 &= b;
Expand Down
140 changes: 140 additions & 0 deletions wolfcrypt/src/fe_x25519_128.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ void fe_cswap(fe f, fe g, int b)

/* Convert conditional into mask. */
m = -m;
FE_CT_BARRIER(m);
t0 = m & (f[0] ^ g[0]);
t1 = m & (f[1] ^ g[1]);
t2 = m & (f[2] ^ g[2]);
Expand Down Expand Up @@ -265,6 +266,7 @@ void fe_add(fe r, const fe a, const fe b)
* a A field element.
* b A field element.
*/
#ifndef CURVED25519_128BIT_FOLDED
void fe_mul(fe r, const fe a, const fe b)
{
const __int128_t k19 = 19;
Expand Down Expand Up @@ -311,12 +313,65 @@ void fe_mul(fe r, const fe a, const fe b)
r[4] = t4 & 0x7ffffffffffff;
}

#else
void fe_mul(fe r, const fe a, const fe b)
{
const __int128_t k19 = 19;
/* Pre-multiply by 19 in 64-bit (unsigned so the compiler cannot widen
* it) so that every product below is a plain signed 64x64->128 multiply.
* Avoids 128-bit multiplies that use the sign word of a limb as an
* operand - operand dependent latency on some CPUs (e.g. Cortex-A55).
*/
sword64 b1_19 = (sword64)((word64)b[1] * 19);
sword64 b2_19 = (sword64)((word64)b[2] * 19);
sword64 b3_19 = (sword64)((word64)b[3] * 19);
sword64 b4_19 = (sword64)((word64)b[4] * 19);
__int128_t t0 = ((__int128_t)a[0]) * b[0]
+ ((__int128_t)a[1]) * b4_19
+ ((__int128_t)a[2]) * b3_19
+ ((__int128_t)a[3]) * b2_19
+ ((__int128_t)a[4]) * b1_19;
__int128_t t1 = ((__int128_t)a[0]) * b[1]
+ ((__int128_t)a[1]) * b[0]
+ ((__int128_t)a[2]) * b4_19
+ ((__int128_t)a[3]) * b3_19
+ ((__int128_t)a[4]) * b2_19;
__int128_t t2 = ((__int128_t)a[0]) * b[2]
+ ((__int128_t)a[1]) * b[1]
+ ((__int128_t)a[2]) * b[0]
+ ((__int128_t)a[3]) * b4_19
+ ((__int128_t)a[4]) * b3_19;
__int128_t t3 = ((__int128_t)a[0]) * b[3]
+ ((__int128_t)a[1]) * b[2]
+ ((__int128_t)a[2]) * b[1]
+ ((__int128_t)a[3]) * b[0]
+ ((__int128_t)a[4]) * b4_19;
__int128_t t4 = ((__int128_t)a[0]) * b[4]
+ ((__int128_t)a[1]) * b[3]
+ ((__int128_t)a[2]) * b[2]
+ ((__int128_t)a[3]) * b[1]
+ ((__int128_t)a[4]) * b[0];

/* Normalize to 51-bits of data per word. */
t0 += (t4 >> 51) * k19; t4 &= 0x7ffffffffffff;

t1 += t0 >> 51; r[0] = t0 & 0x7ffffffffffff;
t2 += t1 >> 51; r[1] = t1 & 0x7ffffffffffff;
t3 += t2 >> 51; r[2] = t2 & 0x7ffffffffffff;
t4 += t3 >> 51; r[3] = t3 & 0x7ffffffffffff;
r[0] += (sword64)((t4 >> 51) * k19);
r[4] = t4 & 0x7ffffffffffff;
}

#endif /* CURVED25519_128BIT_FOLDED */

/* Square a and put result in r. (r = a * a)
*
* r A field element.
* a A field element.
* b A field element.
*/
#ifndef CURVED25519_128BIT_FOLDED
void fe_sq(fe r, const fe a)
{
const __int128_t k19 = 19;
Expand Down Expand Up @@ -354,6 +409,47 @@ void fe_sq(fe r, const fe a)
r[4] = t4 & 0x7ffffffffffff;
}

#else
void fe_sq(fe r, const fe a)
{
const __int128_t k19 = 19;
/* Small multiples in 64-bit (unsigned so the compiler cannot widen them)
* so that every product is a plain signed 64x64->128 multiply. */
sword64 a0_2 = (sword64)((word64)a[0] * 2);
sword64 a1_2 = (sword64)((word64)a[1] * 2);
sword64 a2_2 = (sword64)((word64)a[2] * 2);
sword64 a3_2 = (sword64)((word64)a[3] * 2);
sword64 a3_19 = (sword64)((word64)a[3] * 19);
sword64 a4_19 = (sword64)((word64)a[4] * 19);
__int128_t t0 = ((__int128_t)a[0]) * a[0]
+ ((__int128_t)a1_2) * a4_19
+ ((__int128_t)a2_2) * a3_19;
__int128_t t1 = ((__int128_t)a0_2) * a[1]
+ ((__int128_t)a2_2) * a4_19
+ ((__int128_t)a[3]) * a3_19;
__int128_t t2 = ((__int128_t)a0_2) * a[2]
+ ((__int128_t)a[1]) * a[1]
+ ((__int128_t)a3_2) * a4_19;
__int128_t t3 = ((__int128_t)a0_2) * a[3]
+ ((__int128_t)a1_2) * a[2]
+ ((__int128_t)a[4]) * a4_19;
__int128_t t4 = ((__int128_t)a0_2) * a[4]
+ ((__int128_t)a1_2) * a[3]
+ ((__int128_t)a[2]) * a[2];

/* Normalize to 51-bits of data per word. */
t0 += (t4 >> 51) * k19; t4 &= 0x7ffffffffffff;

t1 += t0 >> 51; r[0] = t0 & 0x7ffffffffffff;
t2 += t1 >> 51; r[1] = t1 & 0x7ffffffffffff;
t3 += t2 >> 51; r[2] = t2 & 0x7ffffffffffff;
t4 += t3 >> 51; r[3] = t3 & 0x7ffffffffffff;
r[0] += (sword64)((t4 >> 51) * k19);
r[4] = t4 & 0x7ffffffffffff;
}

#endif /* CURVED25519_128BIT_FOLDED */

/* Multiply a by 121666 and put result in r. (r = 121666 * a)
*
* r A field element.
Expand Down Expand Up @@ -586,6 +682,7 @@ void fe_cmov(fe f, const fe g, int b)

/* Convert conditional into mask. */
m = -m;
FE_CT_BARRIER(m);
t0 = m & (f[0] ^ g[0]);
t1 = m & (f[1] ^ g[1]);
t2 = m & (f[2] ^ g[2]);
Expand Down Expand Up @@ -627,6 +724,7 @@ void fe_pow22523(fe r, const fe a)
* a A field element.
* b A field element.
*/
#ifndef CURVED25519_128BIT_FOLDED
void fe_sq2(fe r, const fe a)
{
const __int128_t k2 = 2;
Expand Down Expand Up @@ -664,6 +762,48 @@ void fe_sq2(fe r, const fe a)
r[4] = t4 & 0x7ffffffffffff;
}

#else
void fe_sq2(fe r, const fe a)
{
const __int128_t k2 = 2;
const __int128_t k19 = 19;
/* Small multiples in 64-bit (unsigned so the compiler cannot widen them)
* so that every product is a plain signed 64x64->128 multiply. */
sword64 a0_2 = (sword64)((word64)a[0] * 2);
sword64 a1_2 = (sword64)((word64)a[1] * 2);
sword64 a2_2 = (sword64)((word64)a[2] * 2);
sword64 a3_2 = (sword64)((word64)a[3] * 2);
sword64 a3_19 = (sword64)((word64)a[3] * 19);
sword64 a4_19 = (sword64)((word64)a[4] * 19);
__int128_t t0 = k2 * (((__int128_t)a[0]) * a[0]
+ ((__int128_t)a1_2) * a4_19
+ ((__int128_t)a2_2) * a3_19);
__int128_t t1 = k2 * (((__int128_t)a0_2) * a[1]
+ ((__int128_t)a2_2) * a4_19
+ ((__int128_t)a[3]) * a3_19);
__int128_t t2 = k2 * (((__int128_t)a0_2) * a[2]
+ ((__int128_t)a[1]) * a[1]
+ ((__int128_t)a3_2) * a4_19);
__int128_t t3 = k2 * (((__int128_t)a0_2) * a[3]
+ ((__int128_t)a1_2) * a[2]
+ ((__int128_t)a[4]) * a4_19);
__int128_t t4 = k2 * (((__int128_t)a0_2) * a[4]
+ ((__int128_t)a1_2) * a[3]
+ ((__int128_t)a[2]) * a[2]);

/* Normalize to 51-bits of data per word. */
t0 += (t4 >> 51) * k19; t4 &= 0x7ffffffffffff;

t1 += t0 >> 51; r[0] = t0 & 0x7ffffffffffff;
t2 += t1 >> 51; r[1] = t1 & 0x7ffffffffffff;
t3 += t2 >> 51; r[2] = t2 & 0x7ffffffffffff;
t4 += t3 >> 51; r[3] = t3 & 0x7ffffffffffff;
r[0] += (sword64)((t4 >> 51) * k19);
r[4] = t4 & 0x7ffffffffffff;
}

#endif /* CURVED25519_128BIT_FOLDED */

/* Load 3 little endian bytes into a 64-bit word.
*
* in An array of bytes.
Expand Down
Loading