Enable FE25519_ADX Curve25519 base point mult. except on Windows.

Cargo.toml

@@ -45,6 +45,7 @@ include = [
     "crypto/crypto.c",
     "crypto/curve25519/asm/x25519-asm-arm.S",
     "crypto/curve25519/curve25519.c",
+    "crypto/curve25519/curve25519_64_adx.c",
     "crypto/curve25519/curve25519_tables.h",
     "crypto/curve25519/internal.h",
     "crypto/fipsmodule/aes/aes_nohw.c",
@@ -136,6 +137,8 @@ include = [
     "tests/rsa_test_public_key_2048.der",
     "tests/rsa_test_public_key_2048_debug.txt",
     "tests/rsa_test_public_modulus.bin",
+    "third_party/fiat/asm/fiat_curve25519_adx_mul.S",
+    "third_party/fiat/asm/fiat_curve25519_adx_square.S",
     "third_party/fiat/curve25519_32.h",
     "third_party/fiat/curve25519_64.h",
     "third_party/fiat/curve25519_64_msvc.h",

build.rs

@@ -57,6 +57,7 @@ const RING_SRCS: &[(&[&str], &str)] = &[
     (&[X86], "crypto/fipsmodule/modes/asm/ghash-x86.pl"),
 
     (&[X86_64], "crypto/chacha/asm/chacha-x86_64.pl"),
+    (&[X86_64], "crypto/curve25519/curve25519_64_adx.c"),
     (&[X86_64], "crypto/fipsmodule/aes/asm/aesni-x86_64.pl"),
     (&[X86_64], "crypto/fipsmodule/aes/asm/vpaes-x86_64.pl"),
     (&[X86_64], "crypto/fipsmodule/bn/asm/x86_64-mont.pl"),
@@ -67,6 +68,8 @@ const RING_SRCS: &[(&[&str], &str)] = &[
     (&[X86_64], "crypto/poly1305/poly1305_vec.c"),
     (&[X86_64], SHA512_X86_64),
     (&[X86_64], "crypto/cipher_extra/asm/chacha20_poly1305_x86_64.pl"),
+    (&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_mul.S"),
+    (&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_square.S"),
 
     (&[AARCH64, X86_64], "crypto/fipsmodule/ec/p256-nistz.c"),
 
@@ -954,10 +957,13 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
         "bn_sqr8x_internal",
         "bn_sqrx8x_internal",
         "bsaes_ctr32_encrypt_blocks",
+        "bssl_constant_time_test_conditional_memcpy",
         "bssl_constant_time_test_conditional_memxor",
         "bssl_constant_time_test_main",
         "chacha20_poly1305_open",
         "chacha20_poly1305_seal",
+        "fiat_curve25519_adx_mul",
+        "fiat_curve25519_adx_square",
         "gcm_ghash_avx",
         "gcm_ghash_clmul",
         "gcm_ghash_neon",
@@ -966,6 +972,7 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
         "gcm_init_avx",
         "gcm_init_clmul",
         "gcm_init_neon",
+        "k25519Precomp",
         "limbs_mul_add_limb",
         "little_endian_bytes_from_scalar",
         "ecp_nistz256_neg",
@@ -1005,10 +1012,12 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
         "x25519_ge_double_scalarmult_vartime",
         "x25519_ge_frombytes_vartime",
         "x25519_ge_scalarmult_base",
+        "x25519_ge_scalarmult_base_adx",
         "x25519_public_from_private_generic_masked",
         "x25519_sc_mask",
         "x25519_sc_muladd",
         "x25519_sc_reduce",
+        "x25519_scalar_mult_adx",
         "x25519_scalar_mult_generic_masked",
     ];
 

crypto/curve25519/curve25519.c

@@ -733,7 +733,8 @@ static void x25519_ge_scalarmult_small_precomp(
   }
 }
 
-void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
+void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) {
+  (void)use_adx;
   x25519_ge_scalarmult_small_precomp(h, a, k25519SmallPrecomp);
 }
 
@@ -777,10 +778,9 @@ static void table_select(ge_precomp *t, const int pos, const signed char b) {
 //
 // Preconditions:
 //   a[31] <= 127
-void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
+void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) {
 #if defined(BORINGSSL_FE25519_ADX)
-  if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() &&
+  if (use_adx) {
-      CRYPTO_is_ADX_capable()) {
     uint8_t t[4][32];
     x25519_ge_scalarmult_base_adx(t, a);
     fiat_25519_from_bytes(h->X.v, t[0]);
@@ -789,6 +789,8 @@ void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
     fiat_25519_from_bytes(h->T.v, t[3]);
     return;
   }
+#else
+  (void)use_adx;
 #endif
   signed char e[64];
   signed char carry;
@@ -1864,12 +1866,13 @@ void x25519_scalar_mult_generic_masked(uint8_t out[32],
 }
 
 void x25519_public_from_private_generic_masked(uint8_t out_public_value[32],
-                                                   const uint8_t private_key_masked[32]) {
+                                               const uint8_t private_key_masked[32],
+                                               int use_adx) {
   uint8_t e[32];
   OPENSSL_memcpy(e, private_key_masked, 32);
 
   ge_p3 A;
-  x25519_ge_scalarmult_base(&A, e);
+  x25519_ge_scalarmult_base(&A, e, use_adx);
 
   // We only need the u-coordinate of the curve25519 point. The map is
   // u=(y+1)/(1-y). Since y=Y/Z, this gives u=(Z+Y)/(Z-Y).

crypto/curve25519/curve25519_64_adx.c

@@ -14,5 +14,11 @@
 
 #include "internal.h"
 #if defined(BORINGSSL_FE25519_ADX)
+
+#pragma GCC diagnostic ignored "-Wconversion"
+#pragma GCC diagnostic ignored "-Wpedantic"
+#pragma GCC diagnostic ignored "-Wsign-conversion"
+#pragma GCC diagnostic ignored "-Wunused-but-set-parameter"
+
 #include "../../third_party/fiat/curve25519_64_adx.h"
 #endif

crypto/curve25519/curve25519_tables.h

@@ -142,7 +142,7 @@ static const uint8_t k25519SmallPrecomp[15 * 2 * 32] = {
 #else
 
 // k25519Precomp[i][j] = (j+1)*256^i*B
-static const uint8_t k25519Precomp[32][8][3][32] = {
+const uint8_t k25519Precomp[32][8][3][32] = {
     {
         {
             {0x85, 0x3b, 0x8c, 0xf5, 0xc6, 0x93, 0xbc, 0x2f, 0x19, 0xe,  0x8c,

crypto/curve25519/internal.h

@@ -28,6 +28,27 @@ void x25519_NEON(uint8_t out[32], const uint8_t scalar[32],
                      const uint8_t point[32]);
 #endif
 
+#if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_SMALL) && !defined(OPENSSL_WINDOWS) && \
+    defined(__GNUC__) && defined(__x86_64__)
+#define BORINGSSL_FE25519_ADX
+
+// fiat_curve25519_adx_mul is defined in
+// third_party/fiat/asm/fiat_curve25519_adx_mul.S
+void __attribute__((sysv_abi))
+fiat_curve25519_adx_mul(uint64_t out[4], const uint64_t in1[4],
+                        const uint64_t in2[4]);
+
+// fiat_curve25519_adx_square is defined in
+// third_party/fiat/asm/fiat_curve25519_adx_square.S
+void __attribute__((sysv_abi))
+fiat_curve25519_adx_square(uint64_t out[4], const uint64_t in[4]);
+
+// x25519_scalar_mult_adx is defined in third_party/fiat/curve25519_64_adx.h
+void x25519_scalar_mult_adx(uint8_t out[32], const uint8_t scalar[32],
+                            const uint8_t point[32]);
+void x25519_ge_scalarmult_base_adx(uint8_t h[4][32], const uint8_t a[32]);
+#endif
+
 #if defined(OPENSSL_64_BIT)
 // An element t,
 // entries t[0]...t[4], represents the integer t[0]+2^51 t[1]+2^102 t[2]+2^153
@@ -109,4 +130,6 @@ typedef struct {
   fe_loose T2d;
 } ge_cached;
 
+extern const uint8_t k25519Precomp[32][8][3][32];
+
 #endif  // OPENSSL_HEADER_CURVE25519_INTERNAL_H

src/cpu/intel.rs

@@ -41,6 +41,21 @@ impl Feature {
     }
 }
 
+pub(crate) const ADX: Feature = Feature {
+    word: 2,
+    mask: 1 << 19,
+};
+
+pub(crate) const BMI1: Feature = Feature {
+    word: 2,
+    mask: 1 << 3,
+};
+
+pub(crate) const BMI2: Feature = Feature {
+    word: 2,
+    mask: 1 << 8,
+};
+
 pub(crate) const FXSR: Feature = Feature {
     word: 0,
     mask: 1 << 24,

src/ec/curve25519/ed25519/signing.rs

@@ -16,7 +16,7 @@
 
 use super::{super::ops::*, eddsa_digest, ED25519_PUBLIC_KEY_LEN};
 use crate::{
-    digest, error,
+    cpu, digest, error,
     io::der,
     pkcs8, rand,
     signature::{self, KeyPair as SigningKeyPair},
@@ -176,7 +176,7 @@ impl Ed25519KeyPair {
         let private_scalar =
             MaskedScalar::from_bytes_masked(private_scalar.try_into().unwrap()).into();
 
-        let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar);
+        let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar, cpu::features());
 
         Self {
             private_scalar,
@@ -207,7 +207,7 @@ impl Ed25519KeyPair {
             };
             let nonce = Scalar::from_sha512_digest_reduced(nonce);
 
-            let r = ExtPoint::from_scalarmult_base_consttime(&nonce);
+            let r = ExtPoint::from_scalarmult_base_consttime(&nonce, cpu::features());
             signature_r.copy_from_slice(&r.into_encoded_point());
             let hram_digest = eddsa_digest(signature_r, self.public_key.as_ref(), msg);
             let hram = Scalar::from_sha512_digest_reduced(hram_digest);

src/ec/curve25519/ops.rs

@@ -17,7 +17,7 @@
 
 pub use super::scalar::{MaskedScalar, Scalar, SCALAR_LEN};
 use crate::{
-    bssl, error,
+    bssl, c, cpu, error,
     limb::{Limb, LIMB_BITS},
 };
 use core::marker::PhantomData;
@@ -74,7 +74,7 @@ pub struct ExtPoint {
 
 impl ExtPoint {
     // Returns the result of multiplying the base point by the scalar in constant time.
-    pub fn from_scalarmult_base_consttime(scalar: &Scalar) -> Self {
+    pub(super) fn from_scalarmult_base_consttime(scalar: &Scalar, cpu: cpu::Features) -> Self {
         let mut r = Self {
             x: Elem::zero(),
             y: Elem::zero(),
@@ -82,10 +82,10 @@ impl ExtPoint {
             t: Elem::zero(),
         };
         prefixed_extern! {
-            fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar);
+            fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar, has_fe25519_adx: c::int);
         }
         unsafe {
-            x25519_ge_scalarmult_base(&mut r, scalar);
+            x25519_ge_scalarmult_base(&mut r, scalar, has_fe25519_adx(cpu).into());
         }
         r
     }
@@ -157,6 +157,14 @@ fn encode_point(x: Elem<T>, y: Elem<T>, z: Elem<T>) -> EncodedPoint {
     bytes
 }
 
+#[inline]
+pub(super) fn has_fe25519_adx(cpu: cpu::Features) -> bool {
+    cfg!(all(target_arch = "x86_64", not(target_os = "windows")))
+        && cpu::intel::ADX.available(cpu)
+        && cpu::intel::BMI1.available(cpu)
+        && cpu::intel::BMI2.available(cpu)
+}
+
 prefixed_extern! {
     fn x25519_fe_invert(out: &mut Elem<T>, z: &Elem<T>);
     fn x25519_fe_isnegative(elem: &Elem<T>) -> u8;

src/ec/curve25519/x25519.rs

@@ -15,7 +15,7 @@
 //! X25519 Key agreement.
 
 use super::{ops, scalar::SCALAR_LEN};
-use crate::{agreement, constant_time, cpu, ec, error, rand};
+use crate::{agreement, c, constant_time, cpu, ec, error, rand};
 
 static CURVE25519: ec::Curve = ec::Curve {
     public_key_len: PUBLIC_KEY_LEN,
@@ -58,7 +58,6 @@ fn x25519_public_from_private(
 ) -> Result<(), error::Unspecified> {
     let public_out = public_out.try_into()?;
 
-    #[cfg(target_arch = "arm")]
     let cpu_features = private_key.cpu_features;
 
     let private_key: &[u8; SCALAR_LEN] = private_key.bytes_less_safe().try_into()?;
@@ -80,10 +79,15 @@ fn x25519_public_from_private(
         fn x25519_public_from_private_generic_masked(
             public_key_out: &mut PublicKey,
             private_key: &PrivateKey,
+            use_adx: c::int,
         );
     }
     unsafe {
-        x25519_public_from_private_generic_masked(public_out, &private_key);
+        x25519_public_from_private_generic_masked(
+            public_out,
+            &private_key,
+            ops::has_fe25519_adx(cpu_features).into(),
+        );
     }
 
     Ok(())