From af7c37f8af7c3cbbe694de1c7a5c24c820afa142 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Mon, 2 Oct 2023 12:09:29 -0700 Subject: [PATCH] Enable FE25519_ADX Curve25519 base point mult. except on Windows. --- Cargo.toml | 3 +++ build.rs | 9 +++++++++ crypto/curve25519/curve25519.c | 15 +++++++++------ crypto/curve25519/curve25519_64_adx.c | 6 ++++++ crypto/curve25519/curve25519_tables.h | 2 +- crypto/curve25519/internal.h | 23 +++++++++++++++++++++++ src/cpu/intel.rs | 15 +++++++++++++++ src/ec/curve25519/ed25519/signing.rs | 6 +++--- src/ec/curve25519/ops.rs | 16 ++++++++++++---- src/ec/curve25519/x25519.rs | 10 +++++++--- 10 files changed, 88 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c9dbe2521..7f3d6dff5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ include = [ "crypto/crypto.c", "crypto/curve25519/asm/x25519-asm-arm.S", "crypto/curve25519/curve25519.c", + "crypto/curve25519/curve25519_64_adx.c", "crypto/curve25519/curve25519_tables.h", "crypto/curve25519/internal.h", "crypto/fipsmodule/aes/aes_nohw.c", @@ -136,6 +137,8 @@ include = [ "tests/rsa_test_public_key_2048.der", "tests/rsa_test_public_key_2048_debug.txt", "tests/rsa_test_public_modulus.bin", + "third_party/fiat/asm/fiat_curve25519_adx_mul.S", + "third_party/fiat/asm/fiat_curve25519_adx_square.S", "third_party/fiat/curve25519_32.h", "third_party/fiat/curve25519_64.h", "third_party/fiat/curve25519_64_msvc.h", diff --git a/build.rs b/build.rs index c7e05adfb..c091c84e2 100644 --- a/build.rs +++ b/build.rs @@ -57,6 +57,7 @@ const RING_SRCS: &[(&[&str], &str)] = &[ (&[X86], "crypto/fipsmodule/modes/asm/ghash-x86.pl"), (&[X86_64], "crypto/chacha/asm/chacha-x86_64.pl"), + (&[X86_64], "crypto/curve25519/curve25519_64_adx.c"), (&[X86_64], "crypto/fipsmodule/aes/asm/aesni-x86_64.pl"), (&[X86_64], "crypto/fipsmodule/aes/asm/vpaes-x86_64.pl"), (&[X86_64], "crypto/fipsmodule/bn/asm/x86_64-mont.pl"), @@ -67,6 +68,8 @@ const RING_SRCS: &[(&[&str], &str)] = &[ (&[X86_64], "crypto/poly1305/poly1305_vec.c"), (&[X86_64], SHA512_X86_64), (&[X86_64], "crypto/cipher_extra/asm/chacha20_poly1305_x86_64.pl"), + (&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_mul.S"), + (&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_square.S"), (&[AARCH64, X86_64], "crypto/fipsmodule/ec/p256-nistz.c"), @@ -954,10 +957,13 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String { "bn_sqr8x_internal", "bn_sqrx8x_internal", "bsaes_ctr32_encrypt_blocks", + "bssl_constant_time_test_conditional_memcpy", "bssl_constant_time_test_conditional_memxor", "bssl_constant_time_test_main", "chacha20_poly1305_open", "chacha20_poly1305_seal", + "fiat_curve25519_adx_mul", + "fiat_curve25519_adx_square", "gcm_ghash_avx", "gcm_ghash_clmul", "gcm_ghash_neon", @@ -966,6 +972,7 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String { "gcm_init_avx", "gcm_init_clmul", "gcm_init_neon", + "k25519Precomp", "limbs_mul_add_limb", "little_endian_bytes_from_scalar", "ecp_nistz256_neg", @@ -1005,10 +1012,12 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String { "x25519_ge_double_scalarmult_vartime", "x25519_ge_frombytes_vartime", "x25519_ge_scalarmult_base", + "x25519_ge_scalarmult_base_adx", "x25519_public_from_private_generic_masked", "x25519_sc_mask", "x25519_sc_muladd", "x25519_sc_reduce", + "x25519_scalar_mult_adx", "x25519_scalar_mult_generic_masked", ]; diff --git a/crypto/curve25519/curve25519.c b/crypto/curve25519/curve25519.c index 1ab754d3a..1b9e4c898 100644 --- a/crypto/curve25519/curve25519.c +++ b/crypto/curve25519/curve25519.c @@ -733,7 +733,8 @@ static void x25519_ge_scalarmult_small_precomp( } } -void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) { +void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) { + (void)use_adx; x25519_ge_scalarmult_small_precomp(h, a, k25519SmallPrecomp); } @@ -777,10 +778,9 @@ static void table_select(ge_precomp *t, const int pos, const signed char b) { // // Preconditions: // a[31] <= 127 -void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) { +void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) { #if defined(BORINGSSL_FE25519_ADX) - if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && - CRYPTO_is_ADX_capable()) { + if (use_adx) { uint8_t t[4][32]; x25519_ge_scalarmult_base_adx(t, a); fiat_25519_from_bytes(h->X.v, t[0]); @@ -789,6 +789,8 @@ void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) { fiat_25519_from_bytes(h->T.v, t[3]); return; } +#else + (void)use_adx; #endif signed char e[64]; signed char carry; @@ -1864,12 +1866,13 @@ void x25519_scalar_mult_generic_masked(uint8_t out[32], } void x25519_public_from_private_generic_masked(uint8_t out_public_value[32], - const uint8_t private_key_masked[32]) { + const uint8_t private_key_masked[32], + int use_adx) { uint8_t e[32]; OPENSSL_memcpy(e, private_key_masked, 32); ge_p3 A; - x25519_ge_scalarmult_base(&A, e); + x25519_ge_scalarmult_base(&A, e, use_adx); // We only need the u-coordinate of the curve25519 point. The map is // u=(y+1)/(1-y). Since y=Y/Z, this gives u=(Z+Y)/(Z-Y). diff --git a/crypto/curve25519/curve25519_64_adx.c b/crypto/curve25519/curve25519_64_adx.c index 276898964..171d3b390 100644 --- a/crypto/curve25519/curve25519_64_adx.c +++ b/crypto/curve25519/curve25519_64_adx.c @@ -14,5 +14,11 @@ #include "internal.h" #if defined(BORINGSSL_FE25519_ADX) + +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wsign-conversion" +#pragma GCC diagnostic ignored "-Wunused-but-set-parameter" + #include "../../third_party/fiat/curve25519_64_adx.h" #endif diff --git a/crypto/curve25519/curve25519_tables.h b/crypto/curve25519/curve25519_tables.h index 54b346e98..6636a36a2 100644 --- a/crypto/curve25519/curve25519_tables.h +++ b/crypto/curve25519/curve25519_tables.h @@ -142,7 +142,7 @@ static const uint8_t k25519SmallPrecomp[15 * 2 * 32] = { #else // k25519Precomp[i][j] = (j+1)*256^i*B -static const uint8_t k25519Precomp[32][8][3][32] = { +const uint8_t k25519Precomp[32][8][3][32] = { { { {0x85, 0x3b, 0x8c, 0xf5, 0xc6, 0x93, 0xbc, 0x2f, 0x19, 0xe, 0x8c, diff --git a/crypto/curve25519/internal.h b/crypto/curve25519/internal.h index 6e0b62c08..a26e9969b 100644 --- a/crypto/curve25519/internal.h +++ b/crypto/curve25519/internal.h @@ -28,6 +28,27 @@ void x25519_NEON(uint8_t out[32], const uint8_t scalar[32], const uint8_t point[32]); #endif +#if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_SMALL) && !defined(OPENSSL_WINDOWS) && \ + defined(__GNUC__) && defined(__x86_64__) +#define BORINGSSL_FE25519_ADX + +// fiat_curve25519_adx_mul is defined in +// third_party/fiat/asm/fiat_curve25519_adx_mul.S +void __attribute__((sysv_abi)) +fiat_curve25519_adx_mul(uint64_t out[4], const uint64_t in1[4], + const uint64_t in2[4]); + +// fiat_curve25519_adx_square is defined in +// third_party/fiat/asm/fiat_curve25519_adx_square.S +void __attribute__((sysv_abi)) +fiat_curve25519_adx_square(uint64_t out[4], const uint64_t in[4]); + +// x25519_scalar_mult_adx is defined in third_party/fiat/curve25519_64_adx.h +void x25519_scalar_mult_adx(uint8_t out[32], const uint8_t scalar[32], + const uint8_t point[32]); +void x25519_ge_scalarmult_base_adx(uint8_t h[4][32], const uint8_t a[32]); +#endif + #if defined(OPENSSL_64_BIT) // An element t, // entries t[0]...t[4], represents the integer t[0]+2^51 t[1]+2^102 t[2]+2^153 @@ -109,4 +130,6 @@ typedef struct { fe_loose T2d; } ge_cached; +extern const uint8_t k25519Precomp[32][8][3][32]; + #endif // OPENSSL_HEADER_CURVE25519_INTERNAL_H diff --git a/src/cpu/intel.rs b/src/cpu/intel.rs index 3a75ac2f5..36366ba15 100644 --- a/src/cpu/intel.rs +++ b/src/cpu/intel.rs @@ -41,6 +41,21 @@ impl Feature { } } +pub(crate) const ADX: Feature = Feature { + word: 2, + mask: 1 << 19, +}; + +pub(crate) const BMI1: Feature = Feature { + word: 2, + mask: 1 << 3, +}; + +pub(crate) const BMI2: Feature = Feature { + word: 2, + mask: 1 << 8, +}; + pub(crate) const FXSR: Feature = Feature { word: 0, mask: 1 << 24, diff --git a/src/ec/curve25519/ed25519/signing.rs b/src/ec/curve25519/ed25519/signing.rs index cacdcca4f..df2bf93fc 100644 --- a/src/ec/curve25519/ed25519/signing.rs +++ b/src/ec/curve25519/ed25519/signing.rs @@ -16,7 +16,7 @@ use super::{super::ops::*, eddsa_digest, ED25519_PUBLIC_KEY_LEN}; use crate::{ - digest, error, + cpu, digest, error, io::der, pkcs8, rand, signature::{self, KeyPair as SigningKeyPair}, @@ -176,7 +176,7 @@ impl Ed25519KeyPair { let private_scalar = MaskedScalar::from_bytes_masked(private_scalar.try_into().unwrap()).into(); - let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar); + let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar, cpu::features()); Self { private_scalar, @@ -207,7 +207,7 @@ impl Ed25519KeyPair { }; let nonce = Scalar::from_sha512_digest_reduced(nonce); - let r = ExtPoint::from_scalarmult_base_consttime(&nonce); + let r = ExtPoint::from_scalarmult_base_consttime(&nonce, cpu::features()); signature_r.copy_from_slice(&r.into_encoded_point()); let hram_digest = eddsa_digest(signature_r, self.public_key.as_ref(), msg); let hram = Scalar::from_sha512_digest_reduced(hram_digest); diff --git a/src/ec/curve25519/ops.rs b/src/ec/curve25519/ops.rs index 20dfa58b9..3536ebc8c 100644 --- a/src/ec/curve25519/ops.rs +++ b/src/ec/curve25519/ops.rs @@ -17,7 +17,7 @@ pub use super::scalar::{MaskedScalar, Scalar, SCALAR_LEN}; use crate::{ - bssl, error, + bssl, c, cpu, error, limb::{Limb, LIMB_BITS}, }; use core::marker::PhantomData; @@ -74,7 +74,7 @@ pub struct ExtPoint { impl ExtPoint { // Returns the result of multiplying the base point by the scalar in constant time. - pub fn from_scalarmult_base_consttime(scalar: &Scalar) -> Self { + pub(super) fn from_scalarmult_base_consttime(scalar: &Scalar, cpu: cpu::Features) -> Self { let mut r = Self { x: Elem::zero(), y: Elem::zero(), @@ -82,10 +82,10 @@ impl ExtPoint { t: Elem::zero(), }; prefixed_extern! { - fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar); + fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar, has_fe25519_adx: c::int); } unsafe { - x25519_ge_scalarmult_base(&mut r, scalar); + x25519_ge_scalarmult_base(&mut r, scalar, has_fe25519_adx(cpu).into()); } r } @@ -157,6 +157,14 @@ fn encode_point(x: Elem, y: Elem, z: Elem) -> EncodedPoint { bytes } +#[inline] +pub(super) fn has_fe25519_adx(cpu: cpu::Features) -> bool { + cfg!(all(target_arch = "x86_64", not(target_os = "windows"))) + && cpu::intel::ADX.available(cpu) + && cpu::intel::BMI1.available(cpu) + && cpu::intel::BMI2.available(cpu) +} + prefixed_extern! { fn x25519_fe_invert(out: &mut Elem, z: &Elem); fn x25519_fe_isnegative(elem: &Elem) -> u8; diff --git a/src/ec/curve25519/x25519.rs b/src/ec/curve25519/x25519.rs index fe63c8857..665c1faa2 100644 --- a/src/ec/curve25519/x25519.rs +++ b/src/ec/curve25519/x25519.rs @@ -15,7 +15,7 @@ //! X25519 Key agreement. use super::{ops, scalar::SCALAR_LEN}; -use crate::{agreement, constant_time, cpu, ec, error, rand}; +use crate::{agreement, c, constant_time, cpu, ec, error, rand}; static CURVE25519: ec::Curve = ec::Curve { public_key_len: PUBLIC_KEY_LEN, @@ -58,7 +58,6 @@ fn x25519_public_from_private( ) -> Result<(), error::Unspecified> { let public_out = public_out.try_into()?; - #[cfg(target_arch = "arm")] let cpu_features = private_key.cpu_features; let private_key: &[u8; SCALAR_LEN] = private_key.bytes_less_safe().try_into()?; @@ -80,10 +79,15 @@ fn x25519_public_from_private( fn x25519_public_from_private_generic_masked( public_key_out: &mut PublicKey, private_key: &PrivateKey, + use_adx: c::int, ); } unsafe { - x25519_public_from_private_generic_masked(public_out, &private_key); + x25519_public_from_private_generic_masked( + public_out, + &private_key, + ops::has_fe25519_adx(cpu_features).into(), + ); } Ok(())