yggdrasil/ring
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable FE25519_ADX Curve25519 base point mult. except on Windows.

Browse Source
This commit is contained in:
Brian Smith 2023-10-02 12:09:29 -07:00
parent 5aa224c4bc
commit af7c37f8af
10 changed files with 88 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@ -45,6 +45,7 @@ include = [
"crypto/crypto.c",
"crypto/curve25519/asm/x25519-asm-arm.S",
"crypto/curve25519/curve25519.c",
"crypto/curve25519/curve25519_64_adx.c",
"crypto/curve25519/curve25519_tables.h",
"crypto/curve25519/internal.h",
"crypto/fipsmodule/aes/aes_nohw.c",
@ -136,6 +137,8 @@ include = [
"tests/rsa_test_public_key_2048.der",
"tests/rsa_test_public_key_2048_debug.txt",
"tests/rsa_test_public_modulus.bin",
"third_party/fiat/asm/fiat_curve25519_adx_mul.S",
"third_party/fiat/asm/fiat_curve25519_adx_square.S",
"third_party/fiat/curve25519_32.h",
"third_party/fiat/curve25519_64.h",
"third_party/fiat/curve25519_64_msvc.h",

9
build.rs
View File

@ -57,6 +57,7 @@ const RING_SRCS: &[(&[&str], &str)] = &[
(&[X86], "crypto/fipsmodule/modes/asm/ghash-x86.pl"),
(&[X86_64], "crypto/chacha/asm/chacha-x86_64.pl"),
(&[X86_64], "crypto/curve25519/curve25519_64_adx.c"),
(&[X86_64], "crypto/fipsmodule/aes/asm/aesni-x86_64.pl"),
(&[X86_64], "crypto/fipsmodule/aes/asm/vpaes-x86_64.pl"),
(&[X86_64], "crypto/fipsmodule/bn/asm/x86_64-mont.pl"),
@ -67,6 +68,8 @@ const RING_SRCS: &[(&[&str], &str)] = &[
(&[X86_64], "crypto/poly1305/poly1305_vec.c"),
(&[X86_64], SHA512_X86_64),
(&[X86_64], "crypto/cipher_extra/asm/chacha20_poly1305_x86_64.pl"),
(&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_mul.S"),
(&[X86_64], "third_party/fiat/asm/fiat_curve25519_adx_square.S"),
(&[AARCH64, X86_64], "crypto/fipsmodule/ec/p256-nistz.c"),
@ -954,10 +957,13 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
"bn_sqr8x_internal",
"bn_sqrx8x_internal",
"bsaes_ctr32_encrypt_blocks",
"bssl_constant_time_test_conditional_memcpy",
"bssl_constant_time_test_conditional_memxor",
"bssl_constant_time_test_main",
"chacha20_poly1305_open",
"chacha20_poly1305_seal",
"fiat_curve25519_adx_mul",
"fiat_curve25519_adx_square",
"gcm_ghash_avx",
"gcm_ghash_clmul",
"gcm_ghash_neon",
@ -966,6 +972,7 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
"gcm_init_avx",
"gcm_init_clmul",
"gcm_init_neon",
"k25519Precomp",
"limbs_mul_add_limb",
"little_endian_bytes_from_scalar",
"ecp_nistz256_neg",
@ -1005,10 +1012,12 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
"x25519_ge_double_scalarmult_vartime",
"x25519_ge_frombytes_vartime",
"x25519_ge_scalarmult_base",
"x25519_ge_scalarmult_base_adx",
"x25519_public_from_private_generic_masked",
"x25519_sc_mask",
"x25519_sc_muladd",
"x25519_sc_reduce",
"x25519_scalar_mult_adx",
"x25519_scalar_mult_generic_masked",
];

15
crypto/curve25519/curve25519.c
View File

@ -733,7 +733,8 @@ static void x25519_ge_scalarmult_small_precomp(
}
}
void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) {
(void)use_adx;
x25519_ge_scalarmult_small_precomp(h, a, k25519SmallPrecomp);
}
@ -777,10 +778,9 @@ static void table_select(ge_precomp *t, const int pos, const signed char b) {
//
// Preconditions:
// a[31] <= 127
void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32], int use_adx) {
#if defined(BORINGSSL_FE25519_ADX)
if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() &&
CRYPTO_is_ADX_capable()) {
if (use_adx) {
uint8_t t[4][32];
x25519_ge_scalarmult_base_adx(t, a);
fiat_25519_from_bytes(h->X.v, t[0]);
@ -789,6 +789,8 @@ void x25519_ge_scalarmult_base(ge_p3 *h, const uint8_t a[32]) {
fiat_25519_from_bytes(h->T.v, t[3]);
return;
}
#else
(void)use_adx;
#endif
signed char e[64];
signed char carry;
@ -1864,12 +1866,13 @@ void x25519_scalar_mult_generic_masked(uint8_t out[32],
}
void x25519_public_from_private_generic_masked(uint8_t out_public_value[32],
const uint8_t private_key_masked[32]) {
const uint8_t private_key_masked[32],
int use_adx) {
uint8_t e[32];
OPENSSL_memcpy(e, private_key_masked, 32);
ge_p3 A;
x25519_ge_scalarmult_base(&A, e);
x25519_ge_scalarmult_base(&A, e, use_adx);
// We only need the u-coordinate of the curve25519 point. The map is
// u=(y+1)/(1-y). Since y=Y/Z, this gives u=(Z+Y)/(Z-Y).

6
crypto/curve25519/curve25519_64_adx.c
View File

@ -14,5 +14,11 @@
#include "internal.h"
#if defined(BORINGSSL_FE25519_ADX)
#pragma GCC diagnostic ignored "-Wconversion"
#pragma GCC diagnostic ignored "-Wpedantic"
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wunused-but-set-parameter"
#include "../../third_party/fiat/curve25519_64_adx.h"
#endif

2
crypto/curve25519/curve25519_tables.h generated
View File

@ -142,7 +142,7 @@ static const uint8_t k25519SmallPrecomp[15 * 2 * 32] = {
#else
// k25519Precomp[i][j] = (j+1)*256^i*B
static const uint8_t k25519Precomp[32][8][3][32] = {
const uint8_t k25519Precomp[32][8][3][32] = {
{
{
{0x85, 0x3b, 0x8c, 0xf5, 0xc6, 0x93, 0xbc, 0x2f, 0x19, 0xe, 0x8c,

23
crypto/curve25519/internal.h
View File

@ -28,6 +28,27 @@ void x25519_NEON(uint8_t out[32], const uint8_t scalar[32],
const uint8_t point[32]);
#endif
#if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_SMALL) && !defined(OPENSSL_WINDOWS) && \
defined(__GNUC__) && defined(__x86_64__)
#define BORINGSSL_FE25519_ADX
// fiat_curve25519_adx_mul is defined in
// third_party/fiat/asm/fiat_curve25519_adx_mul.S
void __attribute__((sysv_abi))
fiat_curve25519_adx_mul(uint64_t out[4], const uint64_t in1[4],
const uint64_t in2[4]);
// fiat_curve25519_adx_square is defined in
// third_party/fiat/asm/fiat_curve25519_adx_square.S
void __attribute__((sysv_abi))
fiat_curve25519_adx_square(uint64_t out[4], const uint64_t in[4]);
// x25519_scalar_mult_adx is defined in third_party/fiat/curve25519_64_adx.h
void x25519_scalar_mult_adx(uint8_t out[32], const uint8_t scalar[32],
const uint8_t point[32]);
void x25519_ge_scalarmult_base_adx(uint8_t h[4][32], const uint8_t a[32]);
#endif
#if defined(OPENSSL_64_BIT)
// An element t,
// entries t[0]...t[4], represents the integer t[0]+2^51 t[1]+2^102 t[2]+2^153
@ -109,4 +130,6 @@ typedef struct {
fe_loose T2d;
} ge_cached;
extern const uint8_t k25519Precomp[32][8][3][32];
#endif // OPENSSL_HEADER_CURVE25519_INTERNAL_H

15
src/cpu/intel.rs
View File

@ -41,6 +41,21 @@ impl Feature {
}
}
pub(crate) const ADX: Feature = Feature {
word: 2,
mask: 1 << 19,
};
pub(crate) const BMI1: Feature = Feature {
word: 2,
mask: 1 << 3,
};
pub(crate) const BMI2: Feature = Feature {
word: 2,
mask: 1 << 8,
};
pub(crate) const FXSR: Feature = Feature {
word: 0,
mask: 1 << 24,

6
src/ec/curve25519/ed25519/signing.rs
View File

@ -16,7 +16,7 @@
use super::{super::ops::*, eddsa_digest, ED25519_PUBLIC_KEY_LEN};
use crate::{
digest, error,
cpu, digest, error,
io::der,
pkcs8, rand,
signature::{self, KeyPair as SigningKeyPair},
@ -176,7 +176,7 @@ impl Ed25519KeyPair {
let private_scalar =
MaskedScalar::from_bytes_masked(private_scalar.try_into().unwrap()).into();
let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar);
let a = ExtPoint::from_scalarmult_base_consttime(&private_scalar, cpu::features());
Self {
private_scalar,
@ -207,7 +207,7 @@ impl Ed25519KeyPair {
};
let nonce = Scalar::from_sha512_digest_reduced(nonce);
let r = ExtPoint::from_scalarmult_base_consttime(&nonce);
let r = ExtPoint::from_scalarmult_base_consttime(&nonce, cpu::features());
signature_r.copy_from_slice(&r.into_encoded_point());
let hram_digest = eddsa_digest(signature_r, self.public_key.as_ref(), msg);
let hram = Scalar::from_sha512_digest_reduced(hram_digest);

16
src/ec/curve25519/ops.rs
View File

@ -17,7 +17,7 @@
pub use super::scalar::{MaskedScalar, Scalar, SCALAR_LEN};
use crate::{
bssl, error,
bssl, c, cpu, error,
limb::{Limb, LIMB_BITS},
};
use core::marker::PhantomData;
@ -74,7 +74,7 @@ pub struct ExtPoint {
impl ExtPoint {
// Returns the result of multiplying the base point by the scalar in constant time.
pub fn from_scalarmult_base_consttime(scalar: &Scalar) -> Self {
pub(super) fn from_scalarmult_base_consttime(scalar: &Scalar, cpu: cpu::Features) -> Self {
let mut r = Self {
x: Elem::zero(),
y: Elem::zero(),
@ -82,10 +82,10 @@ impl ExtPoint {
t: Elem::zero(),
};
prefixed_extern! {
fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar);
fn x25519_ge_scalarmult_base(h: &mut ExtPoint, a: &Scalar, has_fe25519_adx: c::int);
}
unsafe {
x25519_ge_scalarmult_base(&mut r, scalar);
x25519_ge_scalarmult_base(&mut r, scalar, has_fe25519_adx(cpu).into());
}
r
}
@ -157,6 +157,14 @@ fn encode_point(x: Elem<T>, y: Elem<T>, z: Elem<T>) -> EncodedPoint {
bytes
}
#[inline]
pub(super) fn has_fe25519_adx(cpu: cpu::Features) -> bool {
cfg!(all(target_arch = "x86_64", not(target_os = "windows")))
&& cpu::intel::ADX.available(cpu)
&& cpu::intel::BMI1.available(cpu)
&& cpu::intel::BMI2.available(cpu)
}
prefixed_extern! {
fn x25519_fe_invert(out: &mut Elem<T>, z: &Elem<T>);
fn x25519_fe_isnegative(elem: &Elem<T>) -> u8;

10
src/ec/curve25519/x25519.rs
View File

@ -15,7 +15,7 @@
//! X25519 Key agreement.
use super::{ops, scalar::SCALAR_LEN};
use crate::{agreement, constant_time, cpu, ec, error, rand};
use crate::{agreement, c, constant_time, cpu, ec, error, rand};
static CURVE25519: ec::Curve = ec::Curve {
public_key_len: PUBLIC_KEY_LEN,
@ -58,7 +58,6 @@ fn x25519_public_from_private(
) -> Result<(), error::Unspecified> {
let public_out = public_out.try_into()?;
#[cfg(target_arch = "arm")]
let cpu_features = private_key.cpu_features;
let private_key: &[u8; SCALAR_LEN] = private_key.bytes_less_safe().try_into()?;
@ -80,10 +79,15 @@ fn x25519_public_from_private(
fn x25519_public_from_private_generic_masked(
public_key_out: &mut PublicKey,
private_key: &PrivateKey,
use_adx: c::int,
);
}
unsafe {
x25519_public_from_private_generic_masked(public_out, &private_key);
x25519_public_from_private_generic_masked(
public_out,
&private_key,
ops::has_fe25519_adx(cpu_features).into(),
);
}
Ok(())