// Copyright 2018 Brian Smith. // // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. use super::{Aad, Block, BLOCK_LEN}; use crate::cpu; use libc::size_t; #[repr(transparent)] pub struct Key(GCM128_KEY); impl Key { pub(super) fn new(mut h_be: Block, cpu_features: cpu::Features) -> Self { let h = h_be.u64s_be_to_native(); let mut key = Self(GCM128_KEY { Htable: [u128 { hi: 0, lo: 0 }; GCM128_HTABLE_LEN], }); match detect_implementation(cpu_features) { #[cfg(target_arch = "x86_64")] Implementation::CLMUL if has_avx_movbe(cpu_features) => { extern "C" { fn GFp_gcm_init_avx(key: &mut Key, h: &[u64; 2]); } unsafe { GFp_gcm_init_avx(&mut key, &h); } } Implementation::CLMUL => { extern "C" { fn GFp_gcm_init_clmul(key: &mut Key, h: &[u64; 2]); } unsafe { GFp_gcm_init_clmul(&mut key, &h); } } #[cfg(any(target_arch = "arm"))] Implementation::NEON => { extern "C" { fn GFp_gcm_init_neon(key: &mut Key, h: &[u64; 2]); } unsafe { GFp_gcm_init_neon(&mut key, &h); } } Implementation::Fallback => { extern "C" { fn GFp_gcm_init_4bit(key: &mut Key, h: &[u64; 2]); } unsafe { GFp_gcm_init_4bit(&mut key, &h); } } } key } } #[repr(transparent)] pub struct Context { inner: GCM128_CONTEXT, cpu_features: cpu::Features, } impl Context { pub(crate) fn new(key: &Key, aad: Aad<&[u8]>, cpu_features: cpu::Features) -> Self { let mut ctx = Context { inner: GCM128_CONTEXT { Xi: Block::zero(), H_unused: Block::zero(), key: key.0.clone(), }, cpu_features, }; for ad in aad.0.chunks(BLOCK_LEN) { let mut block = Block::zero(); block.partial_copy_from(ad); ctx.update_block(block); } ctx } pub fn update_blocks(&mut self, input: &[u8]) { debug_assert!(input.len() > 0); debug_assert_eq!(input.len() % BLOCK_LEN, 0); let key_aliasing: *const GCM128_KEY = &self.inner.key; match detect_implementation(self.cpu_features) { #[cfg(target_arch = "x86_64")] Implementation::CLMUL if has_avx_movbe(self.cpu_features) => { extern "C" { fn GFp_gcm_ghash_avx( ctx: &mut Context, h_table: *const GCM128_KEY, inp: *const u8, len: size_t, ); } unsafe { GFp_gcm_ghash_avx(self, key_aliasing, input.as_ptr(), input.len()); } } Implementation::CLMUL => { extern "C" { fn GFp_gcm_ghash_clmul( ctx: &mut Context, h_table: *const GCM128_KEY, inp: *const u8, len: size_t, ); } unsafe { GFp_gcm_ghash_clmul(self, key_aliasing, input.as_ptr(), input.len()); } } #[cfg(any(target_arch = "arm"))] Implementation::NEON => { extern "C" { fn GFp_gcm_ghash_neon( ctx: &mut Context, h_table: *const GCM128_KEY, inp: *const u8, len: size_t, ); } unsafe { GFp_gcm_ghash_neon(self, key_aliasing, input.as_ptr(), input.len()); } } Implementation::Fallback => { extern "C" { fn GFp_gcm_ghash_4bit( ctx: &mut Context, h_table: *const GCM128_KEY, inp: *const u8, len: size_t, ); } unsafe { GFp_gcm_ghash_4bit(self, key_aliasing, input.as_ptr(), input.len()); } } } } pub fn update_block(&mut self, a: Block) { self.inner.Xi.bitxor_assign(a); let key_aliasing: *const GCM128_KEY = &self.inner.key; match detect_implementation(self.cpu_features) { Implementation::CLMUL => { extern "C" { fn GFp_gcm_gmult_clmul(ctx: &mut Context, Htable: *const GCM128_KEY); } unsafe { GFp_gcm_gmult_clmul(self, key_aliasing); } } #[cfg(any(target_arch = "arm"))] Implementation::NEON => { extern "C" { fn GFp_gcm_gmult_neon(ctx: &mut Context, Htable: *const GCM128_KEY); } unsafe { GFp_gcm_gmult_neon(self, key_aliasing); } } Implementation::Fallback => { extern "C" { fn GFp_gcm_gmult_4bit(ctx: &mut Context, Htable: *const GCM128_KEY); } unsafe { GFp_gcm_gmult_4bit(self, key_aliasing); } } } } pub(super) fn pre_finish(self, f: F) -> super::Tag where F: FnOnce(Block) -> super::Tag, { f(self.inner.Xi) } #[cfg(target_arch = "x86_64")] pub(super) fn is_avx2(&self, cpu_features: cpu::Features) -> bool { match detect_implementation(cpu_features) { Implementation::CLMUL => has_avx_movbe(self.cpu_features), _ => false, } } } // Keep in sync with `GCM128_KEY` in modes/internal.h. #[derive(Clone)] #[repr(C, align(16))] struct GCM128_KEY { Htable: [u128; GCM128_HTABLE_LEN], } #[derive(Clone, Copy)] #[repr(C)] struct u128 { hi: u64, lo: u64, } const GCM128_HTABLE_LEN: usize = 16; // Keep in sync with `GCM128_CONTEXT` in modes/internal.h. #[repr(C, align(16))] struct GCM128_CONTEXT { Xi: Block, H_unused: Block, key: GCM128_KEY, } enum Implementation { CLMUL, #[cfg(target_arch = "arm")] NEON, Fallback, } #[inline] fn detect_implementation(cpu: cpu::Features) -> Implementation { if (cpu::intel::FXSR.available(cpu) && cpu::intel::PCLMULQDQ.available(cpu)) || cpu::arm::PMULL.available(cpu) { return Implementation::CLMUL; } #[cfg(target_arch = "arm")] { if cpu::arm::NEON.available(cpu) { return Implementation::NEON; } } Implementation::Fallback } #[cfg(target_arch = "x86_64")] fn has_avx_movbe(cpu_features: cpu::Features) -> bool { return cpu::intel::AVX.available(cpu_features) && cpu::intel::MOVBE.available(cpu_features); }