diff --git a/lib/crypto/src/arch/x86_64.rs b/lib/crypto/src/arch/x86_64.rs index b1270fa473d..1778f7a60ca 100644 --- a/lib/crypto/src/arch/x86_64.rs +++ b/lib/crypto/src/arch/x86_64.rs @@ -127,16 +127,25 @@ impl Features { } } +/// The cache publishes only this public word, not any other memory. +static FEATURES: AtomicU32 = AtomicU32::new(0); + +/// Inline: every instruction path asks, several times per message. +#[inline(always)] pub(super) fn features() -> Features { if cfg!(feature = "portable") { return Features(Features::READ); } - // The cache publishes only this public word, not any other memory. - static FEATURES: AtomicU32 = AtomicU32::new(0); let cached = FEATURES.load(Ordering::Relaxed); if cached & Features::READ != 0 { return Features(cached); } + first_features() +} + +#[cold] +#[inline(never)] +fn first_features() -> Features { let read = Features::read(); FEATURES.store(read.0, Ordering::Relaxed); read diff --git a/lib/crypto/src/arch/x86_64/chacha.rs b/lib/crypto/src/arch/x86_64/chacha.rs index c38d54609ec..6aa4363090d 100644 --- a/lib/crypto/src/arch/x86_64/chacha.rs +++ b/lib/crypto/src/arch/x86_64/chacha.rs @@ -393,14 +393,28 @@ pub(super) unsafe fn seal( 384 * n } +/// The state with a zero counter. Indexing, not slice copies or iterator +/// adaptors, whose debug precondition checks the shipped profile calls out +/// of line on every message. fn initial(key: &[u8; 32], nonce: &[u8; 8]) -> crate::secret::Secret<16, u32> { - let mut initial = [0u32; 16]; - initial[..4].copy_from_slice(&SIGMA); - for (word, bytes) in initial[4..12].iter_mut().zip(key.as_chunks::<4>().0) { - *word = u32::from_le_bytes(*bytes); - } - for (word, bytes) in initial[14..].iter_mut().zip(nonce.as_chunks::<4>().0) { - *word = u32::from_le_bytes(*bytes); - } - crate::secret::Secret(initial) + let k = |i: usize| u32::from_le_bytes([key[4 * i], key[4 * i + 1], key[4 * i + 2], key[4 * i + 3]]); + let n = |i: usize| u32::from_le_bytes([nonce[4 * i], nonce[4 * i + 1], nonce[4 * i + 2], nonce[4 * i + 3]]); + crate::secret::Secret([ + SIGMA[0], + SIGMA[1], + SIGMA[2], + SIGMA[3], + k(0), + k(1), + k(2), + k(3), + k(4), + k(5), + k(6), + k(7), + 0, + 0, + n(0), + n(1), + ]) }