From 74aa231bffa38098b287fb6f423593e55b92e980 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 22 Oct 2026 14:43:07 +0200 Subject: [PATCH] crypto: no out-of-line calls on ChaCha20's per-message path In the shipped profile every ChaCha20 message called out of line: - `features()` once per dispatch, several times per message; - the debug precondition check of `offset_from_unsigned` four times in the AVX state's construction, from `copy_from_slice` and `zip` over slice chunks. These are part of the per-message constant the track records on both benches (17 to 51 ns at 0 and 64 bytes). `initial` now builds the state by indexing, and `features()` inlines its cached load, leaving only the first read out of line and cold. The shipped x86-64 code of AEAD and OpenSSH seal, open and packet_length then calls neither on any instruction path; the remaining precondition calls are in the portable AES and GCM fallbacks, which no AES-NI machine runs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PXEk8BBMDT8tXu4AnENB9Q --- lib/crypto/src/arch/x86_64.rs | 13 +++++++++-- lib/crypto/src/arch/x86_64/chacha.rs | 32 ++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/crypto/src/arch/x86_64.rs b/lib/crypto/src/arch/x86_64.rs index b1270fa473d..1778f7a60ca 100644 --- a/lib/crypto/src/arch/x86_64.rs +++ b/lib/crypto/src/arch/x86_64.rs @@ -127,16 +127,25 @@ impl Features { } } +/// The cache publishes only this public word, not any other memory. +static FEATURES: AtomicU32 = AtomicU32::new(0); + +/// Inline: every instruction path asks, several times per message. +#[inline(always)] pub(super) fn features() -> Features { if cfg!(feature = "portable") { return Features(Features::READ); } - // The cache publishes only this public word, not any other memory. - static FEATURES: AtomicU32 = AtomicU32::new(0); let cached = FEATURES.load(Ordering::Relaxed); if cached & Features::READ != 0 { return Features(cached); } + first_features() +} + +#[cold] +#[inline(never)] +fn first_features() -> Features { let read = Features::read(); FEATURES.store(read.0, Ordering::Relaxed); read diff --git a/lib/crypto/src/arch/x86_64/chacha.rs b/lib/crypto/src/arch/x86_64/chacha.rs index c38d54609ec..6aa4363090d 100644 --- a/lib/crypto/src/arch/x86_64/chacha.rs +++ b/lib/crypto/src/arch/x86_64/chacha.rs @@ -393,14 +393,28 @@ pub(super) unsafe fn seal( 384 * n } +/// The state with a zero counter. Indexing, not slice copies or iterator +/// adaptors, whose debug precondition checks the shipped profile calls out +/// of line on every message. fn initial(key: &[u8; 32], nonce: &[u8; 8]) -> crate::secret::Secret<16, u32> { - let mut initial = [0u32; 16]; - initial[..4].copy_from_slice(&SIGMA); - for (word, bytes) in initial[4..12].iter_mut().zip(key.as_chunks::<4>().0) { - *word = u32::from_le_bytes(*bytes); - } - for (word, bytes) in initial[14..].iter_mut().zip(nonce.as_chunks::<4>().0) { - *word = u32::from_le_bytes(*bytes); - } - crate::secret::Secret(initial) + let k = |i: usize| u32::from_le_bytes([key[4 * i], key[4 * i + 1], key[4 * i + 2], key[4 * i + 3]]); + let n = |i: usize| u32::from_le_bytes([nonce[4 * i], nonce[4 * i + 1], nonce[4 * i + 2], nonce[4 * i + 3]]); + crate::secret::Secret([ + SIGMA[0], + SIGMA[1], + SIGMA[2], + SIGMA[3], + k(0), + k(1), + k(2), + k(3), + k(4), + k(5), + k(6), + k(7), + 0, + 0, + n(0), + n(1), + ]) }