diff --git a/lib/crypto/src/arch/mod.rs b/lib/crypto/src/arch/mod.rs index 35ab95e0e2..c4952793c3 100644 --- a/lib/crypto/src/arch/mod.rs +++ b/lib/crypto/src/arch/mod.rs @@ -1,5 +1,5 @@ -//! Instruction dispatch is public. Each function here reports whether an -//! instruction path ran; when none did, the caller computes the same result +//! Instruction dispatch is public. Each dispatch function here reports whether +//! an instruction path ran; when none did, the caller computes the same result //! portably. UEFI assembly saves and restores every used XMM register; //! ordinary hosts use the target's ABI. Hosted AVX paths additionally require //! CPUID support and OS-enabled extended state in XCR0. @@ -34,6 +34,26 @@ macro_rules! hosted { }}; } +/// `value`, through an assembly block LLVM cannot see into. The block is a +/// comment: it emits no instruction and keeps the word in its register. +#[cfg(not(miri))] +#[inline(always)] +pub(crate) fn value_barrier(mut value: u64) -> u64 { + // The template reads and writes only the operand's register. + unsafe { + core::arch::asm!("/* {0} */", inout(reg) value, options(pure, nomem, nostack, preserves_flags)) + }; + value +} + +/// Miri runs no assembly; a volatile read of the local stands in. +#[cfg(miri)] +#[inline(always)] +pub(crate) fn value_barrier(value: u64) -> u64 { + // The local word remains live for this volatile read. + unsafe { core::ptr::read_volatile(&value) } +} + /// GHASH key powers, (H^(i+1))' at index i in the instruction paths' form. pub(crate) type Powers = [u128; 16]; diff --git a/lib/crypto/src/ct.rs b/lib/crypto/src/ct.rs index a3fb84c8e4..61bc7c9386 100644 --- a/lib/crypto/src/ct.rs +++ b/lib/crypto/src/ct.rs @@ -1,12 +1,12 @@ //! Content-independent comparisons and selection. Lengths are public. //! -//! The volatile barrier prevents LLVM from retaining a boolean's provenance; -//! deployed machine code must still be inspected and tested for timing leakage. +//! The register barrier prevents LLVM from retaining a boolean's provenance, +//! which would let it turn a masked select into a branch; deployed machine +//! code must still be inspected and tested for timing leakage. #[inline(always)] pub(crate) fn opaque(value: u64) -> u64 { - // The local word remains live for this volatile read. - unsafe { core::ptr::read_volatile(&value) } + crate::arch::value_barrier(value) } #[inline(always)] diff --git a/lib/crypto/src/ecc.rs b/lib/crypto/src/ecc.rs index 992d08461d..3c52ff878f 100644 --- a/lib/crypto/src/ecc.rs +++ b/lib/crypto/src/ecc.rs @@ -4,24 +4,19 @@ //! the only disclosed result is whether at least one candidate was valid. macro_rules! curve_api { - ($module:ident,$limbs:literal,$bytes:literal,$encoded:literal,$hash:ident) => { + ($module:ident, $curve:ident, $limbs:literal, $bytes:literal, $encoded:literal, $hash:ident) => { pub mod $module { use crate::{ Error, Random, hash::$hash, - montgomery::{Mod, V}, - nist::{Curve, Point}, + nist::{$curve, Point, Scalar}, secret::Secret, }; pub struct PublicKey { - curve: Curve<$limbs>, - point: Point<$limbs>, + point: Point<$curve, $limbs>, encoded: [u8; $encoded], } - pub struct SecretKey { - curve: Curve<$limbs>, - scalar: V<$limbs>, - } + pub struct SecretKey(Scalar<$curve, $limbs>); pub struct SharedSecret(Secret<$bytes>); impl SharedSecret { pub fn as_bytes(&self) -> &[u8; $bytes] { @@ -30,14 +25,9 @@ macro_rules! curve_api { } impl PublicKey { pub fn from_sec1(input: &[u8]) -> Result { - let curve = crate::nist::$module(); - let point = curve.decode(input)?; + let point = Point::decode(input)?; let encoded = input.try_into().map_err(|_| Error::Length)?; - Ok(Self { - curve, - point, - encoded, - }) + Ok(Self { point, encoded }) } pub fn as_sec1(&self) -> &[u8; $encoded] { &self.encoded @@ -50,49 +40,35 @@ macro_rules! curve_api { digest: &[u8; $bytes], der_signature: &[u8], ) -> Result<(), Error> { - self.curve.verify(&self.point, digest, der_signature) + self.point.verify(digest, der_signature) } } impl SecretKey { pub fn from_bytes(bytes: [u8; $bytes]) -> Result { let bytes = Secret(bytes); - let curve = crate::nist::$module(); - let scalar = curve.scalar(&bytes.0)?; - Ok(Self { curve, scalar }) + Ok(Self(Scalar::from_bytes(&bytes.0)?)) } pub fn generate(random: &mut impl Random) -> Result { - let curve = crate::nist::$module(); - let mut scalar = V::zero(); - let mut selected = 0; - for _ in 0..4 { - let mut candidate = Secret([0; $bytes]); - random.fill(&mut candidate.0)?; - let candidate = Mod::from_be(&candidate.0); - let valid = curve.n.valid(&candidate) & (1 ^ candidate.eq(&V::zero())); - scalar = V::select(&scalar, &candidate, valid & (1 ^ selected)); - selected |= valid; - } - if selected != 1 { - return Err(Error::Entropy); - } - Ok(Self { curve, scalar }) + let mut candidates = Secret([0; 4 * $bytes]); + random.fill(&mut candidates.0)?; + Ok(Self(Scalar::first_valid(&candidates.0)?)) } pub fn public_key(&self) -> PublicKey { - let point = self.curve.base().mul(&self.scalar, &self.curve); + // The base point's multiple by a nonzero scalar below n + // is finite and on the curve. + let point = Point::base().mul(&self.0); let mut encoded = [0; $encoded]; - point - .encode(&self.curve, &mut encoded) - .expect("nonzero prime-order scalar"); - PublicKey::from_sec1(&encoded).expect("curve arithmetic invariant") + point.encode(&mut encoded).expect("nonzero prime-order scalar"); + PublicKey { point, encoded } } pub fn agree(self, peer: &PublicKey) -> Result { let mut out = Secret([0; $bytes]); - self.curve.agree(&self.scalar, &peer.point, &mut out.0)?; + peer.point.agree(&self.0, &mut out.0); Ok(SharedSecret(out)) } } } }; } -curve_api!(p256, 4, 32, 65, Sha256); -curve_api!(p384, 6, 48, 97, Sha384); +curve_api!(p256, P256, 4, 32, 65, Sha256); +curve_api!(p384, P384, 6, 48, 97, Sha384); diff --git a/lib/crypto/src/ed25519.rs b/lib/crypto/src/ed25519.rs index 1b63047482..019709acaf 100644 --- a/lib/crypto/src/ed25519.rs +++ b/lib/crypto/src/ed25519.rs @@ -7,14 +7,34 @@ use crate::{Error, ct, field25519::F, hash::Sha512, scalar25519::S}; #[cfg(feature = "sign")] use crate::{Random, secret::Secret}; -const D: [u8; 32] = [ +/// 2d for the curve's d = -121665/121666 (RFC 8032 section 5.1). +const D2: F = F::from_bytes(&[ + 0x59, 0xf1, 0xb2, 0x26, 0x94, 0x9b, 0xd6, 0xeb, 0x56, 0xb1, 0x83, 0x82, 0x9a, 0x14, 0xe0, 0x00, + 0x30, 0xd1, 0xf3, 0xee, 0xf2, 0x80, 0x8e, 0x19, 0xe7, 0xfc, 0xdf, 0x56, 0xdc, 0xd9, 0x06, 0x24, +]); +const D: F = F::from_bytes(&[ 0xa3, 0x78, 0x59, 0x13, 0xca, 0x4d, 0xeb, 0x75, 0xab, 0xd8, 0x41, 0x41, 0x4d, 0x0a, 0x70, 0x00, 0x98, 0xe8, 0x79, 0x77, 0x79, 0x40, 0xc7, 0x8c, 0x73, 0xfe, 0x6f, 0x2b, 0xee, 0x6c, 0x03, 0x52, -]; -const SQRT_M1: [u8; 32] = [ +]); +const SQRT_M1: F = F::from_bytes(&[ 0xb0, 0xa0, 0x0e, 0x4a, 0x27, 0x1b, 0xee, 0xc4, 0x78, 0xe4, 0x2f, 0xad, 0x06, 0x18, 0x43, 0x2f, 0xa7, 0xd7, 0xfb, 0x3d, 0x99, 0x00, 0x4d, 0x2b, 0x0b, 0xdf, 0xc1, 0x4f, 0x80, 0x24, 0x83, 0x2b, +]); +/// The base point B's encoding (RFC 8032 section 5.1): y = 4/5, x positive. +const B_ENCODED: [u8; 32] = [ + 0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, ]; +const B_X: F = F::from_bytes(&[ + 0x1a, 0xd5, 0x25, 0x8f, 0x60, 0x2d, 0x56, 0xc9, 0xb2, 0xa7, 0x25, 0x95, 0x60, 0xc7, 0x2c, 0x69, + 0x5c, 0xdc, 0xd6, 0xfd, 0x31, 0xe2, 0xa4, 0xc0, 0xfe, 0x53, 0x6e, 0xcd, 0xd3, 0x36, 0x69, 0x21, +]); +const B_Y: F = F::from_bytes(&B_ENCODED); +/// x * y of B. +const B_T: F = F::from_bytes(&[ + 0xa3, 0xdd, 0xb7, 0xa5, 0xb3, 0x8a, 0xde, 0x6d, 0xf5, 0x52, 0x51, 0x77, 0x80, 0x9f, 0xf0, 0x20, + 0x7d, 0xe3, 0xab, 0x64, 0x8e, 0x4e, 0xea, 0x66, 0x65, 0x76, 0x8b, 0xd7, 0x0f, 0x5f, 0x87, 0x67, +]); struct Point { x: F, y: F, @@ -31,9 +51,12 @@ impl Point { } } fn base() -> Self { - let mut b = [0x66; 32]; - b[0] = 0x58; - Self::decode(&b).expect("Ed25519 base point") + Self { + x: B_X, + y: B_Y, + z: F::one(), + t: B_T, + } } fn decode(input: &[u8; 32]) -> Result { let mut b = *input; @@ -45,13 +68,13 @@ impl Point { } let y2 = y.square(); let u = y2.sub(&F::one()); - let v = F::from_bytes(&D).mul(&y2).add(&F::one()); + let v = D.mul(&y2).add(&F::one()); let q = u.mul(&v.invert()); let mut exponent = [0xff; 32]; exponent[0] = 0xfe; exponent[31] = 0x0f; let root = q.pow(&exponent, 252); - let alt = root.mul(&F::from_bytes(&SQRT_M1)); + let alt = root.mul(&SQRT_M1); let root = F::select(&alt, &root, root.square().eq(&q)); if root.square().eq(&q) == 0 || (root.eq(&F::zero()) == 1 && sign == 1) { return Err(Error::InvalidPoint); @@ -76,8 +99,9 @@ impl Point { fn add(&self, q: &Self) -> Self { let a = self.y.sub(&self.x).mul(&q.y.sub(&q.x)); let b = self.y.add(&self.x).mul(&q.y.add(&q.x)); - let c = self.t.mul(&q.t).mul(&F::from_bytes(&D)).mul(&F::small(2)); - let d = self.z.mul(&q.z).mul(&F::small(2)); + let c = self.t.mul(&q.t).mul(&D2); + let zz = self.z.mul(&q.z); + let d = zz.add(&zz); let e = b.sub(&a); let f = d.sub(&c); let g = d.add(&c); @@ -208,3 +232,22 @@ impl SigningKey { Ok(out) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The compile-time constants are what their definitions compute. + #[test] + fn constants_match_their_definitions() { + let decoded = Point::decode(&B_ENCODED).unwrap(); + let base = Point::base(); + for (a, b) in [(&decoded.x, &base.x), (&decoded.y, &base.y), (&decoded.t, &base.t)] { + assert_eq!(a.bytes(), b.bytes()); + } + assert_eq!(D.add(&D).bytes(), D2.bytes()); + let d = F::small(121665).neg().mul(&F::small(121666).invert()); + assert_eq!(d.bytes(), D.bytes()); + assert_eq!(SQRT_M1.square().bytes(), F::one().neg().bytes()); + } +} diff --git a/lib/crypto/src/field25519.rs b/lib/crypto/src/field25519.rs index 2357920c8e..0e25ee987b 100644 --- a/lib/crypto/src/field25519.rs +++ b/lib/crypto/src/field25519.rs @@ -20,10 +20,13 @@ impl F { pub(crate) fn small(x: u32) -> Self { Self([u64::from(x), 0, 0, 0, 0]) } - pub(crate) fn from_bytes(b: &[u8; 32]) -> Self { + /// The low 255 bits of a little-endian encoding. + pub(crate) const fn from_bytes(b: &[u8; 32]) -> Self { let mut v = [0u64; 5]; - for i in 0..255 { - v[i / 51] |= u64::from((b[i / 8] >> (i % 8)) & 1) << (i % 51) + let mut i = 0; + while i < 255 { + v[i / 51] |= (((b[i / 8] >> (i % 8)) & 1) as u64) << (i % 51); + i += 1; } Self(v) } diff --git a/lib/crypto/src/montgomery.rs b/lib/crypto/src/montgomery.rs index 99522d4ab2..c91dd67c76 100644 --- a/lib/crypto/src/montgomery.rs +++ b/lib/crypto/src/montgomery.rs @@ -1,191 +1,288 @@ -//! Bounded Montgomery arithmetic. N is four or six for the NIST curves. -//! Every secret operation uses fixed loop bounds, carry propagation and masked -//! reductions. Moduli and exponents for inversion are public. No general bignum -//! or allocator crosses the primitive boundary. +//! Montgomery arithmetic modulo the NIST curves' primes and orders, N = 4 or 6 +//! words. Each modulus is a type, so a residue modulo one cannot meet a residue +//! modulo another. Every operation on a residue has a fixed schedule and a +//! masked final subtraction; inversion's exponent is public. Residues are +//! erased on drop and workspaces after use. +#![allow( + clippy::needless_range_loop, + reason = "carry chains index parallel fixed-size arrays; iterator adaptors stay out of line in the shipped profile" +)] use crate::{ct, secret::wipe64}; -pub(crate) struct V(pub(crate) [u64; N]); -impl Drop for V { +use core::marker::PhantomData; + +/// An odd modulus p with 2^(64N-1) < p < 2^(64N), and its Montgomery +/// constants for R = 2^(64N), evaluated at compile time. +pub(crate) trait Modulus: 'static { + const P: [u64; N]; + /// -p^-1 mod 2^64. + const N0: u64 = n0(Self::P[0]); + /// R mod p, the Montgomery form of one. + const ONE: [u64; N] = shift_mod(&one(), &Self::P, 64 * N); + /// R^2 mod p. + const R2: [u64; N] = shift_mod(&one(), &Self::P, 128 * N); +} + +/// x * R mod p, the Montgomery form of an integer x < p, for constants. +pub(crate) const fn encode(x: &[u64; N], p: &[u64; N]) -> [u64; N] { + shift_mod(x, p, 64 * N) +} + +/// A residue modulo `M` in Montgomery form, below `M::P`. +pub(crate) struct Residue, const N: usize>([u64; N], PhantomData); + +impl, const N: usize> Drop for Residue { fn drop(&mut self) { wipe64(&mut self.0) } } -pub(crate) struct Mod { - pub(crate) p: [u64; N], - inverse: u64, - r2: V, - one: V, -} -impl V { - pub(crate) fn zero() -> Self { - Self([0; N]) + +impl, const N: usize> Residue { + /// Words already in Montgomery form and below p: a compile-time constant + /// from [`encode`]. + pub(crate) const fn from_montgomery(words: [u64; N]) -> Self { + Self(words, PhantomData) } - pub(crate) fn select(a: &Self, b: &Self, bit: u64) -> Self { - Self(core::array::from_fn(|i| ct::select(a.0[i], b.0[i], bit))) + pub(crate) const fn zero() -> Self { + Self([0; N], PhantomData) } - pub(crate) fn eq(&self, b: &Self) -> u64 { - let mut d = 0; - for i in 0..N { - d |= self.0[i] ^ b.0[i] - } - 1 ^ ct::nonzero(d) + pub(crate) const fn one() -> Self { + Self(M::ONE, PhantomData) } -} -impl Mod { - pub(crate) fn new(p: [u64; N]) -> Self { - // Raw N-limb digests reduce with one subtraction only when 2*p > 2^(64*N). - assert!(matches!(N, 4 | 6) && p[0] & 1 == 1 && p[N - 1] >> 63 == 1); - let mut inverse = 1u64; - for _ in 0..6 { - inverse = inverse.wrapping_mul(2u64.wrapping_sub(p[0].wrapping_mul(inverse))) - } - let mut m = Self { - p, - inverse: inverse.wrapping_neg(), - r2: V::zero(), - one: V::zero(), - }; - let mut x = V::zero(); - x.0[0] = 1; - for _ in 0..2 * N * 64 { - x = m.add(&x, &x) + /// The canonical integer `x`, or `None` when x >= p. Only the answer + /// depends on x. + pub(crate) fn from_integer(x: &[u64; N]) -> Option { + if below(x, &M::P) != 1 { + return None; } - m.r2 = x; - let mut raw = V::zero(); - raw.0[0] = 1; - m.one = m.mul(&raw, &m.r2); - m + Some(Self::from_reduced(x)) } - pub(crate) fn one(&self) -> V { - V(self.one.0) + /// An integer x < 2^(64N) < 2p, reduced once. + pub(crate) fn from_wide(x: &[u64; N]) -> Self { + let mut v = *x; + let reduced = Self(reduce(&v, 0, &M::P), PhantomData); + wipe64(&mut v); + reduced.mul(&Self(M::R2, PhantomData)) } - // The incoming value is below 2*p, including its carry limb. - fn reduce(&self, v: &[u64; N], high: u64) -> V { - let mut d = [0; N]; - let mut borrow = 0u128; - for i in 0..N { - let t = (1u128 << 64) + u128::from(v[i]) - u128::from(self.p[i]) - borrow; - d[i] = t as u64; - borrow = 1 - (t >> 64) - } - let bit = ct::nonzero(high) | (1 - borrow as u64); - let result = V(core::array::from_fn(|i| ct::select(v[i], d[i], bit))); - wipe64(&mut d); - result + fn from_reduced(x: &[u64; N]) -> Self { + Self(*x, PhantomData).mul(&Self(M::R2, PhantomData)) } - pub(crate) fn add(&self, a: &V, b: &V) -> V { + /// The canonical integer this residue represents. + pub(crate) fn to_integer(&self) -> [u64; N] { + let one = Self(one(), PhantomData); + self.mul(&one).0 + } + pub(crate) fn add(&self, b: &Self) -> Self { let mut v = [0; N]; - let mut c = 0u128; - for (i, word) in v.iter_mut().enumerate() { - let t = u128::from(a.0[i]) + u128::from(b.0[i]) + c; - *word = t as u64; - c = t >> 64 + let mut carry = false; + for i in 0..N { + (v[i], carry) = carrying_add(self.0[i], b.0[i], carry); } - let result = self.reduce(&v, c as u64); + let result = Self(reduce(&v, u64::from(carry), &M::P), PhantomData); wipe64(&mut v); result } - pub(crate) fn sub(&self, a: &V, b: &V) -> V { + pub(crate) fn double(&self) -> Self { + self.add(self) + } + pub(crate) fn sub(&self, b: &Self) -> Self { let mut v = [0; N]; - let mut borrow = 0u128; - for (i, word) in v.iter_mut().enumerate() { - let t = (1u128 << 64) + u128::from(a.0[i]) - u128::from(b.0[i]) - borrow; - *word = t as u64; - borrow = 1 - (t >> 64) - } - let mask = ct::mask(borrow as u64); - let mut c = 0u128; - for (i, word) in v.iter_mut().enumerate() { - let t = u128::from(*word) + u128::from(self.p[i] & mask) + c; - *word = t as u64; - c = t >> 64 + let mut borrow = false; + for i in 0..N { + (v[i], borrow) = borrowing_sub(self.0[i], b.0[i], borrow); } - V(v) - } - pub(crate) fn mul(&self, a: &V, b: &V) -> V { - let mut t = [0u64; 14]; + let mask = ct::mask(u64::from(borrow)); + let mut carry = false; for i in 0..N { - let mut c = 0u128; - for j in 0..N { - let value = u128::from(a.0[i]) * u128::from(b.0[j]) + u128::from(t[i + j]) + c; - t[i + j] = value as u64; - c = value >> 64 - } - t[i + N] = c as u64; + (v[i], carry) = carrying_add(v[i], M::P[i] & mask, carry); } + Self(v, PhantomData) + } + /// self * b / R mod p, word by word with each product row followed by its + /// reduction step (CIOS; Koc, Acar and Kaliski, "Analyzing and comparing + /// Montgomery multiplication algorithms", IEEE Micro 1996). + pub(crate) fn mul(&self, b: &Self) -> Self { + let mut t = [0u64; N]; + let mut high = 0u64; for i in 0..N { - let q = t[i].wrapping_mul(self.inverse); - let mut c = 0u128; + let mut c = 0; for j in 0..N { - let value = u128::from(q) * u128::from(self.p[j]) + u128::from(t[i + j]) + c; - t[i + j] = value as u64; - c = value >> 64 + (t[j], c) = mac(t[j], self.0[i], b.0[j], c); } - for word in t.iter_mut().take(2 * N + 1).skip(i + N) { - let value = u128::from(*word) + c; - *word = value as u64; - c = value >> 64 + let (top, overflow) = high.overflowing_add(c); + let q = t[0].wrapping_mul(M::N0); + let (_, mut c) = mac(t[0], q, M::P[0], 0); + for j in 1..N { + (t[j - 1], c) = mac(t[j], q, M::P[j], c); } - assert_eq!(c, 0); + let (top, carry) = top.overflowing_add(c); + t[N - 1] = top; + // t < 2p after each step, so at most one of the two is set. + high = u64::from(overflow) | u64::from(carry); } - let result = self.reduce(t[N..2 * N].try_into().unwrap(), t[2 * N]); + let result = Self(reduce(&t, high, &M::P), PhantomData); wipe64(&mut t); result } - pub(crate) fn square(&self, a: &V) -> V { - self.mul(a, a) - } - pub(crate) fn small(&self, n: u64) -> V { - let mut raw = V::zero(); - raw.0[0] = n; - self.mul(&raw, &self.r2) + pub(crate) fn square(&self) -> Self { + self.mul(self) } - pub(crate) fn encode_raw(&self, raw: &V) -> V { - self.mul(raw, &self.r2) + /// One when self is zero, otherwise zero. + pub(crate) fn is_zero(&self) -> u64 { + let mut d = 0; + for word in self.0 { + d |= word + } + 1 ^ ct::nonzero(d) } - pub(crate) fn raw(&self, v: &V) -> V { - let mut one = V::zero(); - one.0[0] = 1; - self.mul(v, &one) + pub(crate) fn eq(&self, b: &Self) -> u64 { + let mut d = 0; + for i in 0..N { + d |= self.0[i] ^ b.0[i] + } + 1 ^ ct::nonzero(d) } - pub(crate) fn valid(&self, v: &V) -> u64 { - let mut borrow = 0u128; + pub(crate) fn select(a: &Self, b: &Self, bit: u64) -> Self { + let m = ct::mask(bit); + let mut v = [0; N]; for i in 0..N { - let t = (1u128 << 64) + u128::from(v.0[i]) - u128::from(self.p[i]) - borrow; - borrow = 1 - (t >> 64) + v[i] = (a.0[i] & !m) | (b.0[i] & m); } - borrow as u64 - } - pub(crate) fn invert(&self, a: &V) -> V { - let mut e = self.p; - let mut borrow = 2u128; - for word in &mut e { - let t = (1u128 << 64) + u128::from(*word) - borrow; - *word = t as u64; - borrow = 1 - (t >> 64) + Self(v, PhantomData) + } + /// self^(p-2), the inverse of a nonzero residue, by square-and-multiply + /// over the public exponent. + pub(crate) fn invert(&self) -> Self { + let mut e = M::P; + let mut borrow = false; + (e[0], borrow) = borrowing_sub(e[0], 2, borrow); + for word in &mut e[1..] { + (*word, borrow) = borrowing_sub(*word, 0, borrow); } - let mut r = self.one(); + let mut r = Self::one(); for i in (0..N * 64).rev() { - let square = self.square(&r); - let product = self.mul(&square, a); - r = V::select(&square, &product, (e[i / 64] >> (i % 64)) & 1) + r = r.square(); + if (e[i / 64] >> (i % 64)) & 1 == 1 { + r = r.mul(self); + } } r } - pub(crate) fn from_be(bytes: &[u8]) -> V { - assert_eq!(bytes.len(), N * 8); - V(core::array::from_fn(|i| { - u64::from_be_bytes( - bytes[bytes.len() - 8 * (i + 1)..bytes.len() - 8 * i] - .try_into() - .unwrap(), - ) - })) - } - pub(crate) fn to_be(v: &V, bytes: &mut [u8]) { - assert_eq!(bytes.len(), N * 8); - for i in 0..N { - let end = bytes.len() - 8 * i; - bytes[end - 8..end].copy_from_slice(&v.0[i].to_be_bytes()) +} + +/// One when x < p, otherwise zero; the time does not depend on x. +pub(crate) fn below(x: &[u64; N], p: &[u64; N]) -> u64 { + let mut borrow = false; + for i in 0..N { + (_, borrow) = borrowing_sub(x[i], p[i], borrow); + } + u64::from(borrow) +} + +/// Big-endian bytes, exactly 8N of them, as little-endian words. +pub(crate) fn from_be(bytes: &[u8]) -> [u64; N] { + assert_eq!(bytes.len(), N * 8); + let mut words = [0; N]; + for (i, word) in words.iter_mut().enumerate() { + let end = bytes.len() - 8 * i; + *word = u64::from_be_bytes(bytes[end - 8..end].try_into().unwrap()); + } + words +} + +pub(crate) fn to_be(words: &[u64; N], bytes: &mut [u8]) { + assert_eq!(bytes.len(), N * 8); + for (i, word) in words.iter().enumerate() { + let end = bytes.len() - 8 * i; + bytes[end - 8..end].copy_from_slice(&word.to_be_bytes()) + } +} + +/// v mod p for v + high * 2^(64N) < 2p: p subtracted under a mask. +fn reduce(v: &[u64; N], high: u64, p: &[u64; N]) -> [u64; N] { + let mut d = [0; N]; + let mut borrow = false; + for i in 0..N { + (d[i], borrow) = borrowing_sub(v[i], p[i], borrow); + } + let m = ct::mask(ct::nonzero(high) | (1 ^ u64::from(borrow))); + let mut out = [0; N]; + for i in 0..N { + out[i] = (v[i] & !m) | (d[i] & m); + } + wipe64(&mut d); + out +} + +/// (lo, hi) of t + a * b + c, which cannot exceed 2^128 - 1. +#[inline(always)] +pub(crate) fn mac(t: u64, a: u64, b: u64, c: u64) -> (u64, u64) { + let v = u128::from(a) + .wrapping_mul(u128::from(b)) + .wrapping_add(u128::from(t)) + .wrapping_add(u128::from(c)); + (v as u64, (v >> 64) as u64) +} + +#[inline(always)] +fn carrying_add(a: u64, b: u64, carry: bool) -> (u64, bool) { + let (s, c1) = a.overflowing_add(b); + let (s, c2) = s.overflowing_add(u64::from(carry)); + (s, c1 | c2) +} + +#[inline(always)] +fn borrowing_sub(a: u64, b: u64, borrow: bool) -> (u64, bool) { + let (d, b1) = a.overflowing_sub(b); + let (d, b2) = d.overflowing_sub(u64::from(borrow)); + (d, b1 | b2) +} + +const fn one() -> [u64; N] { + let mut x = [0; N]; + x[0] = 1; + x +} + +/// -p0^-1 mod 2^64 for an odd p0. +pub(crate) const fn n0(p0: u64) -> u64 { + // Newton's iteration doubles the correct low bits of p0^-1 from 1 to 64. + let mut inverse = 1u64; + let mut i = 0; + while i < 6 { + inverse = inverse.wrapping_mul(2u64.wrapping_sub(p0.wrapping_mul(inverse))); + i += 1; + } + inverse.wrapping_neg() +} + +/// x * 2^k mod p for x < p, by k modular doublings, at compile time. +const fn shift_mod(x: &[u64; N], p: &[u64; N], k: usize) -> [u64; N] { + assert!(p[0] & 1 == 1 && p[N - 1] >> 63 == 1); + let mut x = *x; + let mut step = 0; + while step < k { + let mut carry = 0; + let mut i = 0; + while i < N { + let top = x[i] >> 63; + x[i] = (x[i] << 1) | carry; + carry = top; + i += 1; + } + let mut d = [0; N]; + let mut borrow = 0u64; + let mut i = 0; + while i < N { + let (t, b1) = x[i].overflowing_sub(p[i]); + let (t, b2) = t.overflowing_sub(borrow); + d[i] = t; + borrow = (b1 | b2) as u64; + i += 1; + } + if carry == 1 || borrow == 0 { + x = d; } + step += 1; } + x } diff --git a/lib/crypto/src/nist.rs b/lib/crypto/src/nist.rs index 6e44d94b06..008b10abcc 100644 --- a/lib/crypto/src/nist.rs +++ b/lib/crypto/src/nist.rs @@ -2,204 +2,236 @@ //! double-and-add schedule with complete masked handling of exceptional points. //! SEC 1 decoding, DER decoding, ECDSA verification and key-validity refusals //! depend on public data. Only uncompressed SEC 1 points are accepted. +#![allow( + clippy::needless_range_loop, + reason = "masked selects index parallel fixed-size arrays, as `montgomery` does" +)] use crate::{ Error, - montgomery::{Mod, V}, + montgomery::{self, Modulus, Residue, below, from_be, to_be}, + secret::wipe64, }; -pub(crate) struct Curve { - pub(crate) p: Mod, - pub(crate) n: Mod, - b: V, - gx: V, - gy: V, + +/// A short Weierstrass curve y^2 = x^3 - 3x + b of prime order and cofactor +/// one, with its constants in Montgomery form at compile time. +pub(crate) trait Curve: 'static { + type Field: Modulus; + type Order: Modulus; + const B: [u64; N]; + const GX: [u64; N]; + const GY: [u64; N]; } -pub(crate) struct Point { - x: V, - y: V, - z: V, +type Coord = Residue<>::Field, N>; +type ModN = Residue<>::Order, N>; + +/// A scalar of curve `C` below n, erased on drop. A secret one is never zero. +pub(crate) struct Scalar, const N: usize>([u64; N], core::marker::PhantomData); +impl, const N: usize> Drop for Scalar { + fn drop(&mut self) { + wipe64(&mut self.0) + } } -impl Point { - fn identity(c: &Curve) -> Self { +impl, const N: usize> Scalar { + /// One when 0 < x < n, otherwise zero; the time does not depend on x. + fn valid(x: &[u64; N]) -> u64 { + let mut d = 0; + for word in x { + d |= word + } + below(x, &C::Order::P) & crate::ct::nonzero(d) + } + /// Big-endian bytes, 8N of them. Only validity depends on their value. + pub(crate) fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != N * 8 { + return Err(Error::Length); + } + let s = Self(from_be(bytes), core::marker::PhantomData); + if Self::valid(&s.0) != 1 { + return Err(Error::InvalidKey); + } + Ok(s) + } + /// The first valid candidate of `candidates`, 8N big-endian bytes each, on + /// a schedule that discloses only whether any is valid. + pub(crate) fn first_valid(candidates: &[u8]) -> Result { + assert_eq!(candidates.len() % (N * 8), 0); + let mut scalar = Self([0; N], core::marker::PhantomData); + let mut selected = 0; + for i in 0..candidates.len() / (N * 8) { + let bytes = &candidates[i * N * 8..(i + 1) * N * 8]; + let mut candidate = from_be::(bytes); + let take = crate::ct::mask(Self::valid(&candidate) & (1 ^ selected)); + for j in 0..N { + scalar.0[j] = (scalar.0[j] & !take) | (candidate[j] & take); + } + selected |= take & 1; + wipe64(&mut candidate); + } + if selected != 1 { + return Err(Error::Entropy); + } + Ok(scalar) + } + fn bit(&self, i: usize) -> u64 { + (self.0[i / 64] >> (i % 64)) & 1 + } +} + +/// Jacobian coordinates (X : Y : Z) for x = X/Z^2, y = Y/Z^3; Z = 0 is the +/// point at infinity. +pub(crate) struct Point, const N: usize> { + x: Coord, + y: Coord, + z: Coord, +} +impl, const N: usize> Point { + fn identity() -> Self { Self { - x: V::zero(), - y: c.p.one(), - z: V::zero(), + x: Residue::zero(), + y: Residue::one(), + z: Residue::zero(), + } + } + pub(crate) fn base() -> Self { + Self { + x: Residue::from_montgomery(C::GX), + y: Residue::from_montgomery(C::GY), + z: Residue::one(), } } fn select(a: &Self, b: &Self, bit: u64) -> Self { Self { - x: V::select(&a.x, &b.x, bit), - y: V::select(&a.y, &b.y, bit), - z: V::select(&a.z, &b.z, bit), + x: Residue::select(&a.x, &b.x, bit), + y: Residue::select(&a.y, &b.y, bit), + z: Residue::select(&a.z, &b.z, bit), } } - fn double(&self, c: &Curve) -> Self { - let p = &c.p; - let delta = p.square(&self.z); - let gamma = p.square(&self.y); - let beta = p.mul(&self.x, &gamma); - let alpha = p.mul( - &p.small(3), - &p.mul(&p.sub(&self.x, &delta), &p.add(&self.x, &delta)), - ); - let x = p.sub(&p.square(&alpha), &p.mul(&p.small(8), &beta)); - let z = p.sub(&p.sub(&p.square(&p.add(&self.y, &self.z)), &gamma), &delta); - let y = p.sub( - &p.mul(&alpha, &p.sub(&p.mul(&p.small(4), &beta), &x)), - &p.mul(&p.small(8), &p.square(&gamma)), - ); - let out = Self { x, y, z }; - Self::select(&out, &Self::identity(c), self.z.eq(&V::zero())) + /// "dbl-2001-b" for a = -3 (Bernstein and Lange, Explicit-Formulas + /// Database), 3M + 5S. Infinity doubles to Z = 0, and no point of a + /// prime-order curve has y = 0. + fn double(&self) -> Self { + let delta = self.z.square(); + let gamma = self.y.square(); + let beta = self.x.mul(&gamma); + let t = self.x.sub(&delta).mul(&self.x.add(&delta)); + let alpha = t.double().add(&t); + let beta4 = beta.double().double(); + let x = alpha.square().sub(&beta4.double()); + let z = self.y.add(&self.z).square().sub(&gamma).sub(&delta); + let gamma2 = gamma.square().double(); + let y = alpha.mul(&beta4.sub(&x)).sub(&gamma2.double().double()); + Self { x, y, z } } - fn add(&self, q: &Self, c: &Curve) -> Self { - let p = &c.p; - let z1 = p.square(&self.z); - let z2 = p.square(&q.z); - let u1 = p.mul(&self.x, &z2); - let u2 = p.mul(&q.x, &z1); - let s1 = p.mul(&self.y, &p.mul(&q.z, &z2)); - let s2 = p.mul(&q.y, &p.mul(&self.z, &z1)); - let h = p.sub(&u2, &u1); - let r = p.sub(&s2, &s1); - let h2 = p.square(&h); - let h3 = p.mul(&h, &h2); - let u = p.mul(&u1, &h2); - let x = p.sub(&p.sub(&p.square(&r), &h3), &p.add(&u, &u)); - let y = p.sub(&p.mul(&r, &p.sub(&u, &x)), &p.mul(&s1, &h3)); - let z = p.mul(&h, &p.mul(&self.z, &q.z)); + fn add(&self, q: &Self) -> Self { + let z1 = self.z.square(); + let z2 = q.z.square(); + let u1 = self.x.mul(&z2); + let u2 = q.x.mul(&z1); + let s1 = self.y.mul(&q.z.mul(&z2)); + let s2 = q.y.mul(&self.z.mul(&z1)); + let h = u2.sub(&u1); + let r = s2.sub(&s1); + let h2 = h.square(); + let h3 = h.mul(&h2); + let u = u1.mul(&h2); + let x = r.square().sub(&h3).sub(&u.double()); + let y = r.mul(&u.sub(&x)).sub(&s1.mul(&h3)); + let z = h.mul(&self.z.mul(&q.z)); let out = Self { x, y, z }; - let out = Self::select(&out, &self.double(c), h.eq(&V::zero()) & r.eq(&V::zero())); - let out = Self::select(&out, self, q.z.eq(&V::zero())); - Self::select(&out, q, self.z.eq(&V::zero())) + let out = Self::select(&out, &self.double(), h.is_zero() & r.is_zero()); + let out = Self::select(&out, self, q.z.is_zero()); + Self::select(&out, q, self.z.is_zero()) } - pub(crate) fn mul(&self, scalar: &V, c: &Curve) -> Self { - let mut r = Self::identity(c); + pub(crate) fn mul(&self, scalar: &Scalar) -> Self { + let mut r = Self::identity(); for i in (0..N * 64).rev() { - let doubled = r.double(c); - let added = doubled.add(self, c); - r = Self::select(&doubled, &added, (scalar.0[i / 64] >> (i % 64)) & 1) + let doubled = r.double(); + let added = doubled.add(self); + r = Self::select(&doubled, &added, scalar.bit(i)) } r } - fn affine(&self, c: &Curve) -> Result<(V, V), Error> { - if self.z.eq(&V::zero()) == 1 { + fn affine(&self) -> Result<([u64; N], [u64; N]), Error> { + if self.z.is_zero() == 1 { return Err(Error::InvalidPoint); } - let inverse = c.p.invert(&self.z); - let inv2 = c.p.square(&inverse); + let inverse = self.z.invert(); + let inv2 = inverse.square(); Ok(( - c.p.raw(&c.p.mul(&self.x, &inv2)), - c.p.raw(&c.p.mul(&self.y, &c.p.mul(&inv2, &inverse))), + self.x.mul(&inv2).to_integer(), + self.y.mul(&inv2.mul(&inverse)).to_integer(), )) } - pub(crate) fn encode(&self, c: &Curve, out: &mut [u8]) -> Result<(), Error> { + /// The uncompressed SEC 1 encoding, 1 + 16N bytes. + pub(crate) fn encode(&self, out: &mut [u8]) -> Result<(), Error> { if out.len() != 1 + N * 16 { return Err(Error::Length); } - let (x, y) = self.affine(c)?; + let (mut x, mut y) = self.affine()?; out[0] = 4; - Mod::to_be(&x, &mut out[1..1 + N * 8]); - Mod::to_be(&y, &mut out[1 + N * 8..]); + to_be(&x, &mut out[1..1 + N * 8]); + to_be(&y, &mut out[1 + N * 8..]); + wipe64(&mut x); + wipe64(&mut y); Ok(()) } -} -impl Curve { - pub(crate) fn new(p: [u64; N], n: [u64; N], b: [u64; N], gx: [u64; N], gy: [u64; N]) -> Self { - let p = Mod::new(p); - let n = Mod::new(n); - let b = p.encode_raw(&V(b)); - let gx = p.encode_raw(&V(gx)); - let gy = p.encode_raw(&V(gy)); - Self { p, n, b, gx, gy } - } - pub(crate) fn base(&self) -> Point { - Point { - x: V(self.gx.0), - y: V(self.gy.0), - z: self.p.one(), - } - } - pub(crate) fn decode(&self, encoded: &[u8]) -> Result, Error> { + /// An uncompressed SEC 1 point on the curve. + pub(crate) fn decode(encoded: &[u8]) -> Result { if encoded.len() != 1 + N * 16 { return Err(Error::Length); } if encoded[0] != 4 { return Err(Error::InvalidEncoding); } - let x = Mod::from_be(&encoded[1..1 + N * 8]); - let y = Mod::from_be(&encoded[1 + N * 8..]); - if self.p.valid(&x) & self.p.valid(&y) != 1 { + let x = Coord::::from_integer(&from_be(&encoded[1..1 + N * 8])); + let y = Coord::::from_integer(&from_be(&encoded[1 + N * 8..])); + let (Some(x), Some(y)) = (x, y) else { return Err(Error::InvalidPoint); - } - let x = self.p.encode_raw(&x); - let y = self.p.encode_raw(&y); - let rhs = self.p.add( - &self.p.sub( - &self.p.mul(&self.p.square(&x), &x), - &self.p.mul(&self.p.small(3), &x), - ), - &self.b, - ); - if self.p.square(&y).eq(&rhs) != 1 { + }; + let rhs = x + .square() + .mul(&x) + .sub(&x.double().add(&x)) + .add(&Residue::from_montgomery(C::B)); + if y.square().eq(&rhs) != 1 { return Err(Error::InvalidPoint); } Ok(Point { x, y, - z: self.p.one(), + z: Residue::one(), }) } - pub(crate) fn scalar(&self, bytes: &[u8]) -> Result, Error> { - if bytes.len() != N * 8 { - return Err(Error::Length); - } - let s = Mod::from_be(bytes); - if self.n.valid(&s) & (1 ^ s.eq(&V::zero())) != 1 { - return Err(Error::InvalidKey); - } - Ok(s) - } - pub(crate) fn agree( - &self, - scalar: &V, - point: &Point, - output: &mut [u8], - ) -> Result<(), Error> { + /// The x-coordinate of scalar * self into `output`, 8N bytes. + pub(crate) fn agree(&self, scalar: &Scalar, output: &mut [u8]) { // Both curves have prime order and cofactor one; validated nonzero // scalars and finite peers cannot multiply to infinity. - let (x, _) = point - .mul(scalar, self) - .affine(self) + let (mut x, mut y) = self + .mul(scalar) + .affine() .expect("validated prime-order point and nonzero scalar"); - Mod::to_be(&x, output); - Ok(()) + to_be(&x, output); + wipe64(&mut x); + wipe64(&mut y); } - pub(crate) fn verify( - &self, - point: &Point, - digest: &[u8], - signature: &[u8], - ) -> Result<(), Error> { + /// ECDSA verification of a DER signature over an 8N-byte digest. All + /// inputs are public. + pub(crate) fn verify(&self, digest: &[u8], signature: &[u8]) -> Result<(), Error> { if digest.len() != N * 8 { return Err(Error::Length); } let (r, s) = der::(signature)?; - let r = self - .scalar(&r[..N * 8]) - .map_err(|_| Error::InvalidSignature)?; - let s = self - .scalar(&s[..N * 8]) - .map_err(|_| Error::InvalidSignature)?; - let z = Mod::from_be(digest); - let z = self.n.add(&V::zero(), &z); - let inverse = self.n.invert(&self.n.encode_raw(&s)); - let u = self.n.raw(&self.n.mul(&self.n.encode_raw(&z), &inverse)); - let v = self.n.raw(&self.n.mul(&self.n.encode_raw(&r), &inverse)); - let result = self.base().mul(&u, self).add(&point.mul(&v, self), self); - let (x, _) = result.affine(self).map_err(|_| Error::InvalidSignature)?; - let x = self.n.add(&V::zero(), &x); - if x.eq(&r) == 1 { + let r = Scalar::::from_bytes(&r[..N * 8]).map_err(|_| Error::InvalidSignature)?; + let s = Scalar::::from_bytes(&s[..N * 8]).map_err(|_| Error::InvalidSignature)?; + let z = ModN::::from_wide(&from_be(digest)); + let inverse = ModN::::from_wide(&s.0).invert(); + let u = Scalar(z.mul(&inverse).to_integer(), core::marker::PhantomData); + let v = Scalar(ModN::::from_wide(&r.0).mul(&inverse).to_integer(), core::marker::PhantomData); + let result = Self::base().mul(&u).add(&self.mul(&v)); + let (x, _) = result.affine().map_err(|_| Error::InvalidSignature)?; + let x = ModN::::from_wide(&x).to_integer(); + if x == r.0 { Ok(()) } else { Err(Error::InvalidSignature) @@ -244,59 +276,67 @@ fn der(input: &[u8]) -> Result<([u8; 48], [u8; 48]), Error> { } Ok((r, s)) } -pub(crate) fn p256() -> Curve<4> { - Curve::new( - [ - 0xffffffffffffffff, - 0x00000000ffffffff, - 0, - 0xffffffff00000001, - ], - [ - 0xf3b9cac2fc632551, - 0xbce6faada7179e84, - 0xffffffffffffffff, - 0xffffffff00000000, - ], - [ - 0x3bce3c3e27d2604b, - 0x651d06b0cc53b0f6, - 0xb3ebbd55769886bc, - 0x5ac635d8aa3a93e7, - ], - [ - 0xf4a13945d898c296, - 0x77037d812deb33a0, - 0xf8bce6e563a440f2, - 0x6b17d1f2e12c4247, - ], - [ - 0xcbb6406837bf51f5, - 0x2bce33576b315ece, - 0x8ee7eb4a7c0f9e16, - 0x4fe342e2fe1a7f9b, - ], - ) +macro_rules! modulus { + ($name:ident, $n:literal, $p:expr) => { + pub(crate) struct $name; + impl Modulus<$n> for $name { + const P: [u64; $n] = $p; + } + }; } -pub(crate) fn p384() -> Curve<6> { - Curve::new( - [ - 0x00000000ffffffff, - 0xffffffff00000000, - 0xfffffffffffffffe, - 0xffffffffffffffff, - 0xffffffffffffffff, - 0xffffffffffffffff, - ], - [ - 0xecec196accc52973, - 0x581a0db248b0a77a, - 0xc7634d81f4372ddf, - 0xffffffffffffffff, - 0xffffffffffffffff, - 0xffffffffffffffff, - ], - [ +modulus!(P256Field, 4, [0xffffffffffffffff, 0x00000000ffffffff, 0, 0xffffffff00000001]); +modulus!(P256Order, 4, [0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000]); +modulus!( + P384Field, + 6, + [ + 0x00000000ffffffff, + 0xffffffff00000000, + 0xfffffffffffffffe, + 0xffffffffffffffff, + 0xffffffffffffffff, + 0xffffffffffffffff, + ] +); +modulus!( + P384Order, + 6, + [ + 0xecec196accc52973, + 0x581a0db248b0a77a, + 0xc7634d81f4372ddf, + 0xffffffffffffffff, + 0xffffffffffffffff, + 0xffffffffffffffff, + ] +); + +/// P-256 (FIPS 186-5, SP 800-186 section 3.2.1.3). +pub(crate) struct P256; +impl Curve<4> for P256 { + type Field = P256Field; + type Order = P256Order; + const B: [u64; 4] = montgomery::encode( + &[0x3bce3c3e27d2604b, 0x651d06b0cc53b0f6, 0xb3ebbd55769886bc, 0x5ac635d8aa3a93e7], + &P256Field::P, + ); + const GX: [u64; 4] = montgomery::encode( + &[0xf4a13945d898c296, 0x77037d812deb33a0, 0xf8bce6e563a440f2, 0x6b17d1f2e12c4247], + &P256Field::P, + ); + const GY: [u64; 4] = montgomery::encode( + &[0xcbb6406837bf51f5, 0x2bce33576b315ece, 0x8ee7eb4a7c0f9e16, 0x4fe342e2fe1a7f9b], + &P256Field::P, + ); +} + +/// P-384 (FIPS 186-5, SP 800-186 section 3.2.1.4). +pub(crate) struct P384; +impl Curve<6> for P384 { + type Field = P384Field; + type Order = P384Order; + const B: [u64; 6] = montgomery::encode( + &[ 0x2a85c8edd3ec2aef, 0xc656398d8a2ed19d, 0x0314088f5013875a, @@ -304,7 +344,10 @@ pub(crate) fn p384() -> Curve<6> { 0x988e056be3f82d19, 0xb3312fa7e23ee7e4, ], - [ + &P384Field::P, + ); + const GX: [u64; 6] = montgomery::encode( + &[ 0x3a545e3872760ab7, 0x5502f25dbf55296c, 0x59f741e082542a38, @@ -312,7 +355,10 @@ pub(crate) fn p384() -> Curve<6> { 0x8eb1c71ef320ad74, 0xaa87ca22be8b0537, ], - [ + &P384Field::P, + ); + const GY: [u64; 6] = montgomery::encode( + &[ 0x7a431d7c90ea0e5f, 0x0a60b1ce1d7e819d, 0xe9da3113b5f0b8c0, @@ -320,5 +366,24 @@ pub(crate) fn p384() -> Curve<6> { 0x5d9e98bf9292dc29, 0x3617de4a96262c6f, ], - ) + &P384Field::P, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The first valid of four candidates is taken, and four invalid ones are + /// refused. + #[test] + fn generation_takes_the_first_valid_candidate() { + let mut n = [0u8; 32]; + to_be(&P256Order::P, &mut n); + let (k1, k2) = ([0x11; 32], [0x22; 32]); + let candidates = [n, [0; 32], k1, k2].concat(); + assert_eq!(Scalar::::first_valid(&candidates).unwrap().0, from_be::<4>(&k1)); + let invalid = [n, [0; 32], [0xff; 32], n].concat(); + assert!(matches!(Scalar::::first_valid(&invalid), Err(Error::Entropy))); + } } diff --git a/lib/crypto/src/rsa.rs b/lib/crypto/src/rsa.rs index 8fe8f5c667..0f9fd7538e 100644 --- a/lib/crypto/src/rsa.rs +++ b/lib/crypto/src/rsa.rs @@ -6,6 +6,7 @@ use crate::{ Error, ct, hash::{Sha256, Sha384, Sha512}, + montgomery::{mac, n0}, }; #[derive(Clone, Copy, Debug)] pub enum Hash { @@ -50,14 +51,17 @@ impl Hash { } } pub struct PublicKey { - modulus: [u64; 64], + modulus: [u64; WORDS], words: usize, bits: usize, exponent: u32, - inverse: u64, - r2: [u64; 64], - one: [u64; 64], + /// -modulus^-1 mod 2^64. + n0: u64, + /// R^2 mod modulus, R = 2^(64 * words). + r2: [u64; WORDS], } +/// The limbs of the largest modulus, 4096 bits. +const WORDS: usize = 64; impl PublicKey { /// Minimal unsigned big-endian integers. ASN.1 belongs to the consumer. pub fn from_components(modulus: &[u8], exponent: &[u8]) -> Result { @@ -76,104 +80,114 @@ impl PublicKey { } let words = modulus.len().div_ceil(8); let m = read(modulus); - let mut inverse = 1u64; - for _ in 0..6 { - inverse = inverse.wrapping_mul(2u64.wrapping_sub(m[0].wrapping_mul(inverse))) - } let mut key = Self { modulus: m, words, bits, exponent: e, - inverse: inverse.wrapping_neg(), - r2: [0; 64], - one: [0; 64], + n0: n0(m[0]), + r2: [0; WORDS], }; - let mut value = [0; 64]; - value[0] = 1; - for _ in 0..2 * words * 64 { - let mut d = [0; 64]; - let mut c = 0; - for i in 0..words { - let t = u128::from(value[i]) * 2 + c; - d[i] = t as u64; - c = t >> 64 - } - value = key.reduce(d, c as u64) - } - key.r2 = value; - let mut raw = [0; 64]; - raw[0] = 1; - key.one = key.mul(&raw, &key.r2); + key.r2 = key.montgomery_r2(); Ok(key) } - fn reduce(&self, value: [u64; 64], high: u64) -> [u64; 64] { - let mut d = [0; 64]; - let mut borrow = 0u128; - for i in 0..self.words { - let t = (1u128 << 64) + u128::from(value[i]) - u128::from(self.modulus[i]) - borrow; - d[i] = t as u64; - borrow = 1 - (t >> 64) + /// R^2 mod n from 2^(bits-1) < n: doublings to 2^words * R, then six + /// Montgomery squarings, each taking 2^t * R to 2^(2t) * R, up to + /// 2^(64 * words) * R. + fn montgomery_r2(&self) -> [u64; WORDS] { + let mut x = [0; WORDS]; + x[(self.bits - 1) / 64] = 1 << ((self.bits - 1) % 64); + for _ in 0..64 * self.words - (self.bits - 1) + self.words { + self.double(&mut x); } - if high != 0 || borrow == 0 { d } else { value } + let mut y = [0; WORDS]; + for _ in 0..3 { + self.mul(&x, &x, &mut y); + self.mul(&y, &y, &mut x); + } + x } - fn mul(&self, a: &[u64; 64], b: &[u64; 64]) -> [u64; 64] { - let n = self.words; - let mut t = [0u64; 130]; - for i in 0..n { - let mut c = 0u128; - for j in 0..n { - let v = u128::from(a[i]) * u128::from(b[j]) + u128::from(t[i + j]) + c; - t[i + j] = v as u64; - c = v >> 64 - } - t[i + n] = c as u64 + /// x = 2x mod n, for x < n. + fn double(&self, x: &mut [u64; WORDS]) { + let mut carry = 0; + for word in &mut x[..self.words] { + let top = *word >> 63; + *word = (*word << 1) | carry; + carry = top; } - for i in 0..n { - let q = t[i].wrapping_mul(self.inverse); - let mut c = 0u128; + if carry != 0 || !less(x, &self.modulus, self.words) { + subtract(x, &self.modulus, self.words); + } + } + /// out = a * b / R mod n, for a, b < n: word-serial Montgomery + /// multiplication with the reduction step interleaved (CIOS; Koc, Acar and + /// Kaliski, "Analyzing and comparing Montgomery multiplication + /// algorithms", IEEE Micro 1996). + fn mul(&self, a: &[u64; WORDS], b: &[u64; WORDS], out: &mut [u64; WORDS]) { + let n = self.words; + let (a, b, m) = (&a[..n], &b[..n], &self.modulus[..n]); + let mut t = [0u64; WORDS + 2]; + let t = &mut t[..n + 2]; + for &bi in b { + let mut c = 0; for j in 0..n { - let v = u128::from(q) * u128::from(self.modulus[j]) + u128::from(t[i + j]) + c; - t[i + j] = v as u64; - c = v >> 64 + (t[j], c) = mac(t[j], a[j], bi, c); } - for word in t.iter_mut().take(2 * n + 1).skip(i + n) { - let v = u128::from(*word) + c; - *word = v as u64; - c = v >> 64 + let (s, overflow) = t[n].overflowing_add(c); + t[n] = s; + t[n + 1] = u64::from(overflow); + let q = t[0].wrapping_mul(self.n0); + let (_, mut c) = mac(t[0], q, m[0], 0); + for j in 1..n { + (t[j - 1], c) = mac(t[j], q, m[j], c); } - assert_eq!(c, 0) + let (s, overflow) = t[n].overflowing_add(c); + t[n - 1] = s; + t[n] = t[n + 1].wrapping_add(u64::from(overflow)); + } + out[..n].copy_from_slice(&t[..n]); + if t[n] != 0 || !less(out, &self.modulus, n) { + subtract(out, &self.modulus, n); } - let mut result = [0; 64]; - result[..n].copy_from_slice(&t[n..2 * n]); - self.reduce(result, t[2 * n]) } + /// The signature raised to e, big-endian in the low bytes of 512. fn representative(&self, signature: &[u8]) -> Result<[u8; 512], Error> { if signature.len() != self.bits.div_ceil(8) { return Err(Error::Length); } - let raw = read(signature); - let reduced = self.reduce(raw, 0); - if raw != reduced { + let s = read(signature); + if !less(&s, &self.modulus, self.words) { return Err(Error::InvalidSignature); } - let base = self.mul(&raw, &self.r2); - let mut r = self.one; - for i in (0..32).rev() { - r = self.mul(&r, &r); - if (self.exponent >> i) & 1 == 1 { - r = self.mul(&r, &base) - } - } - let mut one = [0; 64]; - one[0] = 1; - let value = self.mul(&r, &one); + let value = self.power(&s); let mut bytes = [0; 512]; - for i in 0..64 { + for i in 0..WORDS { bytes[512 - 8 * (i + 1)..512 - 8 * i].copy_from_slice(&value[i].to_be_bytes()) } Ok(bytes) } + /// s^e mod n for s < n: s^(e-1) in Montgomery form by left-to-right + /// square-and-multiply, then one Montgomery multiplication by the + /// unencoded s, which also leaves Montgomery form. 65537 is 18 + /// multiplications. + fn power(&self, s: &[u64; WORDS]) -> [u64; WORDS] { + let mut base = [0; WORDS]; + self.mul(s, &self.r2, &mut base); + // e - 1 is even and at least 2, so its top bit is the starting `base`. + let e = self.exponent - 1; + let mut x = base; + let mut y = [0; WORDS]; + for i in (0..31 - e.leading_zeros()).rev() { + self.mul(&x, &x, &mut y); + if (e >> i) & 1 == 1 { + self.mul(&y, &base, &mut x); + } else { + core::mem::swap(&mut x, &mut y); + } + } + self.mul(&x, s, &mut y); + y + } pub fn verify_pkcs1_v1_5( &self, hash: Hash, @@ -248,10 +262,76 @@ impl PublicKey { Ok(()) } } -fn read(bytes: &[u8]) -> [u64; 64] { - let mut out = [0; 64]; +/// a < b over the low `n` words. +fn less(a: &[u64; WORDS], b: &[u64; WORDS], n: usize) -> bool { + for i in (0..n).rev() { + if a[i] != b[i] { + return a[i] < b[i]; + } + } + false +} +/// a -= b over the low `n` words, modulo 2^(64n). +fn subtract(a: &mut [u64; WORDS], b: &[u64; WORDS], n: usize) { + let mut borrow = false; + for i in 0..n { + let (d, b1) = a[i].overflowing_sub(b[i]); + let (d, b2) = d.overflowing_sub(u64::from(borrow)); + a[i] = d; + borrow = b1 | b2; + } +} +fn read(bytes: &[u8]) -> [u64; WORDS] { + let mut out = [0; WORDS]; for (i, &byte) in bytes.iter().rev().enumerate() { out[i / 8] |= u64::from(byte) << (8 * (i % 8)) } out } +#[cfg(test)] +mod tests { + use super::*; + use num_bigint::BigUint; + + /// Odd moduli of every limb shape, including partial top limbs, and + /// exponents up to four bytes against the integer oracle. + #[test] + fn power_and_r2_match_integer_oracle() { + let mut seed = 0x9e3779b97f4a7c15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }; + for len in [256, 257, 263, 300, 384, 511, 512] { + let mut modulus: std::vec::Vec = (0..len).map(|_| next()).collect(); + modulus[0] |= 0x80 >> (len % 8); + modulus[len - 1] |= 1; + let n = BigUint::from_bytes_be(&modulus); + for e in [3u32, 17, 65537, 0x8000_0001, 0xffff_ffff] { + let key = PublicKey::from_components(&modulus, &e.to_be_bytes()[e.leading_zeros() as usize / 8..]) + .unwrap(); + let r = BigUint::from(1u8) << (64 * key.words); + assert_eq!(integer(&key.r2), (&r * &r) % &n); + for _ in 0..4 { + let s: std::vec::Vec = (0..len).map(|_| next()).collect(); + let s = BigUint::from_bytes_be(&s) % &n; + let mut bytes = std::vec![0; len]; + let raw = s.to_bytes_be(); + bytes[len - raw.len()..].copy_from_slice(&raw); + let ours = key.representative(&bytes).unwrap(); + assert_eq!(BigUint::from_bytes_be(&ours), s.modpow(&BigUint::from(e), &n)); + } + let mut n_bytes = modulus.clone(); + assert_eq!(key.representative(&n_bytes), Err(Error::InvalidSignature)); + n_bytes[len - 1] -= 1; + assert!(key.representative(&n_bytes).is_ok()); + } + } + } + + fn integer(words: &[u64; WORDS]) -> BigUint { + BigUint::from_bytes_le(&words.iter().flat_map(|w| w.to_le_bytes()).collect::>()) + } +} diff --git a/lib/crypto/src/test_arithmetic.rs b/lib/crypto/src/test_arithmetic.rs index c4dfbfa121..3e68a557ee 100644 --- a/lib/crypto/src/test_arithmetic.rs +++ b/lib/crypto/src/test_arithmetic.rs @@ -1,7 +1,8 @@ //! Integer-oracle checks of reductions, carry chains and representation changes. use crate::{ field25519::F, - montgomery::{Mod, V}, + montgomery::{Modulus, Residue}, + nist::{P256Field, P256Order, P384Field, P384Order}, scalar25519::S, }; use num_bigint::BigUint; @@ -93,58 +94,48 @@ fn scalar25519_reduction_and_signing_arithmetic_match_integer_oracle() { } } -fn check_montgomery(m: &Mod) { - let p = BigUint::from_bytes_le( - &m.p.iter() - .flat_map(|word| word.to_le_bytes()) - .collect::>(), - ); - let values = fixtures(&p, N * 8); +fn check_montgomery, const N: usize>() { + let integer = |words: &[u64; N]| { + BigUint::from_bytes_le(&words.iter().flat_map(|word| word.to_le_bytes()).collect::>()) + }; + let p = integer(&M::P); + let r = BigUint::from(1u8) << (64 * N); + assert_eq!(integer(&M::ONE), &r % &p); + assert_eq!(integer(&M::R2), (&r * &r) % &p); + assert_eq!(M::P[0].wrapping_mul(M::N0), u64::MAX); let words = |value: &BigUint| { let source = value.to_bytes_le(); let mut bytes = std::vec![0; N * 8]; bytes[..source.len()].copy_from_slice(&source); - V(core::array::from_fn(|i| { - u64::from_le_bytes(bytes[8 * i..8 * i + 8].try_into().unwrap()) - })) + core::array::from_fn(|i| u64::from_le_bytes(bytes[8 * i..8 * i + 8].try_into().unwrap())) }; - let integer = |value: &V| { - BigUint::from_bytes_le( - &value - .0 - .iter() - .flat_map(|word| word.to_le_bytes()) - .collect::>(), - ) - }; - assert_eq!(integer(&m.raw(&m.one())), BigUint::from(1u8)); + let values = fixtures(&p, N * 8); + assert!(Residue::::from_integer(&M::P).is_none()); + assert_eq!(integer(&Residue::::one().to_integer()), BigUint::from(1u8)); + let all_ones = (BigUint::from(1u8) << (64 * N)) - 1u8; + assert_eq!(integer(&Residue::::from_wide(&words(&all_ones)).to_integer()), &all_ones % &p); for (i, a) in values.iter().enumerate() { - let a = words(a); - let a_encoded = m.encode_raw(&a); - assert_eq!(integer(&m.raw(&a_encoded)), integer(&a)); + let a_encoded = Residue::::from_integer(&words(a)).unwrap(); + assert_eq!(&integer(&a_encoded.to_integer()), a); for b in [&values[(i * 17 + 3) % values.len()], &values[2]] { - let b_encoded = m.encode_raw(&words(b)); - let sum = m.add(&a_encoded, &b_encoded); - let difference = m.sub(&a_encoded, &b_encoded); - let product = m.mul(&a_encoded, &b_encoded); - for value in [&sum, &difference, &product] { - assert_eq!(m.valid(value), 1); - } - assert_eq!(integer(&m.raw(&sum)), (integer(&a) + b) % &p); - assert_eq!(integer(&m.raw(&difference)), (integer(&a) + &p - b) % &p); - assert_eq!(integer(&m.raw(&product)), (integer(&a) * b) % &p); + let b_encoded = Residue::::from_integer(&words(b)).unwrap(); + let sum = a_encoded.add(&b_encoded); + let difference = a_encoded.sub(&b_encoded); + let product = a_encoded.mul(&b_encoded); + assert_eq!(integer(&sum.to_integer()), (a + b) % &p); + assert_eq!(integer(&difference.to_integer()), (a + &p - b) % &p); + assert_eq!(integer(&product.to_integer()), (a * b) % &p); + assert_eq!(integer(&a_encoded.square().to_integer()), (a * a) % &p); } if (1..=4).contains(&i) { - assert_eq!(m.mul(&a_encoded, &m.invert(&a_encoded)).eq(&m.one()), 1); + assert_eq!(a_encoded.mul(&a_encoded.invert()).eq(&Residue::one()), 1); } } } #[test] fn nist_montgomery_domains_and_carries_match_integer_oracle() { - let c = crate::nist::p256(); - check_montgomery(&c.p); - check_montgomery(&c.n); - let c = crate::nist::p384(); - check_montgomery(&c.p); - check_montgomery(&c.n); + check_montgomery::(); + check_montgomery::(); + check_montgomery::(); + check_montgomery::(); }