Skip to content

toyos-crypto: 2^255 - 19 in four words with BMI2/ADX kernels on x86-64 #1046

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Japabu merged 4 commits into from
Oct 23, 2026
Merged

toyos-crypto: 2^255 - 19 in four words with BMI2/ADX kernels on x86-64 #1046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3be4d6b
toyos-crypto: 2^255 - 19 in four words with BMI2/ADX kernels on x86-64
Japabu Oct 23, 2026
23840e4
issues: the crypto track records #1046's T14 constant-time verdicts
Japabu Oct 23, 2026
9a005e0
toyos-crypto: review r1 of #1046: the UEFI sentence once
Japabu Oct 23, 2026
2840639
issues: the crypto track records #1046's AMD X25519 constant-time ver…
Japabu Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,18 @@ Not consumers:
removed it is not measured. The AMD bench has not run the curves program on
current code. Evidence:
https://github.com/ToyOSOrg/evidence/tree/6993f41cdaa886860b4d40bf766ac092cbe4d14f/pr-1036/r3
- **The T14 at #1046's 6ea592362** (2^255 - 19 in four words with
BMI2/ADX kernels on x86-64), signature 0x806c1, microcode 0xbe,
1,000,000 measurements a routine: X25519 agreement passes with max
|t| 2.30, X25519 public key 1.83, Ed25519 derivation and signature
2.58. The controls held: null 1.25 to 2.29, leaky 1,258 to 1,309.
Evidence:
https://github.com/ToyOSOrg/evidence/tree/84fd34d3e1ad6f631101c8c559bdc2511e7d0677/pr-1046/r3
The AMD bench at 23840e4ed (signature 0x860f01, microcode
0x860010d): X25519 agreement passes with max |t| 3.32 and public key
2.65; null 1.82 and 1.04, leaky 210.6 and 194.7. Its Ed25519 run is
pending. Evidence:
https://github.com/ToyOSOrg/evidence/tree/6488b4fbb6785e65332d03fb504fe3849d89810a/pr-1046/r4
- Their judge asks only that each job ran and its controls held; the
verdicts are this record's. The first boots, of the fast program at
ec5a0cd18, read LEAKS everywhere: that harness made each random
Expand Down
16 changes: 16 additions & 0 deletions lib/crypto/src/arch/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,22 @@ pub(crate) fn p256_square(a: &[u64; 4]) -> Option<[u64; 4]> {
})
}

/// a b mod 2^255 - 19 in `field25519`'s four-word form, by an instruction
/// path; `None` where the CPU has no such path. The time does not depend on
/// a or b.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn f25519_mul(a: &[u64; 4], b: &[u64; 4]) -> Option<[u64; 4]> {
x86_64::f25519_mul(a, b)
}

/// a^2 mod 2^255 - 19, as [`f25519_mul`].
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn f25519_square(a: &[u64; 4]) -> Option<[u64; 4]> {
x86_64::f25519_square(a)
}

/// GHASH key powers, (H^(i+1))' at index i in the instruction paths' form.
pub(crate) type Powers = [u128; 16];

Expand Down
28 changes: 25 additions & 3 deletions lib/crypto/src/arch/x86_64.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! x86-64 instruction paths, admitted by CPUID and, for AVX state, XCR0.
//! UEFI preserves every used register inside each assembly block and takes
//! only AES and CLMUL. Ordinary hosts use Rust intrinsics and the ABI.
//! UEFI takes AES, CLMUL and SHA, whose assembly blocks preserve every vector
//! register they use, and the general-register 2^255 - 19 kernels. Ordinary
//! hosts use Rust intrinsics and the ABI.
//! Unaligned loads admit byte-aligned keys and buffers.

#[cfg(target_os = "uefi")]
Expand All @@ -15,6 +16,7 @@ use super::Powers;
mod aesni;
#[cfg(not(target_os = "uefi"))]
mod chacha;
mod f25519;
#[cfg(not(target_os = "uefi"))]
mod montgomery;
#[cfg(not(target_os = "uefi"))]
Expand Down Expand Up @@ -72,7 +74,6 @@ impl Features {
self.has(Self::BMI2)
}
/// MULX and ADCX/ADOX.
#[cfg(not(target_os = "uefi"))]
pub(super) fn bmi2_adx(self) -> bool {
self.has(Self::BMI2 | Self::ADX)
}
Expand Down Expand Up @@ -203,6 +204,27 @@ pub(super) fn p256_square(a: &[u64; 4]) -> Option<[u64; 4]> {
Some(unsafe { p256::square(a) })
}

/// 2^255 - 19 multiplication by `f25519::mul` where BMI2 and ADX are
/// admitted.
#[inline]
pub(super) fn f25519_mul(a: &[u64; 4], b: &[u64; 4]) -> Option<[u64; 4]> {
if !features().bmi2_adx() {
return None;
}
// SAFETY: CPUID admitted BMI2 and ADX.
Some(unsafe { f25519::mul(a, b) })
}

/// 2^255 - 19 squaring by `f25519::square`, as [`f25519_mul`].
#[inline]
pub(super) fn f25519_square(a: &[u64; 4]) -> Option<[u64; 4]> {
if !features().bmi2_adx() {
return None;
}
// SAFETY: CPUID admitted BMI2 and ADX.
Some(unsafe { f25519::square(a) })
}

/// A vector from 16 bytes by value. With no pointer there is no copy
/// precondition, which debug assertions would check per access in hot loops.
#[cfg(not(target_os = "uefi"))]
Expand Down
229 changes: 229 additions & 0 deletions lib/crypto/src/arch/x86_64/f25519.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
//! 2^255 - 19 multiplication and squaring on BMI2 and ADX (Intel SDM Vol.
//! 2A, ADCX, ADOX and MULX), in `field25519`'s four-word form: any value
//! below 2^256 in, such a value out. No branch or address depends on the
//! operands.

use core::arch::asm;

/// a b mod p, below 2^256 but not necessarily below p.
///
/// Row i adds a_i b to the running words, low halves on CF through `adcx`
/// and high halves on OF through `adox`; the eight-word product is then
/// folded as t_lo + 38 t_hi (2^256 = 38 mod p), and the fold's top word
/// times 38 once more. A carry out of that last addition leaves the low
/// word below 2^11, so its 38 is added without a carry.
///
/// # Safety
/// The CPU has BMI2 and ADX.
#[inline]
pub(super) unsafe fn mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] {
let (r0, r1, r2, r3): (u64, u64, u64, u64);
// SAFETY: the template reads the four words of `a` and of `b`, writes
// only its declared registers and leaves the stack alone.
unsafe {
asm!(
// Row 0 into (T0 .. T4).
"mov rdx, [{a}]",
"mulx {T1}, {T0}, [{b}]",
"mulx {T2}, {lo}, [{b} + 8]",
"add {T1}, {lo}",
"mulx {T3}, {lo}, [{b} + 16]",
"adc {T2}, {lo}",
"mulx {T4}, {lo}, [{b} + 24]",
"adc {T3}, {lo}",
"adc {T4}, 0",
// Row 1 into (T1 .. T5).
"mov rdx, [{a} + 8]",
"xor {lo:e}, {lo:e}",
"mulx {hi}, {lo}, [{b}]",
"adcx {T1}, {lo}",
"adox {T2}, {hi}",
"mulx {hi}, {lo}, [{b} + 8]",
"adcx {T2}, {lo}",
"adox {T3}, {hi}",
"mulx {hi}, {lo}, [{b} + 16]",
"adcx {T3}, {lo}",
"adox {T4}, {hi}",
"mulx {T5}, {lo}, [{b} + 24]",
"adcx {T4}, {lo}",
"mov {lo:e}, 0",
"adox {T5}, {lo}",
"adcx {T5}, {lo}",
// Row 2 into (T2 .. T6).
"mov rdx, [{a} + 16]",
"xor {lo:e}, {lo:e}",
"mulx {hi}, {lo}, [{b}]",
"adcx {T2}, {lo}",
"adox {T3}, {hi}",
"mulx {hi}, {lo}, [{b} + 8]",
"adcx {T3}, {lo}",
"adox {T4}, {hi}",
"mulx {hi}, {lo}, [{b} + 16]",
"adcx {T4}, {lo}",
"adox {T5}, {hi}",
"mulx {T6}, {lo}, [{b} + 24]",
"adcx {T5}, {lo}",
"mov {lo:e}, 0",
"adox {T6}, {lo}",
"adcx {T6}, {lo}",
// Row 3 into (T3 .. T7).
"mov rdx, [{a} + 24]",
"xor {lo:e}, {lo:e}",
"mulx {hi}, {lo}, [{b}]",
"adcx {T3}, {lo}",
"adox {T4}, {hi}",
"mulx {hi}, {lo}, [{b} + 8]",
"adcx {T4}, {lo}",
"adox {T5}, {hi}",
"mulx {hi}, {lo}, [{b} + 16]",
"adcx {T5}, {lo}",
"adox {T6}, {hi}",
"mulx {T7}, {lo}, [{b} + 24]",
"adcx {T6}, {lo}",
"mov {lo:e}, 0",
"adox {T7}, {lo}",
"adcx {T7}, {lo}",
// (T0 .. T3) + 38 (T4 .. T7), the top word into T4.
"mov edx, 38",
"xor {lo:e}, {lo:e}",
"mulx {hi}, {lo}, {T4}",
"adcx {T0}, {lo}",
"adox {T1}, {hi}",
"mulx {hi}, {lo}, {T5}",
"adcx {T1}, {lo}",
"adox {T2}, {hi}",
"mulx {hi}, {lo}, {T6}",
"adcx {T2}, {lo}",
"adox {T3}, {hi}",
"mulx {T4}, {lo}, {T7}",
"adcx {T3}, {lo}",
"mov {lo:e}, 0",
"adox {T4}, {lo}",
"adcx {T4}, {lo}",
// + 38 T4, then 38 for a carry out.
"imul {T4}, {T4}, 38",
"add {T0}, {T4}",
"adc {T1}, 0",
"adc {T2}, 0",
"adc {T3}, 0",
"sbb {lo}, {lo}",
"and {lo}, 38",
"add {T0}, {lo}",
a = in(reg) a.as_ptr(),
b = in(reg) b.as_ptr(),
T0 = out(reg) r0,
T1 = out(reg) r1,
T2 = out(reg) r2,
T3 = out(reg) r3,
T4 = out(reg) _,
T5 = out(reg) _,
T6 = out(reg) _,
T7 = out(reg) _,
lo = out(reg) _,
hi = out(reg) _,
out("rdx") _,
options(pure, readonly, nostack),
);
}
[r0, r1, r2, r3]
}

/// a^2 mod p as [`mul`]: the six cross products once, doubled, the four
/// squares added, then `mul`'s fold.
///
/// # Safety
/// The CPU has BMI2 and ADX.
#[inline]
pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] {
let (r0, r1, r2, r3): (u64, u64, u64, u64);
// SAFETY: the template reads the four words of `a`, writes only its
// declared registers and leaves the stack alone.
unsafe {
asm!(
// Cross products into (T1 .. T6).
"mov rdx, [{a}]",
"mulx {T2}, {T1}, [{a} + 8]",
"mulx {T3}, {lo}, [{a} + 16]",
"add {T2}, {lo}",
"mulx {T4}, {lo}, [{a} + 24]",
"adc {T3}, {lo}",
"adc {T4}, 0",
"mov rdx, [{a} + 8]",
"mulx {hi}, {lo}, [{a} + 16]",
"add {T3}, {lo}",
"adc {T4}, {hi}",
"mulx {T5}, {lo}, [{a} + 24]",
"adc {T5}, 0",
"add {T4}, {lo}",
"adc {T5}, 0",
"mov rdx, [{a} + 16]",
"mulx {T6}, {lo}, [{a} + 24]",
"add {T5}, {lo}",
"adc {T6}, 0",
// Doubled into (T1 .. T7).
"xor {T7:e}, {T7:e}",
"add {T1}, {T1}",
"adc {T2}, {T2}",
"adc {T3}, {T3}",
"adc {T4}, {T4}",
"adc {T5}, {T5}",
"adc {T6}, {T6}",
"adc {T7}, 0",
// The squares added into (T0 .. T7).
"mov rdx, [{a}]",
"mulx {hi}, {T0}, rdx",
"add {T1}, {hi}",
"mov rdx, [{a} + 8]",
"mulx {hi}, {lo}, rdx",
"adc {T2}, {lo}",
"adc {T3}, {hi}",
"mov rdx, [{a} + 16]",
"mulx {hi}, {lo}, rdx",
"adc {T4}, {lo}",
"adc {T5}, {hi}",
"mov rdx, [{a} + 24]",
"mulx {hi}, {lo}, rdx",
"adc {T6}, {lo}",
"adc {T7}, {hi}",
// The fold, as `mul`'s.
"mov edx, 38",
"xor {lo:e}, {lo:e}",
"mulx {hi}, {lo}, {T4}",
"adcx {T0}, {lo}",
"adox {T1}, {hi}",
"mulx {hi}, {lo}, {T5}",
"adcx {T1}, {lo}",
"adox {T2}, {hi}",
"mulx {hi}, {lo}, {T6}",
"adcx {T2}, {lo}",
"adox {T3}, {hi}",
"mulx {T4}, {lo}, {T7}",
"adcx {T3}, {lo}",
"mov {lo:e}, 0",
"adox {T4}, {lo}",
"adcx {T4}, {lo}",
"imul {T4}, {T4}, 38",
"add {T0}, {T4}",
"adc {T1}, 0",
"adc {T2}, 0",
"adc {T3}, 0",
"sbb {lo}, {lo}",
"and {lo}, 38",
"add {T0}, {lo}",
a = in(reg) a.as_ptr(),
T0 = out(reg) r0,
T1 = out(reg) r1,
T2 = out(reg) r2,
T3 = out(reg) r3,
T4 = out(reg) _,
T5 = out(reg) _,
T6 = out(reg) _,
T7 = out(reg) _,
lo = out(reg) _,
hi = out(reg) _,
out("rdx") _,
options(pure, readonly, nostack),
);
}
[r0, r1, r2, r3]
}
5 changes: 3 additions & 2 deletions lib/crypto/src/ed25519.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,8 @@ impl Projective {
fn double(&self) -> Completed {
let a = self.x.square();
let b = self.y.square();
let c = self.z.square().mul_small(2);
let zz = self.z.square();
let c = zz.add(&zz).carry();
let h = a.add(&b).carry();
let g = a.sub(&b).carry();
Completed {
Expand Down Expand Up @@ -338,7 +339,7 @@ const fn normalize<const N: usize>(points: &[[Tight; 4]; N]) -> [Niels; N] {
prefix[i] = prefix[i - 1].times(&points[i][2]);
i += 1;
}
let mut inverse = prefix[N - 1].invert();
let mut inverse = prefix[N - 1].inverted();
let mut table = [Niels::IDENTITY; N];
let mut i = N;
while i > 0 {
Expand Down
Loading