From 329f883894a0d31c0582856d4da3c2002f2aefa5 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 13:29:34 -0700 Subject: [PATCH 01/25] perf(carmenta): depthwise kernel, fused epilogues, 1x1-as-matmul: 4.35x on PdfCraft's pages PdfCraft (artcraft-15) measured mobiledet-svtr at 6.7-7.3x slower than ocrs, recognition ~300 ms a line. The cause was candle's grouped convolution: it runs a depthwise layer as one im2col convolution PER CHANNEL (candle-core 0.11 conv.rs:262), and SVTR has 14 of them per line, MobileDet 14 per page. - dwconv: a direct depthwise stencil, per-axis stride and padding, parallel over planes. On AVX2+FMA it accumulates with fused multiply-adds in candle's (ky, kx) order and is BIT-IDENTICAL to candle's grouped path (unit test on every SVTR and MobileDet shape). FFAI_DWCONV=0 restores candle. - SVTR: each backbone block's bias + LAB + hardswish + LAB (eight candle passes) is one pass with candle's exact arithmetic; 1x1 convs are a plain NCHW matmul; the (2,2) stem uses candle's native stride. FFAI_SVTR_FUSED=0 restores the old chain. The forward now reports rec_cnn / rec_rnn / rec_head on FFAI_PROFILE. - MobileDet: the same epilogue fusion and 1x1 matmul, 3x3 neck and head convs through conv3x3's register-tiled kernel, and the two k=s=2 transposed convs as a matmul plus interleave. FFAI_MDET_FAST=0 restores candle. Gates: - SVTR probabilities on 30 real line crops (5 pages): bit-identical to the candle path, so the paddle oracle's 8.857e-5 carries over unchanged. - MobileDet paddle oracle: max |dprob| 8.294e-5 -> 7.537e-5, 0/65536 binarised disagreement, boxes 4/4. - PdfCraft's 42-page corpus: identical text and line boxes on every page. Speed (examples/pdfcraft_bench, i7-14650HX): - 42 pages, 24 threads: 175.2 s -> 40.3 s (4.35x). - one 300 dpi page, 1 thread: 6.07 s -> 3.54 s; det 2.24 -> 0.92 s, rec 164 -> 109 ms a line. Also: the embedded-manifest sync test compares with line endings normalised; a core.autocrlf checkout can hold the two copies with different endings while git considers both clean. Co-Authored-By: Claude Opus 5.5 --- .../ffai-carmenta/examples/pdfcraft_bench.rs | 115 +++++++ crates/ffai-carmenta/examples/svtr_logits.rs | 45 +++ crates/ffai-carmenta/src/dwconv.rs | 298 ++++++++++++++++++ crates/ffai-carmenta/src/lib.rs | 1 + crates/ffai-carmenta/src/manifests.rs | 5 +- crates/ffai-carmenta/src/mobiledet.rs | 185 ++++++++++- crates/ffai-carmenta/src/par.rs | 24 ++ crates/ffai-carmenta/src/svtr.rs | 172 +++++++++- 8 files changed, 820 insertions(+), 25 deletions(-) create mode 100644 crates/ffai-carmenta/examples/pdfcraft_bench.rs create mode 100644 crates/ffai-carmenta/examples/svtr_logits.rs create mode 100644 crates/ffai-carmenta/src/dwconv.rs diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs new file mode 100644 index 00000000..13bf467d --- /dev/null +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -0,0 +1,115 @@ +//! Read a list of pages with `mobiledet-svtr` from an OFFLINE weights root and +//! write one JSON line per page in the shape PdfCraft's `score.py` reads: +//! `{"page", "ms", "lines": [[[text, [l, t, r, b]], ...], ...]}`. +//! +//! It exists so a caller's corpus (artcraft's pdfcraft-ocr pages, rendered +//! with ground truth by construction) can be scored against THIS tree, not +//! only against the release on crates.io. A line with word geometry emits one +//! entry per word; a line without it emits the whole line as one entry with +//! the line box, which the scorer's CER is indifferent to. +//! +//! Usage: `pdfcraft_bench ...` +//! Env: `BENCH_REC=crnn` for mobiledet-crnn, `BENCH_ORIENT=1` for +//! `auto_orient`, `BENCH_LANG=en,fr` for `languages`. + +#[global_allocator] +static GLOBAL: rusty_alloc_api::RustyAlloc = rusty_alloc_api::RustyAlloc; + +use std::fmt::Write as _; +use std::path::Path; +use std::time::Instant; + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_core::engine::{OcrEngine, OcrOptions}; + +fn esc(s: &str) -> String { + let mut o = String::with_capacity(s.len() + 2); + for c in s.chars() { + match c { + '"' => o.push_str("\\\""), + '\\' => o.push_str("\\\\"), + c if (c as u32) < 0x20 => { + let _ = write!(o, "\\u{:04x}", c as u32); + } + c => o.push(c), + } + } + o +} + +fn main() { + let mut args = std::env::args().skip(1); + let root = args.next().expect("usage: pdfcraft_bench ..."); + let pages: Vec = args.collect(); + let rec = match std::env::var("BENCH_REC").as_deref() { + Ok("crnn") => RecStage::Crnn, + _ => RecStage::Svtr, + }; + let engine = CraftCrnn::offline(rec, DetStage::MobileDet, Path::new(&root)); + let t = Instant::now(); + engine.preload().expect("preload"); + eprintln!("load {:.1} ms", t.elapsed().as_secs_f64() * 1e3); + let opts = OcrOptions { + auto_orient: std::env::var("BENCH_ORIENT").is_ok_and(|v| v == "1"), + languages: std::env::var("BENCH_LANG") + .map(|v| v.split(',').map(str::to_string).collect()) + .unwrap_or_default(), + ..OcrOptions::default() + }; + let all = Instant::now(); + let (mut words_total, mut lines_with_words, mut lines_total) = (0usize, 0usize, 0usize); + for p in &pages { + let img = ffai_media::load_image(Path::new(p)).expect("load image"); + let t = Instant::now(); + let out = engine.recognize(&img, &opts).expect("recognize"); + let ms = t.elapsed().as_secs_f64() * 1e3; + let name = Path::new(p).file_stem().unwrap().to_string_lossy().into_owned(); + let mut s = format!("{{\"page\":\"{}\",\"ms\":{ms:.1},\"lines\":[", esc(&name)); + for (li, line) in out.lines().enumerate() { + lines_total += 1; + if li > 0 { + s.push(','); + } + s.push('['); + let entries: Vec<(String, [f32; 4])> = if line.words.is_empty() { + let b = line.bbox.map_or([0.0; 4], |b| [b.x, b.y, b.x + b.width, b.y + b.height]); + vec![(line.text.clone(), b)] + } else { + lines_with_words += 1; + words_total += line.words.len(); + line.words + .iter() + .map(|w| { + let b = w.bbox.map_or([0.0; 4], |b| [b.x, b.y, b.x + b.width, b.y + b.height]); + (w.text.clone(), b) + }) + .collect() + }; + for (wi, (text, b)) in entries.iter().enumerate() { + if wi > 0 { + s.push(','); + } + let _ = write!( + s, + "[\"{}\",[{:.1},{:.1},{:.1},{:.1}]]", + esc(text), + b[0], + b[1], + b[2], + b[3] + ); + } + s.push(']'); + } + s.push_str("]}"); + println!("{s}"); + } + eprintln!( + "pages {} total {:.1} ms; lines {lines_total}, with words {lines_with_words}, words {words_total}", + pages.len(), + all.elapsed().as_secs_f64() * 1e3 + ); + if std::env::var_os("FFAI_PROFILE").is_some() { + eprint!("{}", ffai_carmenta::profile::profile().report()); + } +} diff --git a/crates/ffai-carmenta/examples/svtr_logits.rs b/crates/ffai-carmenta/examples/svtr_logits.rs new file mode 100644 index 00000000..4f29b734 --- /dev/null +++ b/crates/ffai-carmenta/examples/svtr_logits.rs @@ -0,0 +1,45 @@ +//! Write SVTR's output probabilities for real line crops to a file, so two +//! builds — or one build under two environments — can be compared tensor for +//! tensor. +//! +//! This is the gate for changing how the recognizer COMPUTES without changing +//! what it computes. `svtr_oracle` holds the port against paddle, but its +//! input fixture is a binary that is not in every checkout; this needs only +//! the weights and an image. Run it once with `FFAI_SVTR_FUSED=0 +//! FFAI_DWCONV=0` (candle's own ops, the arm the oracle pinned at 8.857e-5) +//! and once without, then compare the files: the distance between them is +//! what the fast path spends of the oracle's 1e-4 bar. +//! +//! Usage: `svtr_logits ...` +//! The output is every crop's `(T, n_class)` probabilities as little-endian +//! f32, concatenated in argument order. + +use std::io::Write as _; +use std::path::Path; + +use candle_core::Device; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let usage = "usage: svtr_logits ..."; + let root = args.next().ok_or(usage)?; + let image = args.next().ok_or(usage)?; + let out = args.next().ok_or(usage)?; + let dev = Device::Cpu; + let rec = Path::new(&root).join("ppocrv5-mobile-rec"); + let charset = ffai_carmenta::svtr::load_charset(&rec.join("charset.txt"))?; + let m = ffai_carmenta::svtr::load(&rec.join("rec.safetensors"), &dev, charset.len() + 1)?; + let img = ffai_media::load_image(Path::new(&image))?; + let mut f = std::io::BufWriter::new(std::fs::File::create(&out)?); + for b in args { + let v: Vec = b.split(',').map(str::parse).collect::>()?; + let [x0, y0, x1, y1] = v[..] else { return Err(usage.into()) }; + let x = ffai_carmenta::svtr::svtr_input(&img, x0, y0, x1, y1, &dev)?; + let p = m.forward(&x)?.flatten_all()?.to_vec1::()?; + for v in &p { + f.write_all(&v.to_le_bytes())?; + } + eprintln!("{b}: {} values", p.len()); + } + Ok(()) +} diff --git a/crates/ffai-carmenta/src/dwconv.rs b/crates/ffai-carmenta/src/dwconv.rs new file mode 100644 index 00000000..741aaea8 --- /dev/null +++ b/crates/ffai-carmenta/src/dwconv.rs @@ -0,0 +1,298 @@ +//! Depthwise convolution, because candle's CPU backend has none. +//! +//! # Why this exists +//! +//! `Tensor::conv2d` with `groups > 1` does `chunk(groups, 1)`, one +//! `conv2d_single_group` per group, then `cat` (candle-core 0.11 +//! `conv.rs:262`). A depthwise layer has `groups == channels`, so SVTR's +//! 480-channel depthwise runs **480 separate single-channel convolutions**, +//! each through the full im2col machinery, plus 480 allocations and a 480-way +//! concatenation. ffai-diana measured the same mechanism at 72.9x slower per +//! FLOP than the dense convolution beside it (`ffai-diana/src/dwconv.rs`). +//! +//! SVTR has 14 depthwise layers per forward and runs one forward per text +//! line; PdfCraft measured recognition at ~300 ms a line, 97 % of the page. +//! +//! # What this does instead +//! +//! A direct stencil: one pass per output row, one AXPY per tap over the row, +//! parallel over `(batch, channel)` planes. Any kernel size, independent +//! strides and paddings per axis (SVTR uses `(2, 1)` and `(1, 2)`), so the +//! callers no longer compute a stride-1 output and throw half of it away. +//! +//! # Correctness +//! +//! On AVX2+FMA hardware the result is BIT-IDENTICAL to candle's grouped path: +//! see [`plane_fma`]. Elsewhere it is the same products summed in the same +//! order with a separate multiply and add, so it agrees to a tolerance. +//! `tests::matches_candle_grouped` holds both against candle on every shape +//! the two models use. + +use candle_core::{CpuStorage, CustomOp2, Layout, Result, Shape, Tensor}; + +use crate::par::prelude::*; + +struct DwOp { + stride: (usize, usize), + pad: (usize, usize), +} + +/// Output length of a strided, padded, valid convolution along one axis. +fn out_len(n: usize, k: usize, s: usize, p: usize) -> usize { + (n + 2 * p).saturating_sub(k) / s + 1 +} + +/// One output plane. `inp` is one `h x w` input plane, `wt` its `kh x kw` +/// kernel, `out` the `ho x wo` result (overwritten). `madd(w, x, acc)` is the +/// accumulation step: see [`plane_fma`] for why it is a parameter. +#[allow(clippy::too_many_arguments)] +#[inline(always)] +fn plane_with( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + (sh, sw): (usize, usize), + (ph, pw): (usize, usize), + out: &mut [f32], + wo: usize, + madd: impl Fn(f32, f32, f32) -> f32, +) { + out.fill(0.0); + for (oy, orow) in out.chunks_exact_mut(wo).enumerate() { + for ky in 0..kh { + // iy = oy*sh + ky - ph, skipped when it falls in the zero padding. + let Some(iy) = (oy * sh + ky).checked_sub(ph) else { continue }; + if iy >= h { + continue; + } + let irow = &inp[iy * w..(iy + 1) * w]; + for kx in 0..kw { + let wv = wt[ky * kw + kx]; + // Valid ox: 0 <= ox*sw + kx - pw < w. + let lo = pw.saturating_sub(kx).div_ceil(sw); + let hi = if w + pw > kx { (w + pw - kx - 1) / sw + 1 } else { 0 }; + let hi = hi.min(wo); + if lo >= hi { + continue; + } + if sw == 1 { + // Contiguous on both sides: the loop the compiler vectorises. + let src = &irow[lo + kx - pw..hi + kx - pw]; + for (o, &v) in orow[lo..hi].iter_mut().zip(src) { + *o = madd(wv, v, *o); + } + } else { + for ox in lo..hi { + orow[ox] = madd(wv, irow[ox * sw + kx - pw], orow[ox]); + } + } + } + } + } +} + +/// The portable plane: a separate multiply and add per tap. +#[allow(clippy::too_many_arguments)] +fn plane( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + stride: (usize, usize), + pad: (usize, usize), + out: &mut [f32], + wo: usize, +) { + plane_with(inp, h, w, wt, kh, kw, stride, pad, out, wo, |w, x, acc| acc + w * x); +} + +/// The same plane accumulated with FUSED multiply-adds, which is what makes it +/// agree with candle bit for bit rather than to a tolerance. +/// +/// Candle's grouped path im2cols each channel and hands the `kh * kw` taps to +/// gemm, whose microkernel accumulates `acc = fma(a, b, acc)` over them in +/// `(ky, kx)` order from zero. This walks the taps in that same order with the +/// same fused step, and a tap that lands in the padding is skipped where gemm +/// adds `w * 0` — which leaves the accumulator unchanged. `mul_add` is a libm +/// call unless the function is compiled with FMA, hence the target feature. +/// +/// # Safety +/// The CPU must support AVX2 and FMA (checked by the caller). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +#[allow(clippy::too_many_arguments)] +unsafe fn plane_fma( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + stride: (usize, usize), + pad: (usize, usize), + out: &mut [f32], + wo: usize, +) { + plane_with(inp, h, w, wt, kh, kw, stride, pad, out, wo, |w, x, acc| w.mul_add(x, acc)); +} + +fn has_fma() -> bool { + #[cfg(target_arch = "x86_64")] + { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| { + std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") + }) + } + #[cfg(not(target_arch = "x86_64"))] + { + false + } +} + +impl CustomOp2 for DwOp { + fn name(&self) -> &'static str { + "ffai-carmenta-dwconv" + } + + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + ) -> Result<(CpuStorage, Shape)> { + let (CpuStorage::F32(inp), CpuStorage::F32(wt)) = (s1, s2) else { + candle_core::bail!("ffai-carmenta-dwconv: f32 only") + }; + let (n, c, h, w) = l1.shape().dims4()?; + let (wc, one, kh, kw) = l2.shape().dims4()?; + if wc != c || one != 1 { + candle_core::bail!( + "ffai-carmenta-dwconv: weight {:?} is not depthwise for {c} channels", + l2.shape() + ); + } + let inp = match l1.contiguous_offsets() { + Some((o, e)) => &inp[o..e], + None => candle_core::bail!("ffai-carmenta-dwconv: non-contiguous input"), + }; + let wt = match l2.contiguous_offsets() { + Some((o, e)) => &wt[o..e], + None => candle_core::bail!("ffai-carmenta-dwconv: non-contiguous weight"), + }; + let (sh, sw) = self.stride; + let (ph, pw) = self.pad; + if h + 2 * ph < kh || w + 2 * pw < kw || sh == 0 || sw == 0 { + candle_core::bail!("ffai-carmenta-dwconv: kernel larger than padded input"); + } + let (ho, wo) = (out_len(h, kh, sh, ph), out_len(w, kw, sw, pw)); + let mut out = vec![0f32; n * c * ho * wo]; + let fma = has_fma(); + out.par_chunks_mut(ho * wo).enumerate().for_each(|(i, o)| { + let ch = i % c; + let (x, k) = (&inp[i * h * w..(i + 1) * h * w], &wt[ch * kh * kw..(ch + 1) * kh * kw]); + #[cfg(target_arch = "x86_64")] + if fma { + // SAFETY: `has_fma` confirmed AVX2 and FMA at run time. + unsafe { plane_fma(x, h, w, k, kh, kw, self.stride, self.pad, o, wo) }; + return; + } + let _ = fma; + plane(x, h, w, k, kh, kw, self.stride, self.pad, o, wo); + }); + Ok((CpuStorage::F32(out), (n, c, ho, wo).into())) + } +} + +/// Depthwise convolution of `x` `(n, c, h, w)` with `weight` `(c, 1, kh, kw)`, +/// zero padding `pad = (ph, pw)` on both sides of each axis and per-axis +/// `stride`. No bias; callers add their own. +pub fn depthwise( + x: &Tensor, + weight: &Tensor, + stride: (usize, usize), + pad: (usize, usize), +) -> Result { + let x = x.contiguous()?; + let weight = weight.contiguous()?; + x.apply_op2_no_bwd(&weight, &DwOp { stride, pad }) +} + +/// Whether a conv with this weight and group count is one [`depthwise`] +/// handles: one group per channel, one input channel per group, f32, CPU. +/// `FFAI_DWCONV=0` sends every call back to candle's grouped path, so the +/// two arms stay one environment variable apart for an A/B. +#[must_use] +pub fn applies(x: &Tensor, weight: &Tensor, groups: usize) -> bool { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + let off = *OFF.get_or_init(|| std::env::var("FFAI_DWCONV").as_deref() == Ok("0")); + !off && groups > 1 + && weight.dims4().is_ok_and(|(oc, icg, _, _)| oc == groups && icg == 1) + && x.dims4().is_ok_and(|(_, c, _, _)| c == groups) + && x.dtype() == candle_core::DType::F32 + && x.device().is_cpu() +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + /// Candle's grouped conv with SVTR's emulation of asymmetric stride and + /// padding: pad, stride-1 conv, keep every `stride`-th row and column. + fn reference(x: &Tensor, w: &Tensor, stride: (usize, usize), pad: (usize, usize)) -> Tensor { + let c = x.dim(1).unwrap(); + let xp = x.pad_with_zeros(2, pad.0, pad.0).unwrap().pad_with_zeros(3, pad.1, pad.1).unwrap(); + let y = xp.conv2d(w, 0, 1, 1, c).unwrap(); + let pick = |y: Tensor, dim: usize, s: usize| { + if s == 1 { + return y; + } + let n = y.dim(dim).unwrap().div_ceil(s); + let idx: Vec = (0..n).map(|i| (i * s) as u32).collect(); + y.index_select(&Tensor::from_vec(idx, n, y.device()).unwrap(), dim).unwrap() + }; + pick(pick(y, 2, stride.0), 3, stride.1) + } + + #[test] + fn matches_candle_grouped() { + let d = Device::Cpu; + // (n, c, h, w, k, stride, pad): every depthwise shape in SVTR and + // MobileDet, plus odd sizes and a batch. + let cases = [ + (1, 16, 24, 161, 3, (1, 1), (1, 1)), + (1, 64, 24, 161, 3, (2, 1), (1, 1)), + (1, 128, 12, 161, 3, (1, 2), (1, 1)), + (1, 240, 12, 81, 5, (1, 1), (2, 2)), + (1, 240, 12, 81, 5, (2, 1), (2, 2)), + (1, 480, 6, 81, 5, (2, 1), (2, 2)), + (1, 32, 37, 53, 3, (2, 2), (1, 1)), + (1, 192, 19, 23, 5, (1, 1), (2, 2)), + (3, 48, 9, 17, 3, (1, 1), (1, 1)), + (2, 8, 5, 4, 5, (1, 1), (2, 2)), + ]; + for (n, c, h, w, k, stride, pad) in cases { + let x = Tensor::randn(0f32, 1.0, (n, c, h, w), &d).unwrap(); + let wt = Tensor::randn(0f32, 1.0, (c, 1, k, k), &d).unwrap(); + let got = depthwise(&x, &wt, stride, pad).unwrap(); + let want = reference(&x, &wt, stride, pad); + assert_eq!(got.dims(), want.dims(), "shape for {:?}", (n, c, h, w, k, stride, pad)); + let diff = (got.clone() - &want).unwrap().abs().unwrap().max_all().unwrap(); + let diff = diff.to_scalar::().unwrap(); + assert!(diff < 1e-4, "max abs diff {diff} for {:?}", (n, c, h, w, k, stride, pad)); + if has_fma() { + let (gv, wv) = (got.flatten_all().unwrap().to_vec1::().unwrap(), + want.flatten_all().unwrap().to_vec1::().unwrap()); + assert!(gv == wv, "FMA arm not bit-identical (max {diff}) for {:?}", + (n, c, h, w, k, stride, pad)); + } + } + } +} diff --git a/crates/ffai-carmenta/src/lib.rs b/crates/ffai-carmenta/src/lib.rs index 7d8bc0ce..b8e0c536 100644 --- a/crates/ffai-carmenta/src/lib.rs +++ b/crates/ffai-carmenta/src/lib.rs @@ -45,6 +45,7 @@ pub mod rules; pub mod harvest; pub mod image; pub mod conv3x3; +pub mod dwconv; /// The LIVE frame reader — **native only.** /// /// It is the one part of Carmenta that is architecturally threaded: a diff --git a/crates/ffai-carmenta/src/manifests.rs b/crates/ffai-carmenta/src/manifests.rs index 6055db4d..963dd8f8 100644 --- a/crates/ffai-carmenta/src/manifests.rs +++ b/crates/ffai-carmenta/src/manifests.rs @@ -85,8 +85,11 @@ mod tests { for (name, toml) in EMBEDDED { let on_disk = std::fs::read_to_string(repo.join(format!("{name}.toml"))) .unwrap_or_else(|e| panic!("repository models/{name}.toml: {e}")); + // Line endings normalised: with `core.autocrlf` a Windows checkout + // can hold the two copies with different endings while git + // considers both clean, and that is not drift. assert!( - on_disk == *toml, + on_disk.replace("\r\n", "\n") == toml.replace("\r\n", "\n"), "crates/ffai-carmenta/models/{name}.toml has drifted from models/{name}.toml — \ copy the repository file over the crate's" ); diff --git a/crates/ffai-carmenta/src/mobiledet.rs b/crates/ffai-carmenta/src/mobiledet.rs index a160448e..7d8c320d 100644 --- a/crates/ffai-carmenta/src/mobiledet.rs +++ b/crates/ffai-carmenta/src/mobiledet.rs @@ -109,11 +109,176 @@ impl RepConv { } fn forward(&self, x: &Tensor) -> Result { - let x = self.conv.forward(x)?; - match self.act { - Some((s, b)) => hardswish(&x)?.affine(s, b), - None => Ok(x), + let cfg = self.conv.config(); + let w = self.conv.weight(); + let y = if crate::dwconv::applies(x, w, cfg.groups) { + // The 14 `dw_conv`s: candle runs a grouped conv as one + // convolution per channel (see `dwconv`). + let p = cfg.padding; + crate::dwconv::depthwise(x, w, (cfg.stride, cfg.stride), (p, p))? + } else if fast() && is_pointwise(w, cfg) { + pointwise(x, w)? + } else { + let y = self.conv.forward(x)?; + return match self.act { + Some((s, b)) => hardswish(&y)?.affine(s, b), + None => Ok(y), + }; + }; + if !fast() { + let y = match self.conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?)?, + None => y, + }; + return match self.act { + Some((s, b)) => hardswish(&y)?.affine(s, b), + None => Ok(y), + }; } + let bias = match self.conv.bias() { + Some(b) => b.to_vec1::()?, + None => Vec::new(), + }; + y.contiguous()?.apply_op1_no_bwd(&RepEpilogue { bias, act: self.act }) + } +} + +/// `FFAI_MDET_FAST=0` sends the detector back through candle's own ops — 1x1 +/// convs, 3x3 convs, transposed convs and the unfused activation chain — so +/// every fast path below stays one variable away from its oracle. +fn fast() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("FFAI_MDET_FAST").as_deref() != Ok("0")) +} + +fn is_pointwise(w: &Tensor, cfg: &Conv2dConfig) -> bool { + w.dims4().is_ok_and(|(_, _, kh, kw)| (kh, kw) == (1, 1)) + && cfg.stride == 1 + && cfg.padding == 0 + && cfg.groups == 1 + && cfg.dilation == 1 +} + +/// A 1x1 conv in NCHW is `(oc, ic) x (ic, h*w)`: no im2col copy and no layout +/// permute, both of which candle's conv path builds. No bias. +fn pointwise(x: &Tensor, w: &Tensor) -> Result { + let (oc, ic, _, _) = w.dims4()?; + let (n, _, h, wd) = x.dims4()?; + w.reshape((oc, ic))? + .broadcast_matmul(&x.reshape((n, ic, h * wd))?)? + .reshape((n, oc, h, wd)) +} + +/// A conv layer through the fastest path that applies: 1x1 as a matmul, 3x3 +/// through `conv3x3`'s register-tiled kernel (which falls back to candle for +/// any shape it does not cover). +fn conv_fast(conv: &Conv2d, x: &Tensor) -> Result { + if !fast() { + return conv.forward(x); + } + if is_pointwise(conv.weight(), conv.config()) { + let y = pointwise(x, conv.weight())?; + return match conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?), + None => Ok(y), + }; + } + crate::conv3x3::apply(x, conv) +} + +/// A transposed conv whose kernel equals its stride (DBHead's two, k = s = 2) +/// tiles its output without overlap: each input pixel writes its own +/// `kh x kw` block. That is one matmul, `(oc*kh*kw, ic) x (ic, h*w)`, and an +/// interleave, where candle's conv-transpose walks every tap with a dot +/// product per output pixel. +fn conv_transpose_fast(conv: &ConvTranspose2d, x: &Tensor) -> Result { + let cfg = conv.config(); + let w = conv.weight(); + let (ic, oc, kh, kw) = w.dims4()?; + let tiles = cfg.padding == 0 + && cfg.output_padding == 0 + && cfg.dilation == 1 + && kh == cfg.stride + && kw == cfg.stride; + if !fast() || !tiles { + return conv.forward(x); + } + let (n, _, h, wd) = x.dims4()?; + let wt = w.permute((1, 2, 3, 0))?.reshape((oc * kh * kw, ic))?; + let y = wt.broadcast_matmul(&x.reshape((n, ic, h * wd))?)?; // (n, oc*kh*kw, h*w) + let y = y + .reshape((n, oc, kh, kw, h, wd))? + .permute((0, 1, 4, 2, 5, 3))? // (n, oc, h, kh, w, kw) + .reshape((n, oc, h * kh, wd * kw))?; + match conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, oc, 1, 1))?), + None => Ok(y), + } +} + +/// Bias, then (when the layer has one) hardswish and the scalar affine, in one +/// pass where candle's chain is up to seven full passes over the activation. +/// The arithmetic is candle's, operation for operation (`affine(m, a)` is +/// `v * m + a` with both cast to f32, `clamp` is maximum-then-minimum), so the +/// result is bit-identical. +struct RepEpilogue { + bias: Vec, + act: Option<(f64, f64)>, +} + +impl candle_core::CustomOp1 for RepEpilogue { + fn name(&self) -> &'static str { + "ffai-mobiledet-epilogue" + } + + fn cpu_fwd( + &self, + st: &candle_core::CpuStorage, + l: &candle_core::Layout, + ) -> Result<(candle_core::CpuStorage, candle_core::Shape)> { + use crate::par::prelude::*; + let candle_core::CpuStorage::F32(inp) = st else { + candle_core::bail!("ffai-mobiledet-epilogue: f32 only") + }; + let Some((o0, o1)) = l.contiguous_offsets() else { + candle_core::bail!("ffai-mobiledet-epilogue: non-contiguous input") + }; + let inp = &inp[o0..o1]; + let (_, c, h, w) = l.shape().dims4()?; + if !self.bias.is_empty() && self.bias.len() != c { + candle_core::bail!("ffai-mobiledet-epilogue: bias length {} for {c} channels", self.bias.len()) + } + let sixth = (1.0f64 / 6.0) as f32; + let mut out = vec![0f32; inp.len()]; + let plane = (h * w).max(1); + out.par_chunks_mut(plane).zip(inp.par_chunks(plane)).enumerate().for_each( + |(i, (o, x))| { + let bias = self.bias.get(i % c).copied(); + match (bias, self.act) { + (b, Some((s, a))) => { + let (s, a) = (s as f32, a as f32); + for (o, &x) in o.iter_mut().zip(x) { + let v = match b { + Some(b) => x + b, + None => x, + }; + let t = v * 1.0 + 3.0; + let t = if t < 0.0 { 0.0 } else { t }; + let t = if t > 6.0 { 6.0 } else { t }; + let hs = v * (t * sixth + 0.0); + *o = hs * s + a; + } + } + (Some(b), None) => { + for (o, &x) in o.iter_mut().zip(x) { + *o = x + b; + } + } + (None, None) => o.copy_from_slice(x), + } + }, + ); + Ok((candle_core::CpuStorage::F32(out), l.shape().clone())) } } @@ -138,7 +303,7 @@ impl Se { fn forward(&self, x: &Tensor) -> Result { let w = x.mean_keepdim(D::Minus1)?.mean_keepdim(D::Minus2)?; - let w = self.conv2.forward(&self.conv1.forward(&w)?.relu()?)?; + let w = conv_fast(&self.conv2, &conv_fast(&self.conv1, &w)?.relu()?)?; x.broadcast_mul(&hardsigmoid(&w, self.slope)?) } } @@ -167,7 +332,7 @@ impl RseLayer { } fn forward(&self, x: &Tensor) -> Result { - let x = self.conv.forward(x)?; + let x = conv_fast(&self.conv, x)?; &x + self.se.forward(&x)? } } @@ -249,7 +414,7 @@ impl MobileDet { } cur = blk.pw.forward(&cur)?; if let Some(t) = TAPS.iter().position(|&(n, _, _)| n == BLOCKS[i].0) { - feats.push(self.taps[t].forward(&cur)?); + feats.push(conv_fast(&self.taps[t], &cur)?); } } @@ -279,9 +444,9 @@ impl MobileDet { 1, )?; - let y = self.head1.forward(&fused)?.relu()?; - let y = self.head2.forward(&y)?.relu()?; - candle_nn::ops::sigmoid(&self.head3.forward(&y)?) + let y = conv_fast(&self.head1, &fused)?.relu()?; + let y = conv_transpose_fast(&self.head2, &y)?.relu()?; + candle_nn::ops::sigmoid(&conv_transpose_fast(&self.head3, &y)?) } } diff --git a/crates/ffai-carmenta/src/par.rs b/crates/ffai-carmenta/src/par.rs index d54fc3ef..2121a528 100644 --- a/crates/ffai-carmenta/src/par.rs +++ b/crates/ffai-carmenta/src/par.rs @@ -62,6 +62,30 @@ pub mod serial { } } + /// The `par_chunks` of `rayon::prelude`, serially. + pub trait ParallelSliceChunks { + fn par_chunks(&self, n: usize) -> core::slice::Chunks<'_, T>; + } + + impl ParallelSliceChunks for [T] { + #[inline] + fn par_chunks(&self, n: usize) -> core::slice::Chunks<'_, T> { + self.chunks(n) + } + } + + /// The `par_chunks_mut` of `rayon::prelude`, serially. + pub trait ParallelSliceMut { + fn par_chunks_mut(&mut self, n: usize) -> core::slice::ChunksMut<'_, T>; + } + + impl ParallelSliceMut for [T] { + #[inline] + fn par_chunks_mut(&mut self, n: usize) -> core::slice::ChunksMut<'_, T> { + self.chunks_mut(n) + } + } + /// `(0..n).into_par_iter()`, serially. pub trait IntoParallelIterator { type Iter; diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index 09d8b875..f269bdd5 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -136,6 +136,72 @@ fn swish(x: &Tensor) -> Result { x * candle_nn::ops::sigmoid(x)? } +/// `FFAI_SVTR_FUSED=0` sends the backbone through the unfused candle chain +/// and candle's conv for 1x1s, so the fast path stays one variable from its +/// oracle. +fn fused() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("FFAI_SVTR_FUSED").as_deref() != Ok("0")) +} + +/// See [`Svtr::block_epilogue`]. Each parameter is per channel, or one value +/// broadcast to every channel. +struct Epilogue { + bias: Vec, + s1: Vec, + b1: Vec, + s2: Vec, + b2: Vec, +} + +impl candle_core::CustomOp1 for Epilogue { + fn name(&self) -> &'static str { + "ffai-svtr-epilogue" + } + + fn cpu_fwd( + &self, + s: &candle_core::CpuStorage, + l: &candle_core::Layout, + ) -> Result<(candle_core::CpuStorage, candle_core::Shape)> { + use crate::par::prelude::*; + let candle_core::CpuStorage::F32(inp) = s else { + candle_core::bail!("ffai-svtr-epilogue: f32 only") + }; + let Some((o0, o1)) = l.contiguous_offsets() else { + candle_core::bail!("ffai-svtr-epilogue: non-contiguous input") + }; + let inp = &inp[o0..o1]; + let (_, c, h, w) = l.shape().dims4()?; + let ok = |p: &[f32]| p.len() == 1 || p.len() == c; + if !(ok(&self.bias) && ok(&self.s1) && ok(&self.b1) && ok(&self.s2) && ok(&self.b2)) { + candle_core::bail!("ffai-svtr-epilogue: parameter length is neither 1 nor {c}") + } + let at = |p: &[f32], ch: usize| if p.len() == 1 { p[0] } else { p[ch] }; + // Candle's `/ 6.0` is an affine with this multiplier. + let sixth = (1.0f64 / 6.0) as f32; + let mut out = vec![0f32; inp.len()]; + let plane = (h * w).max(1); + out.par_chunks_mut(plane).zip(inp.par_chunks(plane)).enumerate().for_each( + |(i, (o, x))| { + let ch = i % c; + let (bias, s1, b1) = (at(&self.bias, ch), at(&self.s1, ch), at(&self.b1, ch)); + let (s2, b2) = (at(&self.s2, ch), at(&self.b2, ch)); + for (o, &x) in o.iter_mut().zip(x) { + let v = x + bias; + let v = v * s1 + b1; + let t = v * 1.0 + 3.0; + let t = if t < 0.0 { 0.0 } else { t }; + let t = if t > 6.0 { 6.0 } else { t }; + let v = (t * v) * sixth + 0.0; + *o = v * s2 + b2; + } + }, + ); + Ok((candle_core::CpuStorage::F32(out), l.shape().clone())) + } +} + impl Svtr { pub fn new(vb: VarBuilder<'static>, n_class: usize) -> Result { Ok(Self { vb, n_class, cache: RwLock::new(HashMap::new()) }) @@ -185,15 +251,53 @@ impl Svtr { bias: bool, ) -> Result { let w = self.fetch(&format!("{name}.w_0"))?; + let y = if crate::dwconv::applies(x, &w, groups) { + // 14 of the 30 backbone convs. Candle splits a grouped conv into + // one convolution per channel (see `dwconv`), and the kernel takes + // the asymmetric stride directly instead of computing a stride-1 + // output and keeping every other row of it. + crate::dwconv::depthwise(x, &w, stride, pad)? + } else { + self.dense(x, w, stride, pad, groups)? + }; + if bias { + let b = self.fetch(&format!("{name}.b_0"))?; + y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?) + } else { + Ok(y) + } + } + + /// Candle's conv for everything that is not depthwise: the stem and the + /// pointwise and head convs. + fn dense( + &self, + x: &Tensor, + w: Tensor, + stride: (usize, usize), + pad: (usize, usize), + groups: usize, + ) -> Result { let (oc, icg, kh, kw) = w.dims4()?; - let _ = (oc, icg, kh, kw); + if fused() && (kh, kw) == (1, 1) && stride == (1, 1) && pad == (0, 0) && groups == 1 { + // A 1x1 conv in NCHW IS a matmul — `(oc, ic) x (ic, h*w)` — with + // no im2col copy and no layout permute on either side. Candle's + // conv path builds both; 16 of the 30 backbone convs are 1x1. + let (n, _, h, wd) = x.dims4()?; + let y = w.reshape((oc, icg))?.broadcast_matmul(&x.reshape((n, icg, h * wd))?)?; + return y.reshape((n, oc, h, wd)); + } // candle's conv2d takes ONE stride and ONE padding; the recorded model // uses asymmetric values (e.g. (2,1) to collapse height faster than - // width). Emulate by padding manually and striding per axis. + // width). Emulate by padding manually and striding per axis. A + // symmetric stride (the (2,2) stem) is candle's own, so it computes a + // quarter of the outputs rather than all of them. let x = x.pad_with_zeros(2, pad.0, pad.0)?.pad_with_zeros(3, pad.1, pad.1)?; - let cfg = candle_nn::Conv2dConfig { padding: 0, stride: 1, dilation: 1, groups, + let native = if stride.0 == stride.1 { stride.0 } else { 1 }; + let cfg = candle_nn::Conv2dConfig { padding: 0, stride: native, dilation: 1, groups, ..Default::default() }; let y = candle_nn::Conv2d::new(w, None, cfg).forward(&x)?; + let stride = if native > 1 { (1, 1) } else { stride }; let y = if stride.0 > 1 { let n = y.dim(2)?.div_ceil(stride.0); y.index_select( @@ -220,12 +324,7 @@ impl Svtr { } else { y }; - if bias { - let b = self.fetch(&format!("{name}.b_0"))?; - y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?) - } else { - Ok(y) - } + Ok(y) } fn bn(&self, x: &Tensor, name: &str) -> Result { @@ -240,6 +339,28 @@ impl Svtr { .broadcast_add(&shift.reshape((1, c, 1, 1))?) } + /// `lab(hardswish(lab(conv + bias)))` in ONE pass over the activation. + /// + /// The unfused form is eight candle ops — bias add, two affine blocks of + /// two ops each, and hardswish's add, clamp, multiply and divide — each a + /// full read, write and allocation of the block's output. PdfCraft's page + /// profile put those passes at a third of the backbone. The arithmetic + /// below is candle's own, operation for operation (`+ 3.0` is + /// `v * 1 + 3`, `clamp` is maximum-then-minimum, `/ 6.0` is + /// `v * (1/6) + 0`), and Rust never contracts a multiply-add on its own, so + /// the result is bit-identical to the unfused chain. `FFAI_SVTR_FUSED=0` + /// restores that chain for an A/B. + fn block_epilogue(&self, x: &Tensor, conv: &str, i: &mut usize) -> Result { + let v = |t: Tensor| t.flatten_all().and_then(|t| t.to_vec1::()); + let bias = v(self.fetch(&format!("{conv}.b_0"))?)?; + let s1 = v(self.fetch(&format!("learnable_affine_block_{i}.w_0"))?)?; + let b1 = v(self.fetch(&format!("learnable_affine_block_{i}.w_1"))?)?; + let s2 = v(self.fetch(&format!("learnable_affine_block_{}.w_0", *i + 1))?)?; + let b2 = v(self.fetch(&format!("learnable_affine_block_{}.w_1", *i + 1))?)?; + *i += 2; + x.contiguous()?.apply_op1_no_bwd(&Epilogue { bias, s1, b1, s2, b2 }) + } + /// Learnable affine block: per-channel scale then shift, consumed in order. fn lab(&self, x: &Tensor, i: &mut usize) -> Result { let w = self.fetch(&format!("learnable_affine_block_{i}.w_0"))?; @@ -337,15 +458,28 @@ impl Svtr { /// (1, 3, 48, W) normalised to [-1, 1] -> (1, T, n_class) probabilities. pub fn forward(&self, x: &Tensor) -> Result { + // Three stages on the profile's `rec_*` rows, so `FFAI_PROFILE=1` + // says WHERE a recognizer forward goes rather than one opaque number. + let x = crate::profile::timed(|p| &p.rec_cnn, || self.backbone(x))?; + let (x, shortcut) = crate::profile::timed(|p| &p.rec_rnn, || self.encoder(&x))?; + crate::profile::timed(|p| &p.rec_head, || self.head(&x, &shortcut)) + } + + fn backbone(&self, x: &Tensor) -> Result { let mut x = self.bn(&self.conv(x, "conv2d_0", (2, 2), (1, 1), 1, false)?, "batch_norm2d_0")?; let mut lab = 0usize; for st in backbone_spec() { match st { Stage::Blk(Blk(name, groups, stride, pad)) => { - x = self.conv(&x, name, stride, pad, groups, true)?; - x = self.lab(&x, &mut lab)?; - x = hardswish(&x)?; - x = self.lab(&x, &mut lab)?; + if fused() { + x = self.conv(&x, name, stride, pad, groups, false)?; + x = self.block_epilogue(&x, name, &mut lab)?; + } else { + x = self.conv(&x, name, stride, pad, groups, true)?; + x = self.lab(&x, &mut lab)?; + x = hardswish(&x)?; + x = self.lab(&x, &mut lab)?; + } } Stage::Se(Se(reduce, expand)) => { let s = x.mean_keepdim(2)?.mean_keepdim(3)?; @@ -355,6 +489,12 @@ impl Svtr { } } } + Ok(x) + } + + /// Neck pool, the two SVTR blocks and the final norm. Returns the encoder + /// output and the pooled SHORTCUT the head concatenates with it. + fn encoder(&self, x: &Tensor) -> Result<(Tensor, Tensor)> { // Neck pool k=(3,2) s=(3,2): H 3->1, W 80->40 (the timesteps). let (_, _, h, w) = x.dims4()?; let x = x.avg_pool2d_with_stride((3, 2), (3, 2))?; @@ -369,11 +509,15 @@ impl Svtr { let x = self.enc_block(&x, "layer_norm_0", "linear_0", "linear_1", "layer_norm_1", "linear_2", "linear_3")?; let x = self.enc_block(&x, "layer_norm_2", "linear_4", "linear_5", "layer_norm_3", "linear_6", "linear_7")?; let x = self.ln(&x, "layer_norm_4", LN_EPS_FINAL)?; + Ok((x, shortcut)) + } + + fn head(&self, x: &Tensor, shortcut: &Tensor) -> Result { // Head: (B,N,C) -> (B,C,1,N) -> conv133 -> cat(shortcut) -> conv134 -> conv135 let (b, n, c) = x.dims3()?; let hd = x.reshape((b, 1, n, c))?.permute((0, 3, 1, 2))?.contiguous()?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_133", (1, 1), (0, 0), 1, false)?, "batch_norm2d_148")?)?; - let hd = Tensor::cat(&[&shortcut, &hd], 1)?; + let hd = Tensor::cat(&[shortcut, &hd], 1)?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_134", (1, 1), (0, 1), 1, false)?, "batch_norm2d_149")?)?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_135", (1, 1), (0, 0), 1, false)?, "batch_norm2d_150")?)?; let x = hd.squeeze(2)?.transpose(1, 2)?.contiguous()?; From 502c9614069d35fc047dc3aa7f7b1089851b16b4 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 14:20:31 -0700 Subject: [PATCH 02/25] docs(carmenta): record the refuted single-level-parallelism re-measurement at the line fan-out Built and measured after the depthwise work: a thread-local serial flag for inner ops plus direct gemm calls with Parallelism::None inside the fan-out (output bit-identical). Slower: 1.066x at 24 threads (0/6 ABBA wins), 1.040x at 8 (1/6). Serialising the lines is worse (55.6 s vs 40.3 s). Reverted; the comment keeps the numbers so it is not rebuilt. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/engine.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index b6648689..e9b64a25 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -1295,6 +1295,16 @@ impl OcrEngine for CraftCrnn { // z = +1.50 over 16 ABBA-paired rounds, INSIDE a 17-27 % noise // floor. Reverted per revert-if-unproven; §8.100 carries the // numbers so it is not rebuilt. + // + // Re-measured 2026-10-30 after the depthwise/epilogue work removed + // the per-channel conv overhead, with a thread-local "inner ops + // serial" flag and SVTR's matmuls calling gemm with + // `Parallelism::None` inside this fan-out (bit-identical output): + // SLOWER, 1.066x at 24 threads (0/6 ABBA wins) and 1.040x at 8 + // (1/6), on 10 of PdfCraft's pages. Lines are uneven, and the + // inner fan-out is what lets the long ones finish with the rest. + // Serialising the LINES instead is worse still (55.6 s against + // 40.3 s on the 42-page corpus). Both refuted; keep this shape. lines.par_iter().map(&recognize_line).collect::>>()? } else { lines.iter().map(&recognize_line).collect::>>()? From 83496a73e930fecd4f0d4a7def60bd767ba6e4e1 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 14:38:38 -0700 Subject: [PATCH 03/25] feat(carmenta): built-in deskew (OcrOptions::deskew): skewed pages 48.29% -> 0.10% CER PdfCraft measured an axis-aligned crop of a tilted line reading the lines above and below it: 0.41% CER at 0.5 degrees, 93.3% at 2, 98.8% at 5, and 61% on a phone photo tilted 1.5 degrees. Its own deskew pass fixed that; this puts the same fix in the engine, for every document caller. - mobiledet::skew_degrees: the page angle from the detector's own regions -- the min-area rectangles the DB postprocess already fits -- as the length-weighted median direction of the line-shaped ones, reported only when 60% of the weight agrees with it to a degree. Measured in the normal detection pass, so a square page costs nothing extra. - A page at least 0.3 degrees off is rotated back once (bilinear, onto a canvas holding every corner, filled with the page's paper colour), read, and its boxes mapped back onto the caller's image. - ffai-core: OcrOptions::deskew (opt-in, like auto_orient; engines that cannot honour it error) and OcrOutput::skew_degrees. deskew::rotated_corners recovers a line's turned rectangle from its box and the angle, for a text layer drawn along the page's slope. PdfCraft's 42-page corpus: skew 0.5/1/2/5 group 48.29% -> 0.10% (PdfCraft's own pass: 0.14%), photo 61.00% -> 0.11% (0.11%). The other 37 pages read byte-identical text. Co-Authored-By: Claude Opus 5.5 --- .../ffai-carmenta/examples/pdfcraft_bench.rs | 4 +- crates/ffai-carmenta/src/deskew.rs | 208 ++++++++++++++++++ crates/ffai-carmenta/src/engine.rs | 51 ++++- crates/ffai-carmenta/src/forms.rs | 1 + crates/ffai-carmenta/src/lib.rs | 1 + crates/ffai-carmenta/src/live.rs | 5 +- crates/ffai-carmenta/src/mobiledet.rs | 107 ++++++++- crates/ffai-core/src/engine.rs | 10 + crates/ffai-core/src/types.rs | 11 + 9 files changed, 391 insertions(+), 7 deletions(-) create mode 100644 crates/ffai-carmenta/src/deskew.rs diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs index 13bf467d..a7244f5e 100644 --- a/crates/ffai-carmenta/examples/pdfcraft_bench.rs +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -10,7 +10,8 @@ //! //! Usage: `pdfcraft_bench ...` //! Env: `BENCH_REC=crnn` for mobiledet-crnn, `BENCH_ORIENT=1` for -//! `auto_orient`, `BENCH_LANG=en,fr` for `languages`. +//! `auto_orient`, `BENCH_DESKEW=1` for `deskew`, `BENCH_LANG=en,fr` for +//! `languages`. #[global_allocator] static GLOBAL: rusty_alloc_api::RustyAlloc = rusty_alloc_api::RustyAlloc; @@ -51,6 +52,7 @@ fn main() { eprintln!("load {:.1} ms", t.elapsed().as_secs_f64() * 1e3); let opts = OcrOptions { auto_orient: std::env::var("BENCH_ORIENT").is_ok_and(|v| v == "1"), + deskew: std::env::var("BENCH_DESKEW").is_ok_and(|v| v == "1"), languages: std::env::var("BENCH_LANG") .map(|v| v.split(',').map(str::to_string).collect()) .unwrap_or_default(), diff --git a/crates/ffai-carmenta/src/deskew.rs b/crates/ffai-carmenta/src/deskew.rs new file mode 100644 index 00000000..a1004897 --- /dev/null +++ b/crates/ffai-carmenta/src/deskew.rs @@ -0,0 +1,208 @@ +//! Straightening a slightly rotated page (`OcrOptions::deskew`). +//! +//! Detection and recognition both cut axis-aligned boxes. On a page scanned a +//! couple of degrees off square, a long line climbs or falls by more than its +//! own height across the page, so its crop takes in the lines above and below +//! and the recognizer reads noise: PdfCraft measured 0.41 % CER at 0.5 +//! degrees, 0.68 % at 1, 93.3 % at 2 and 98.8 % at 5 on rendered pages, and +//! 61 % on a phone photo tilted 1.5 degrees. +//! +//! The engine measures the skew from the detector's own regions +//! ([`crate::mobiledet::skew_degrees`]), and a page at least [`MIN_DEGREES`] +//! off is rotated back ONCE, read, and its boxes mapped back onto the input. +//! A square page is never resampled, so it reads exactly as before. +//! +//! # Coordinates +//! +//! Boxes stay in the INPUT image's coordinates, as with `auto_orient`. A +//! straightened line is a rectangle; on the input page it is that rectangle +//! turned by the skew, and the reported box is its axis-aligned bounds. +//! `OcrOutput::skew_degrees` carries the angle, and [`rotated_corners`] +//! recovers the turned rectangle from a reported box, which is what a text +//! layer drawn along the page's slope needs. + +use ffai_core::types::{BoundingBox, ImageBuffer}; + +use crate::par::prelude::*; + +/// Below this the page is read as it is. +pub const MIN_DEGREES: f32 = 0.3; + +/// A rotation about the image centres: the point `q` of the straightened image +/// shows the point `centre + R(angle) (q - straight_centre)` of the input, +/// `R` the rotation by `degrees` clockwise on screen (y down). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Skew { + pub degrees: f32, + pub centre: (f32, f32), + pub straight_centre: (f32, f32), +} + +impl Skew { + /// The input-image point that `(x, y)` of the straightened image shows. + #[must_use] + pub fn back(&self, (x, y): (f32, f32)) -> (f32, f32) { + let (s, c) = self.degrees.to_radians().sin_cos(); + let (dx, dy) = (x - self.straight_centre.0, y - self.straight_centre.1); + (self.centre.0 + c * dx - s * dy, self.centre.1 + s * dx + c * dy) + } + + /// A box of the straightened image as the axis-aligned bounds of its + /// turned rectangle on the input, clamped to the input's `w x h`. + #[must_use] + pub fn back_box(&self, b: BoundingBox, w: f32, h: f32) -> BoundingBox { + let corners = [ + (b.x, b.y), + (b.x + b.width, b.y), + (b.x + b.width, b.y + b.height), + (b.x, b.y + b.height), + ] + .map(|p| self.back(p)); + let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN); + for (x, y) in corners { + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x); + y1 = y1.max(y); + } + let (x0, y0) = (x0.clamp(0.0, w), y0.clamp(0.0, h)); + let (x1, y1) = (x1.clamp(0.0, w), y1.clamp(0.0, h)); + BoundingBox { x: x0, y: y0, width: (x1 - x0).max(0.0), height: (y1 - y0).max(0.0) } + } +} + +/// Rotate `img` by `-degrees` about its centre onto a canvas large enough to +/// hold all of it, so text the skew carried past an edge is not cut off. The +/// new corners are filled with the page's own paper colour (the median of its +/// border pixels), not black, which the detector would read as ink. +#[must_use] +pub fn straighten(img: &ImageBuffer, degrees: f32) -> (ImageBuffer, Skew) { + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let (s, c) = degrees.to_radians().sin_cos(); + let nw = (w as f32 * c.abs() + h as f32 * s.abs()).ceil() as usize; + let nh = (w as f32 * s.abs() + h as f32 * c.abs()).ceil() as usize; + let skew = Skew { + degrees, + centre: (w as f32 / 2.0, h as f32 / 2.0), + straight_centre: (nw as f32 / 2.0, nh as f32 / 2.0), + }; + let paper = paper_colour(img); + let mut out = vec![0u8; nw * nh * bpp]; + out.par_chunks_mut(nw * bpp).enumerate().for_each(|(qy, row)| { + for qx in 0..nw { + // Pixel centres: sample the input where this output centre lands. + let (px, py) = skew.back((qx as f32 + 0.5, qy as f32 + 0.5)); + let (sx, sy) = (px - 0.5, py - 0.5); + let dst = &mut row[qx * bpp..(qx + 1) * bpp]; + if sx < -1.0 || sy < -1.0 || sx > w as f32 || sy > h as f32 { + dst.copy_from_slice(&paper[..bpp]); + continue; + } + let (x0, y0) = (sx.floor(), sy.floor()); + let (fx, fy) = (sx - x0, sy - y0); + let tap = |x: f32, y: f32, ch: usize| -> f32 { + if x < 0.0 || y < 0.0 || x >= w as f32 || y >= h as f32 { + f32::from(paper[ch]) + } else { + f32::from(img.data[(y as usize * w + x as usize) * bpp + ch]) + } + }; + for (ch, d) in dst.iter_mut().enumerate() { + let top = tap(x0, y0, ch) * (1.0 - fx) + tap(x0 + 1.0, y0, ch) * fx; + let bot = tap(x0, y0 + 1.0, ch) * (1.0 - fx) + tap(x0 + 1.0, y0 + 1.0, ch) * fx; + *d = (top * (1.0 - fy) + bot * fy).round().clamp(0.0, 255.0) as u8; + } + } + }); + ( + ImageBuffer { + width: u32::try_from(nw).unwrap_or(u32::MAX), + height: u32::try_from(nh).unwrap_or(u32::MAX), + format: img.format, + data: out, + }, + skew, + ) +} + +/// Per-channel median of the image's border pixels (up to 4 channels). +fn paper_colour(img: &ImageBuffer) -> [u8; 4] { + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let mut out = [255u8; 4]; + if w == 0 || h == 0 { + return out; + } + let mut idx: Vec = Vec::with_capacity(2 * (w + h)); + idx.extend((0..w).map(|x| x)); + idx.extend((0..w).map(|x| (h - 1) * w + x)); + idx.extend((0..h).map(|y| y * w)); + idx.extend((0..h).map(|y| y * w + w - 1)); + for (ch, o) in out.iter_mut().enumerate().take(bpp) { + let mut v: Vec = idx.iter().map(|&i| img.data[i * bpp + ch]).collect(); + v.sort_unstable(); + *o = v[v.len() / 2]; + } + out +} + +/// The four corners, on the input page, of the turned rectangle whose +/// axis-aligned bounds are `b`, for a page skewed `degrees` (as +/// `OcrOutput::skew_degrees` reports it). In order: top-left, top-right, +/// bottom-right, bottom-left of the straightened rectangle. +/// +/// A `W x H` rectangle turned by `a` has bounds `w = W|cos a| + H|sin a|` and +/// `h = W|sin a| + H|cos a|`, which this solves for `W` and `H`; the centre is +/// shared. Exact up to the clamping at the page edge. +#[must_use] +pub fn rotated_corners(b: BoundingBox, degrees: f32) -> [(f32, f32); 4] { + let (s, c) = degrees.to_radians().sin_cos(); + let (sa, ca) = (s.abs(), c.abs()); + let det = ca * ca - sa * sa; + let (cx, cy) = (b.x + b.width / 2.0, b.y + b.height / 2.0); + let (rw, rh) = if det.abs() < 1e-3 { + (b.width, b.height) + } else { + ( + ((b.width * ca - b.height * sa) / det).max(0.0), + ((b.height * ca - b.width * sa) / det).max(0.0), + ) + }; + [(-rw / 2.0, -rh / 2.0), (rw / 2.0, -rh / 2.0), (rw / 2.0, rh / 2.0), (-rw / 2.0, rh / 2.0)] + .map(|(dx, dy)| (cx + c * dx - s * dy, cy + s * dx + c * dy)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ffai_core::types::PixelFormat; + + #[test] + fn back_box_and_rotated_corners_invert() { + let img = ImageBuffer { width: 2000, height: 3000, format: PixelFormat::Gray8, data: vec![255; 6_000_000] }; + let (straight, skew) = straighten(&img, 2.0); + assert!(straight.width > 2000 && straight.height > 3000); + // A line on the straightened page, mapped back and recovered. + let line = BoundingBox { x: 400.0, y: 900.0, width: 1200.0, height: 40.0 }; + let on_input = skew.back_box(line, 2000.0, 3000.0); + let corners = rotated_corners(on_input, 2.0); + let want = [ + (line.x, line.y), + (line.x + line.width, line.y), + (line.x + line.width, line.y + line.height), + (line.x, line.y + line.height), + ] + .map(|p| skew.back(p)); + for (g, w) in corners.iter().zip(&want) { + assert!((g.0 - w.0).abs() < 0.05 && (g.1 - w.1).abs() < 0.05, "{g:?} vs {w:?}"); + } + } + + #[test] + fn straighten_keeps_the_paper_colour() { + let img = ImageBuffer { width: 64, height: 32, format: PixelFormat::Rgb8, data: vec![200; 64 * 32 * 3] }; + let (s, _) = straighten(&img, 5.0); + assert!(s.data.iter().all(|&v| v == 200), "corners must be filled with paper, not black"); + } +} diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index e9b64a25..ec7b6d30 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -186,6 +186,32 @@ impl CraftCrnn { Self::with_source(rec, det, Source::Offline(weights_root.to_path_buf())) } + /// Read `img` rotated back by `degrees` (see `crate::deskew`) and map the + /// result onto `img`'s own coordinates. + fn recognize_straightened(&self, img: &ImageBuffer, opts: &OcrOptions, degrees: f32) -> Result { + let (straight, sk) = crate::deskew::straighten(img, degrees); + let inner = OcrOptions { deskew: false, ..opts.clone() }; + let mut out = self.recognize(&straight, &inner)?; + let (w, h) = (img.width as f32, img.height as f32); + for block in &mut out.blocks { + if let Some(b) = block.bbox { + block.bbox = Some(sk.back_box(b, w, h)); + } + for line in &mut block.lines { + if let Some(b) = line.bbox { + line.bbox = Some(sk.back_box(b, w, h)); + } + for word in &mut line.words { + if let Some(b) = word.bbox { + word.bbox = Some(sk.back_box(b, w, h)); + } + } + } + } + out.skew_degrees = Some(degrees); + Ok(out) + } + /// Load every model now, returning the error instead of deferring it to /// the first `recognize`. For a batch job, a missing weight file should /// stop the job before page one, not after it has started writing output. @@ -774,7 +800,17 @@ impl OcrEngine for CraftCrnn { } return Ok(out); } + if opts.deskew && self.det != DetStage::MobileDet { + return Err(Error::Other( + "deskew is measured from the mobile detector's text regions; use a mobiledet-* \ + engine" + .into(), + )); + } let m = self.models()?; + // The caller's image, before rule removal: a page found skewed is + // straightened from THIS and read again from the top. + let caller_img = img; // Form rules come out BEFORE detection as well as recognition: a rule // joins the boxes DBNet draws, and the crop the recognizer reads is cut @@ -863,7 +899,7 @@ impl OcrEngine for CraftCrnn { confidence, }); } - return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }] }); + return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], skew_degrees: None }); } // ---- detect ---- @@ -872,6 +908,8 @@ impl OcrEngine for CraftCrnn { // `scale` is carried so the crop stage can map detector coordinates // back to image pixels via `to_img` below; mobile-det already emits // image coordinates, which is why its scale is the identity 2.0. + // Set by the mobile-det arm when `opts.deskew` asks for it. + let mut skew: Option = None; let (lines, region, affinity, mw, scale) = match self.det { DetStage::Craft => { // Detection sees COLOR (chroma contrast is real signal on @@ -937,6 +975,9 @@ impl OcrEngine for CraftCrnn { } let _ = std::fs::write(format!("{dir}/{tag}.prob"), out); } + if opts.deskew { + skew = crate::mobiledet::skew_degrees(&flat, pw, ph, sx, sy); + } let mut b = crate::mobiledet::boxes_from_probability( &flat, pw, ph, sx, sy, unclip, ); @@ -1071,6 +1112,12 @@ impl OcrEngine for CraftCrnn { // pool contention with candle's internal rayon: measured, a 1-line // band strip pays 177 ms/line under par_iter vs 82 ms serial, while // a 7-line frame wins 2.07s -> 1.65s. Threshold 3 splits the cases. + // A page found skewed is straightened and read again from the top, + // its boxes mapped back onto the caller's image. Only skewed pages pay + // the second detection pass; a square one carries on with these boxes. + if let Some(deg) = skew.filter(|d| d.abs() >= crate::deskew::MIN_DEGREES) { + return self.recognize_straightened(caller_img, opts, deg); + } let recognize_line = |line: &Vec| -> Result> { let lb = boxes::line_bbox(line); let line_h = to_img(lb.y1) - to_img(lb.y0); @@ -1382,7 +1429,7 @@ impl OcrEngine for CraftCrnn { // v1: one block per page — paragraph segmentation is the DOCUMENT // milestone's work, and inventing it early would be unearned. - Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }] }) + Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }], skew_degrees: None }) } } diff --git a/crates/ffai-carmenta/src/forms.rs b/crates/ffai-carmenta/src/forms.rs index 72843946..5649ad54 100644 --- a/crates/ffai-carmenta/src/forms.rs +++ b/crates/ffai-carmenta/src/forms.rs @@ -153,6 +153,7 @@ mod tests { fn page(lines: Vec) -> OcrOutput { OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], + skew_degrees: None, } } diff --git a/crates/ffai-carmenta/src/lib.rs b/crates/ffai-carmenta/src/lib.rs index b8e0c536..e15a7b22 100644 --- a/crates/ffai-carmenta/src/lib.rs +++ b/crates/ffai-carmenta/src/lib.rs @@ -45,6 +45,7 @@ pub mod rules; pub mod harvest; pub mod image; pub mod conv3x3; +pub mod deskew; pub mod dwconv; /// The LIVE frame reader — **native only.** /// diff --git a/crates/ffai-carmenta/src/live.rs b/crates/ffai-carmenta/src/live.rs index 1e0f6c11..2670af28 100644 --- a/crates/ffai-carmenta/src/live.rs +++ b/crates/ffai-carmenta/src/live.rs @@ -434,7 +434,10 @@ impl LiveSession { for (k, blocks) in fresh { self.bands[k].cached = blocks; } - OcrOutput { blocks: self.bands.iter().flat_map(|b| b.cached.clone()).collect() } + OcrOutput { + blocks: self.bands.iter().flat_map(|b| b.cached.clone()).collect(), + skew_degrees: None, + } } else { let full = self.engine.recognize(img, &self.opts)?; self.bands = calibrate_bands(&full, img.height as usize); diff --git a/crates/ffai-carmenta/src/mobiledet.rs b/crates/ffai-carmenta/src/mobiledet.rs index 7d8c320d..3587b2df 100644 --- a/crates/ffai-carmenta/src/mobiledet.rs +++ b/crates/ffai-carmenta/src/mobiledet.rs @@ -680,14 +680,115 @@ fn box_score_fast(prob: &[f32], w: usize, h: usize, quad: &[(f32, f32); 4]) -> f } } +/// The page's text skew in degrees, read from the probability map's own text +/// regions: positive when lines fall to the right (clockwise on screen, y +/// down). `None` when the map holds too little line-shaped text to say. +/// +/// Each 8-connected region above the binarisation threshold gets the same +/// minimum-area rectangle [`boxes_from_probability`] fits. A region at least +/// [`SKEW_MIN_ASPECT`] times longer than it is tall is a text line (or a long +/// word), and its long axis is the direction the text runs; the estimate is +/// the median of those directions weighted by length, so a page's long lines +/// outvote its stray marks. It is reported only when [`SKEW_AGREE`] of that +/// weight sits within a degree of the median: a page whose regions disagree +/// (a collage, a rotated table beside upright prose) is left as it is. +/// +/// This is the measurement §8.8 did not need: on its corpora deskew moved CER +/// by 0.1 pp. PdfCraft's rendered pages measured 93.3 % CER at 2 degrees and +/// 98.8 % at 5, because an axis-aligned crop of a long tilted line takes in +/// the lines above and below it. +#[must_use] +/// +/// `sx` and `sy` are the map's scale against the image per axis (the same +/// pair [`boxes_from_probability`] takes): the detector input is rounded to +/// multiples of 32, so the two can differ slightly, and an angle is measured +/// in IMAGE pixels. +pub fn skew_degrees(prob: &[f32], w: usize, h: usize, sx: f32, sy: f32) -> Option { + let bin_thr = env_f32("FFAI_DB_BIN", BIN_THRESHOLD); + let mut mask: Vec = prob.iter().map(|&p| p > bin_thr).collect(); + let mut stack = Vec::new(); + let mut pts: Vec<(f32, f32)> = Vec::new(); + // (angle in degrees, weight = long side in map pixels) + let mut votes: Vec<(f32, f32)> = Vec::new(); + for start in 0..w * h { + if !mask[start] { + continue; + } + pts.clear(); + stack.push(start); + mask[start] = false; + while let Some(i) = stack.pop() { + let (x, y) = (i % w, i / w); + pts.push((x as f32, y as f32)); + for dy in -1i32..=1 { + for dx in -1i32..=1 { + let (nx, ny) = (x as i32 + dx, y as i32 + dy); + if nx < 0 || ny < 0 || nx >= w as i32 || ny >= h as i32 { + continue; + } + let j = ny as usize * w + nx as usize; + if mask[j] { + mask[j] = false; + stack.push(j); + } + } + } + } + let Some(rect) = min_area_rect(&convex_hull(pts.clone())) else { continue }; + let (rw, rh) = rect.dims(); + let (long, short) = (rw.max(rh), rw.min(rh)); + if short < MIN_SIDE || long < SKEW_MIN_ASPECT * short { + continue; + } + // Direction of the long side, folded into (-90, 90] degrees. + let (ux, uy) = rect.u; + let (dx, dy) = if rw >= rh { (ux, uy) } else { (-uy, ux) }; + let (dx, dy) = (dx / sx, dy / sy); + let mut a = dy.atan2(dx).to_degrees(); + if a > 90.0 { + a -= 180.0; + } else if a <= -90.0 { + a += 180.0; + } + // Text running vertically is the orientation stage's business. + if a.abs() <= SKEW_MAX_DEGREES { + votes.push((a, long)); + } + } + let total: f32 = votes.iter().map(|v| v.1).sum(); + if votes.len() < 3 || total <= 0.0 { + return None; + } + votes.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut acc = 0.0; + let median = votes + .iter() + .find(|v| { + acc += v.1; + acc >= total / 2.0 + }) + .map_or(0.0, |v| v.0); + let agree: f32 = votes.iter().filter(|v| (v.0 - median).abs() <= 1.0).map(|v| v.1).sum(); + (agree >= SKEW_AGREE * total).then_some(median) +} + +/// A region this many times longer than tall votes on the skew. +const SKEW_MIN_ASPECT: f32 = 4.0; +/// Share of the vote weight that must agree with the median to within a degree. +const SKEW_AGREE: f32 = 0.6; +/// Widest skew [`skew_degrees`] looks for, either way. +pub const SKEW_MAX_DEGREES: f32 = 20.0; + /// Probability map -> word boxes in SOURCE image coordinates. /// /// This is `PaddleOCR`'s `DBPostProcess`, with one deliberate deviation recorded /// rather than hidden: the reference emits rotated quadrilaterals and we emit /// their axis-aligned bounds, because everything downstream — line grouping and -/// crop extraction — is axis-aligned. §8.8 measured deskew at 0.1 pp on these -/// corpora, so the rotation is not currently worth the cascade. The rotated -/// rect is still computed, because its area and perimeter drive the unclip. +/// crop extraction — is axis-aligned. §8.8 measured deskew at 0.1 pp on its +/// corpora; a skewed page is now straightened BEFORE this runs instead (see +/// [`skew_degrees`] and `OcrOptions::deskew`), so the boxes stay axis-aligned. +/// The rotated rect is still computed, because its area and perimeter drive +/// the unclip. /// /// The unclip is not optional. DB is trained against *shrunk* polygons, so a /// raw thresholded region is systematically tighter than the text it covers — diff --git a/crates/ffai-core/src/engine.rs b/crates/ffai-core/src/engine.rs index b7a69d29..e1478990 100644 --- a/crates/ffai-core/src/engine.rs +++ b/crates/ffai-core/src/engine.rs @@ -273,6 +273,16 @@ pub struct OcrOptions { /// /// Engines that do not implement it must return an error when it is set. pub auto_orient: bool, + /// Measure how far the page's text runs off horizontal and, when it is + /// skewed, straighten the page before reading it. A scan a couple of + /// degrees off square otherwise reads as noise: an axis-aligned crop of a + /// long tilted line takes in the lines above and below it. A square page + /// is read exactly as without it. Boxes stay in the INPUT image's + /// coordinates; `OcrOutput::skew_degrees` reports the angle when the page + /// was straightened. + /// + /// Engines that do not implement it must return an error when it is set. + pub deskew: bool, } /// Detection options (Diana). diff --git a/crates/ffai-core/src/types.rs b/crates/ffai-core/src/types.rs index 60fbb8f4..7fd33f89 100644 --- a/crates/ffai-core/src/types.rs +++ b/crates/ffai-core/src/types.rs @@ -355,6 +355,16 @@ pub struct OcrBlock { #[derive(Debug, Clone, Default, PartialEq)] pub struct OcrOutput { pub blocks: Vec, + /// The angle the page's text ran at, in degrees clockwise on screen + /// (y down), when the engine straightened it before reading + /// (`OcrOptions::deskew`); `None` when the page was read as given. + /// + /// Boxes are always in the INPUT image's coordinates. On a straightened + /// page each line and word box is the axis-aligned bounds of its + /// rectangle turned by this angle, so the rectangle itself — what a text + /// layer drawn along the page's slope wants — is recoverable from the box + /// and the angle (`ffai_carmenta::deskew::rotated_corners`). + pub skew_degrees: Option, } impl OcrOutput { @@ -388,6 +398,7 @@ impl OcrOutput { .collect(), bbox: None, }], + skew_degrees: None, } } From e22c28fe00fdae016753e533ec1a002fc7268d5f Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 14:45:17 -0700 Subject: [PATCH 04/25] chore(deps): drop carmenta's unused candle-transformers and tokenizers; ffai-models off dirs PdfCraft (artcraft-15) measured an OCR-only consumer going from 67 to 185 crates and from 172 s to 312 s for a clean build, and cargo-deny red under a permissive-only licence policy. - ffai-carmenta listed candle-transformers and tokenizers (native: with Oniguruma, a C library; wasm: with unstable_wasm and getrandom for it). Nothing in the crate used either. Removed; native, all-targets and wasm32 builds check. candle-core 0.11 still pulls tokenizers + onig on native targets itself; that needs candle to make it optional. - ffai-models used `dirs` for one call, the cache directory. Its dirs-sys pulls option-ext (MPL-2.0). Replaced with the same platform rules written against the environment. With `fetch` off (the offline path) option-ext is gone from the tree; with `fetch` on, hf-hub's xet stack still brings dirs. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 18 +----------------- Cargo.toml | 1 - crates/ffai-carmenta/Cargo.toml | 18 +++++++----------- crates/ffai-models/Cargo.toml | 1 - crates/ffai-models/src/lib.rs | 23 ++++++++++++++++++++++- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bdc333b..70c0047a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,7 +454,7 @@ dependencies = [ "byteorder", "candle-core", "candle-nn", - "fancy-regex 0.18.0", + "fancy-regex", "num-traits", "rand 0.9.5", "rayon", @@ -1181,17 +1181,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -[[package]] -name = "fancy-regex" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fancy-regex" version = "0.18.0" @@ -1278,19 +1267,16 @@ version = "0.12.0" dependencies = [ "candle-core", "candle-nn", - "candle-transformers", "ffai-bench", "ffai-core", "ffai-media", "ffai-models", - "getrandom 0.3.4", "mimalloc", "png", "rayon", "rusty_alloc-api", "serde", "serde_json", - "tokenizers 0.23.2", ] [[package]] @@ -1425,7 +1411,6 @@ dependencies = [ name = "ffai-models" version = "0.6.6" dependencies = [ - "dirs", "ffai-core", "hf-hub", "serde", @@ -4720,7 +4705,6 @@ dependencies = [ "dary_heap", "derive_builder", "esaxx-rs", - "fancy-regex 0.17.0", "getrandom 0.3.4", "itertools", "log", diff --git a/Cargo.toml b/Cargo.toml index d63caa29..93d43aff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,6 @@ toml = "0.8" anyhow = "1" clap = { version = "4", features = ["derive"] } hound = "3" -dirs = "6" sha2 = "0.10" serde_json = "1" regex = "1" diff --git a/crates/ffai-carmenta/Cargo.toml b/crates/ffai-carmenta/Cargo.toml index f7aad95e..63aa2895 100644 --- a/crates/ffai-carmenta/Cargo.toml +++ b/crates/ffai-carmenta/Cargo.toml @@ -53,7 +53,6 @@ ffai-core = { workspace = true } ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep candle-core = { workspace = true } candle-nn = { workspace = true } -candle-transformers = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } png = { workspace = true } @@ -64,17 +63,14 @@ png = { workspace = true } # panicking in someone's browser. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rayon = { workspace = true } -tokenizers = { workspace = true } -[target.'cfg(target_arch = "wasm32")'.dependencies] -# getrandom 0.3 needs its `wasm_js` FEATURE in addition to the -# `--cfg getrandom_backend` that .cargo/config.toml supplies; it errors if -# either half is missing. Arrives here through tokenizers -> ahash. -getrandom = { version = "0.3", features = ["wasm_js"] } -# `onig` is Oniguruma, a C regex engine: there is no `cc` for wasm32, and a C -# dependency is not something this house ships anyway. `unstable_wasm` swaps it -# for fancy-regex, which is pure Rust. -tokenizers = { version = "0.23", default-features = false, features = ["unstable_wasm"] } +# `candle-transformers` and `tokenizers` used to be listed here and nothing in +# this crate used either (PdfCraft, 2026-10-30): an OCR-only consumer compiled +# all of candle-transformers, and `tokenizers` with Oniguruma, a C library, for +# nothing. candle-core 0.11 still pulls `tokenizers` + onig on native targets on +# its own (its `quantized::tokenizer`); in this workspace the root +# `[patch.crates-io]` points that at the onig-free fork, a crates.io consumer +# gets onig until candle makes it optional. [dev-dependencies] # Examples only, never the library. `ffai-cli` sets mimalloc as its global diff --git a/crates/ffai-models/Cargo.toml b/crates/ffai-models/Cargo.toml index e9bb220f..166bb27e 100644 --- a/crates/ffai-models/Cargo.toml +++ b/crates/ffai-models/Cargo.toml @@ -20,7 +20,6 @@ categories = ["caching", "filesystem"] ffai-core = { workspace = true } serde = { workspace = true } toml = { workspace = true } -dirs = { workspace = true } # Optional: the Hugging Face downloader. # # It is 226 crates — reqwest, hyper, rustls, aws-lc-sys — and a consumer that diff --git a/crates/ffai-models/src/lib.rs b/crates/ffai-models/src/lib.rs index bb137784..ff4e994d 100644 --- a/crates/ffai-models/src/lib.rs +++ b/crates/ffai-models/src/lib.rs @@ -299,11 +299,32 @@ pub fn cache_dir() -> PathBuf { if let Ok(dir) = std::env::var("FFAI_CACHE") { return PathBuf::from(dir); } - dirs::cache_dir() + os_cache_dir() .unwrap_or_else(std::env::temp_dir) .join("ffai") } +/// The platform's per-user cache directory, by the rules `dirs::cache_dir` +/// applies: `%LOCALAPPDATA%` on Windows, `~/Library/Caches` on macOS, +/// `$XDG_CACHE_HOME` (when absolute) or `~/.cache` elsewhere. +/// +/// Written out rather than taken from `dirs`, whose `dirs-sys` pulls +/// `option-ext`, an MPL-2.0 crate: one call site was enough to turn a +/// permissive-only `cargo deny` policy red for every consumer (PdfCraft, +/// 2026-10-30). +fn os_cache_dir() -> Option { + let abs = |v: std::ffi::OsString| Some(PathBuf::from(v)).filter(|p| p.is_absolute()); + if cfg!(windows) { + std::env::var_os("LOCALAPPDATA").and_then(abs) + } else if cfg!(target_os = "macos") { + std::env::var_os("HOME").and_then(abs).map(|h| h.join("Library/Caches")) + } else { + std::env::var_os("XDG_CACHE_HOME") + .and_then(abs) + .or_else(|| std::env::var_os("HOME").and_then(abs).map(|h| h.join(".cache"))) + } +} + #[cfg(test)] mod tests { use super::*; From 72a81a246093a7a5953daeb5a103ffcdb558d629 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 14:54:59 -0700 Subject: [PATCH 05/25] feat(carmenta): word boxes for mobiledet-svtr, from the CTC frames and the ink OcrLine::words was always empty for the line recognizers, so a caller drawing a word-level text layer re-split each line itself. PdfCraft's attempt with split_ink_words matched the word count on 38 of 472 lines; its own splitter boxed 92% of words. The recognizer already knows. ctc_greedy_emits records the frames of every emission (text and confidence byte-identical to ctc_greedy / _masked: the same per-frame choice, the same loop), the decoded spaces group characters into words, and words.rs places them: - the cut between two words is the widest ink-free column run between the last frame of one and the first frame of the next; - each word's box is its ink inside its cuts: the gap-bridged band of ink rows carrying the most ink (not the densest row: a DB line box reaches into the neighbouring lines, ~2.5x the ink height), then its first and last ink columns. Ink is measured against the crop's own paper level, so inverted and low-contrast lines work. Word texts are the line text split at its spaces (with the same fold), so they join back into it. Word boxes follow auto_orient and deskew back into the caller's coordinates; the formula router clears them when it rewrites a line's text. PdfCraft's corpus, deskew on: word boxes (same text, IoU >= 0.5 against ground-truth ink boxes) 1% -> 96% overall; clean 100%, fonts 100% (PdfCraft 79%), money 99% (88%), small 98% (92%), table 100%, accents 83%. CER unchanged on every page group. 574 of 575 lines carry words. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/engine.rs | 21 ++- crates/ffai-carmenta/src/lib.rs | 1 + crates/ffai-carmenta/src/route.rs | 6 + crates/ffai-carmenta/src/svtr.rs | 64 +++++++ crates/ffai-carmenta/src/words.rs | 260 +++++++++++++++++++++++++++++ 5 files changed, 344 insertions(+), 8 deletions(-) create mode 100644 crates/ffai-carmenta/src/words.rs diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index ec7b6d30..b536d0dd 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -1108,16 +1108,16 @@ impl OcrEngine for CraftCrnn { // components, so the photographic per-word path has nothing to iterate; // the ink-gap splitter supplies word boundaries inside each line. let det_is_line_level = self.det == DetStage::MobileDet; - // Parallel across lines ONLY when there are enough to amortize the - // pool contention with candle's internal rayon: measured, a 1-line - // band strip pays 177 ms/line under par_iter vs 82 ms serial, while - // a 7-line frame wins 2.07s -> 1.65s. Threshold 3 splits the cases. // A page found skewed is straightened and read again from the top, // its boxes mapped back onto the caller's image. Only skewed pages pay // the second detection pass; a square one carries on with these boxes. if let Some(deg) = skew.filter(|d| d.abs() >= crate::deskew::MIN_DEGREES) { return self.recognize_straightened(caller_img, opts, deg); } + // Parallel across lines ONLY when there are enough to amortize the + // pool contention with candle's internal rayon: measured, a 1-line + // band strip pays 177 ms/line under par_iter vs 82 ms serial, while + // a 7-line frame wins 2.07s -> 1.65s. Threshold 3 splits the cases. let recognize_line = |line: &Vec| -> Result> { let lb = boxes::line_bbox(line); let line_h = to_img(lb.y1) - to_img(lb.y0); @@ -1141,6 +1141,8 @@ impl OcrEngine for CraftCrnn { let x1 = (to_img(lb.x1) + px) as usize; let y1 = (to_img(lb.y1) + py) as usize; + // Word geometry, where the recognizer can give it (SVTR). + let mut words: Vec = Vec::new(); let (text, confidence) = match self.rec { RecStage::Svtr => { let (svtr, charset) = @@ -1153,14 +1155,16 @@ impl OcrEngine for CraftCrnn { }; let probs = crate::profile::timed(|p| &p.rec_fwd, || svtr.forward(&crop)) .map_err(image::candle_err)?; - let (mut text, conf) = crate::profile::timed(|p| &p.decode, || match &mask { - Some(mk) => crate::svtr::ctc_greedy_masked(&probs, charset, mk), - None => crate::svtr::ctc_greedy(&probs, charset), + let (mut text, conf, emits, steps) = crate::profile::timed(|p| &p.decode, || { + crate::svtr::ctc_greedy_emits(&probs, charset, mask.as_deref()) }) .map_err(image::candle_err)?; // SVTR's multilingual charset can put a full-width // `0` or `,` into a Latin line; see `fold`. crate::fold::apply(&mut text); + // The crop exactly as `svtr_input` clamped it. + let crop = (x0, y0, x1.min(w), y1.min(h)); + words = crate::words::boxes(&emits, steps, &gray, w, crop, crate::fold::apply); (text, conf) } RecStage::Crnn => { @@ -1314,7 +1318,8 @@ impl OcrEngine for CraftCrnn { } Ok(Some(OcrLine { text, - words: Vec::new(), // line-level recognition; word detail is open work + // Empty for CRNN and the PARSeq path; see `crate::words`. + words, bbox: Some(BoundingBox { x: x0 as f32, y: y0 as f32, diff --git a/crates/ffai-carmenta/src/lib.rs b/crates/ffai-carmenta/src/lib.rs index e15a7b22..daa36549 100644 --- a/crates/ffai-carmenta/src/lib.rs +++ b/crates/ffai-carmenta/src/lib.rs @@ -63,6 +63,7 @@ pub mod live; pub mod par; pub mod suppress; pub mod svtr; +pub mod words; pub mod mobiledet; pub mod onnx_graph; pub mod doclayout; diff --git a/crates/ffai-carmenta/src/route.rs b/crates/ffai-carmenta/src/route.rs index 620d99af..3c6588e8 100644 --- a/crates/ffai-carmenta/src/route.rs +++ b/crates/ffai-carmenta/src/route.rs @@ -451,6 +451,9 @@ pub fn apply( ); } out[hi].text = t; + // The splice changes the text, so the decoder's word boxes no + // longer describe it. + out[hi].words.clear(); spliced.push(g as *const Region); } } @@ -609,6 +612,9 @@ pub fn apply( ); } out[hi].text = t; + // The splice changes the text, so the decoder's word boxes no + // longer describe it. + out[hi].words.clear(); } if dbg { eprintln!( diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index f269bdd5..e9dfc1bc 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -708,6 +708,70 @@ pub fn ctc_greedy_masked( Ok((out, crate::crnn::line_confidence(&kept))) } +/// [`ctc_greedy`] (or [`ctc_greedy_masked`] when `mask` is given) that also +/// returns every emission with the frames it spans, for word geometry. +/// +/// The per-frame choice is made exactly as in those two — the tensor argmax +/// and max unmasked, the same masked fold otherwise — and the string and +/// confidence are built by the same loop, so both are byte-identical to theirs. +/// Returns `(text, confidence, emissions, timesteps)`. +pub fn ctc_greedy_emits( + probs: &Tensor, + charset: &[String], + mask: Option<&[bool]>, +) -> Result<(String, Option, Vec, usize)> { + let (_, t, n) = probs.dims3()?; + let (best, conf): (Vec, Vec) = match mask { + None => { + let best = probs.argmax(D::Minus1)?.flatten_all()?.to_vec1::()?; + let conf = probs.max(D::Minus1)?.flatten_all()?.to_vec1::()?; + (best.into_iter().map(|k| k as usize).collect(), conf) + } + Some(mask) => { + if mask.len() != n || !mask[0] { + return Err(candle_core::Error::Msg(format!( + "class mask has {} entries; the head has {n} classes and the blank must be allowed", + mask.len() + ))); + } + let flat = probs.flatten_all()?.to_vec1::()?; + (0..t) + .map(|i| { + flat[i * n..(i + 1) * n] + .iter() + .enumerate() + .filter(|(j, _)| mask[*j]) + .fold((0usize, f32::NEG_INFINITY), |b, (j, &p)| if p > b.1 { (j, p) } else { b }) + }) + .unzip() + } + }; + let mut out = String::new(); + let mut kept = Vec::new(); + let mut emits: Vec = Vec::new(); + let mut prev = usize::MAX; + // Whether the run `prev` belongs to was pushed as an emission. + let mut open = false; + for i in 0..t { + let k = best[i]; + if k != 0 && k != prev { + open = false; + if let Some(s) = charset.get(k - 1) { + out.push_str(s); + emits.push(crate::words::Emit { text: s.clone(), t0: i, t1: i, p: conf[i] }); + open = true; + } + kept.push(conf[i]); + } else if k != 0 && open { + if let Some(e) = emits.last_mut() { + e.t1 = i; + } + } + prev = k; + } + Ok((out, crate::crnn::line_confidence(&kept), emits, t)) +} + /// One emitted run of characters, with WHERE ON THE LINE it came from. #[derive(Debug, Clone)] pub struct CharSpan { diff --git a/crates/ffai-carmenta/src/words.rs b/crates/ffai-carmenta/src/words.rs new file mode 100644 index 00000000..4482252d --- /dev/null +++ b/crates/ffai-carmenta/src/words.rs @@ -0,0 +1,260 @@ +//! Word boxes for a line-level recognizer, from where its decoder emitted. +//! +//! The document engines recognise whole LINES, so `OcrLine::words` used to be +//! empty and a caller drawing a word-level text layer had to re-split the line +//! itself. PdfCraft's attempt with [`crate::image::split_ink_words`] matched +//! the word count on 38 of 472 lines: a fixed gap rule cannot tell a word space +//! from the space inside "r n" at one size and "W A" at another. +//! +//! The recognizer already knows. CTC walks the crop left to right one +//! timestep per column band, so each emitted character comes with the frames +//! that emitted it, and the decoded SPACES say which characters form a word. +//! That fixes how many words there are and roughly where each one sits; the +//! pixels then say exactly where its ink is: +//! +//! 1. between two neighbouring words, the cut is the widest ink-free column +//! run between the last frame of one and the first frame of the next (the +//! midpoint when there is none); +//! 2. inside its cuts, a word's box is snapped to its ink — the first and last +//! ink columns, and the band of ink rows carrying the most ink, which keeps +//! the neighbouring lines that a DB line box (unclipped to ~2.5x the ink +//! height) reaches into out of it. +//! +//! The words' texts are the line's text split at its spaces, so joining them +//! with single spaces gives the line back. + +use ffai_core::types::{BoundingBox, OcrWord}; + +/// One decoded character run and the CTC frames (inclusive) that emitted it. +#[derive(Debug, Clone, PartialEq)] +pub struct Emit { + pub text: String, + pub t0: usize, + pub t1: usize, + /// Probability of the emitting class at its first frame. + pub p: f32, +} + +/// Ink mask over a crop of the grayscale image: a pixel is ink when it is +/// further from the crop's median (its paper) than 40 % of the crop's 98th +/// percentile deviation, and at least 10 grey levels — so light-on-dark and +/// grey-on-grey lines work as well as black on white. +struct Ink { + x0: usize, + y0: usize, + w: usize, + h: usize, + on: Vec, +} + +impl Ink { + fn new(gray: &[f32], img_w: usize, (x0, y0, x1, y1): (usize, usize, usize, usize)) -> Option { + let (w, h) = (x1.checked_sub(x0)?, y1.checked_sub(y0)?); + if w == 0 || h == 0 { + return None; + } + let px = |x: usize, y: usize| gray[(y0 + y) * img_w + x0 + x]; + let mut sample: Vec = (0..h).flat_map(|y| (0..w).map(move |x| (x, y))).step_by(7).map(|(x, y)| px(x, y)).collect(); + sample.sort_by(f32::total_cmp); + let paper = sample[sample.len() / 2]; + let mut dev: Vec = sample.iter().map(|v| (v - paper).abs()).collect(); + dev.sort_by(f32::total_cmp); + let p98 = dev[(dev.len() * 98 / 100).min(dev.len() - 1)]; + let thr = (0.4 * p98).max(10.0); + let on = (0..h).flat_map(|y| (0..w).map(move |x| (x, y))).map(|(x, y)| (px(x, y) - paper).abs() > thr).collect(); + Some(Self { x0, y0, w, h, on }) + } + + fn at(&self, x: usize, y: usize) -> bool { + self.on[y * self.w + x] + } + + /// Whether column `x` (crop coordinates) has ink in rows `ys`. + fn col(&self, x: usize, ys: std::ops::Range) -> bool { + ys.into_iter().any(|y| self.at(x, y)) + } +} + +/// Split `emits` (one line's decode) into words and box each one on the page. +/// +/// `crop` is the rectangle the recognizer read, in image pixels, `t` its +/// number of timesteps, and `fold` the same per-string normalisation applied +/// to the line's text, so word texts agree with it. +#[must_use] +pub fn boxes( + emits: &[Emit], + t: usize, + gray: &[f32], + img_w: usize, + crop: (usize, usize, usize, usize), + fold: impl Fn(&mut String), +) -> Vec { + // Group emissions into words at the spaces. + let mut groups: Vec<&[Emit]> = Vec::new(); + let mut start = None; + for (i, e) in emits.iter().enumerate() { + let space = e.text.chars().all(char::is_whitespace); + match (space, start) { + (true, Some(s)) => { + groups.push(&emits[s..i]); + start = None; + } + (false, None) => start = Some(i), + _ => {} + } + } + if let Some(s) = start { + groups.push(&emits[s..]); + } + if groups.is_empty() || t == 0 { + return Vec::new(); + } + let (cx0, cy0, cx1, cy1) = crop; + let cw = (cx1 - cx0) as f32; + // Frame edges in crop columns. + let left = |f: usize| ((f as f32 / t as f32) * cw) as usize; + let right = |f: usize| ((((f + 1) as f32 / t as f32) * cw).ceil() as usize).min(cx1 - cx0); + let ink = Ink::new(gray, img_w, crop); + + // Cuts between neighbouring words. + let mut cuts = vec![0usize]; + for pair in groups.windows(2) { + let a = right(pair[0].last().map_or(0, |e| e.t1)); + let b = left(pair[1].first().map_or(0, |e| e.t0)).max(a); + let cut = ink + .as_ref() + .and_then(|ink| { + // Widest ink-free run of columns in [a, b). + let (mut best, mut run_start, mut best_len) = (None, None, 0usize); + for x in a..=b.min(ink.w) { + let free = x < b.min(ink.w) && !ink.col(x, 0..ink.h); + match (free, run_start) { + (true, None) => run_start = Some(x), + (false, Some(s)) => { + if x - s > best_len { + best_len = x - s; + best = Some((s + x) / 2); + } + run_start = None; + } + _ => {} + } + } + best + }) + .unwrap_or((a + b) / 2); + cuts.push(cut.max(*cuts.last().unwrap_or(&0))); + } + cuts.push(cx1 - cx0); + + groups + .iter() + .enumerate() + .map(|(i, g)| { + let mut text: String = g.iter().map(|e| e.text.as_str()).collect(); + fold(&mut text); + let conf = g.iter().map(|e| e.p).sum::() / g.len() as f32; + let (lo, hi) = (cuts[i], cuts[i + 1].max(cuts[i] + 1)); + let bbox = ink.as_ref().and_then(|ink| snap(ink, lo, hi)).unwrap_or(BoundingBox { + x: (cx0 + lo) as f32, + y: cy0 as f32, + width: (hi - lo) as f32, + height: (cy1 - cy0) as f32, + }); + OcrWord { text, bbox: Some(bbox), confidence: Some(conf) } + }) + .collect() +} + +/// The ink box of crop columns `[lo, hi)`: the band of ink rows carrying the +/// most ink (gaps up to a tenth of the crop height bridged, so an accent stays +/// with its letter), then the first and last ink columns in that band. +/// +/// Most ink, not the densest row: the DB line box reaches into the lines above +/// and below, and a neighbour's full row of descenders can out-count any one +/// row of this word, but never this word's whole band. +fn snap(ink: &Ink, lo: usize, hi: usize) -> Option { + let hi = hi.min(ink.w); + if lo >= hi { + return None; + } + let rows: Vec = (0..ink.h).map(|y| (lo..hi).filter(|&x| ink.at(x, y)).count()).collect(); + let tol = (ink.h / 10).max(1); + // (top, bottom, mass) of each gap-bridged band. + let mut bands: Vec<(usize, usize, usize)> = Vec::new(); + let mut cur: Option<(usize, usize, usize)> = None; + for (y, &n) in rows.iter().enumerate() { + if n == 0 { + continue; + } + cur = match cur { + Some((t, b, m)) if y - b <= tol + 1 => Some((t, y, m + n)), + Some(band) => { + bands.push(band); + Some((y, y, n)) + } + None => Some((y, y, n)), + }; + } + bands.extend(cur); + let &(top, bot, _) = bands.iter().max_by_key(|b| b.2)?; + let cols: Vec = (lo..hi).filter(|&x| ink.col(x, top..bot + 1)).collect(); + let (&x0, &x1) = (cols.first()?, cols.last()?); + Some(BoundingBox { + x: (ink.x0 + x0) as f32, + y: (ink.y0 + top) as f32, + width: (x1 + 1 - x0) as f32, + height: (bot + 1 - top) as f32, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn e(text: &str, t0: usize, t1: usize) -> Emit { + Emit { text: text.into(), t0, t1, p: 0.9 } + } + + /// Two ink blobs on white with a wide gap; the decoder put its space in + /// the gap. Each word's box must be exactly its blob. + #[test] + fn words_snap_to_their_ink() { + let (w, h) = (100usize, 30usize); + let mut gray = vec![255f32; w * h]; + for y in 10..20 { + for x in (10..30).chain(60..90) { + gray[y * w + x] = 0.0; + } + } + // 10 timesteps over the 100-px crop: "ab" in frames 1-2, space 4-5, + // "cd" in frames 6-8. + let emits = [e("a", 1, 1), e("b", 2, 2), e(" ", 4, 5), e("c", 6, 7), e("d", 8, 8)]; + let words = boxes(&emits, 10, &gray, w, (0, 0, w, h), |_| {}); + assert_eq!(words.iter().map(|w| w.text.as_str()).collect::>(), ["ab", "cd"]); + let b0 = words[0].bbox.unwrap(); + let b1 = words[1].bbox.unwrap(); + assert_eq!((b0.x, b0.y, b0.width, b0.height), (10.0, 10.0, 20.0, 10.0)); + assert_eq!((b1.x, b1.y, b1.width, b1.height), (60.0, 10.0, 30.0, 10.0)); + } + + /// A neighbouring line's descenders inside the crop's top rows stay out. + #[test] + fn a_neighbouring_line_is_not_in_the_box() { + let (w, h) = (60usize, 40usize); + let mut gray = vec![255f32; w * h]; + for y in 0..3 { + for x in 5..55 { + gray[y * w + x] = 0.0; // the line above + } + } + for y in 15..28 { + for x in 10..50 { + gray[y * w + x] = 0.0; + } + } + let words = boxes(&[e("word", 1, 8)], 10, &gray, w, (0, 0, w, h), |_| {}); + let b = words[0].bbox.unwrap(); + assert_eq!((b.x, b.y, b.width, b.height), (10.0, 15.0, 40.0, 13.0)); + } +} From cb2cf0fbcb0c3abd5a61e1f3bd5d591b850bab77 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 15:03:00 -0700 Subject: [PATCH 06/25] perf(carmenta): auto_orient reads the page first and decides only when unconvincing PdfCraft measured auto_orient at 2.29x the time of an upright 300 dpi page on one core (18.3 -> 43.6 s) and 2.17x on eight: the decision (a detection pass plus two dozen crop reads) ran on every page, and an upright page was then read in full anyway. Its corpus also showed the page's own confidence separating the cases: upright 0.967-0.997 (0.806 for all capitals), turned 90/180/270 0.000 / 0.474 / 0.267, blank none. Now the page is read as given; a read at confidence >= 0.7 (characters- weighted mean of its lines) is returned as is, and only otherwise does the decision run (returning the first read when it says upright). PdfCraft's corpus, deskew on: upright pages identical output, time 1.004x (ABBA over 10 pages, 3/6 wins); rotated 90/180/270 91.27% -> 0.00% CER; ALL 6.23% -> 0.90%. README: orientation row corrected (the 48% figure was the decision alone), deskew and word-box rows added. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/README.md | 7 ++++-- crates/ffai-carmenta/src/engine.rs | 36 ++++++++++++++++++++++++++++-- crates/ffai-core/src/engine.rs | 4 +++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index fd80ff56..c98f6a85 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -158,7 +158,9 @@ What else 0.11 adds, each measured: | capability | API | measured | |---|---|---| -| orientation, 0/90/180/270 | `OcrOptions::auto_orient`, `detect_orientation` | **144 / 144** turns undone on document pages and single form lines; a decision costs **48 %** of one page read | +| orientation, 0/90/180/270 | `OcrOptions::auto_orient`, `detect_orientation` | **144 / 144** turns undone on document pages and single form lines. The page is read as given first and the decision runs only when that read's confidence is below 0.7: **1.004x** on upright pages (ABBA, 3/6), where the unconditional decision cost 2.2x (PdfCraft); turned pages 91 % → 0 % CER | +| deskew, small tilt | `OcrOptions::deskew`, `OcrOutput::skew_degrees` | angle from the detector's own text regions; pages 0.5-5° off read **0.10 %** CER (48.3 % without), a 1.5°-tilted phone photo **0.11 %** (61 %); square pages untouched (PdfCraft's corpus) | +| word boxes | `OcrLine::words` (mobiledet-svtr) | from the CTC frames, snapped to ink: **96 %** of ground-truth words at IoU ≥ 0.5 with the right text (PdfCraft's corpus) | | checkboxes, frame + marked/empty | `checkbox::find` | **100 %** recall and **100 %** state on 960 synthetic boxes (stroked and glyph boxes; X, overflowing tick, fill). Known false positives: the square ideograph `口`, and small square plot markers | | label → value pairs | `forms::pair_fields` | **220 / 221** correctly-read values paired with their label | | restrict a field's characters | `OcrOptions::charset` | forced characters show in the confidence: a prose page read as digits drops from 0.918 to 0.781 mean | @@ -248,7 +250,8 @@ turns it off. pipeline still trails PaddleOCR (20.9 % vs 15.6 % CER) even though our recognition stage beats theirs on identical crops — the diagnosed cause is tilt-sensitive line grouping, with deskew as the named fix. `auto_orient` -handles quarter turns, not small tilt: deskew is still open. +handles quarter turns and `deskew` small tilt (measured on rendered and +synthetic-photo pages; not yet re-run on the receipt corpus). ## Status: `experimental`, honestly diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index b536d0dd..4eab24ff 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -53,6 +53,26 @@ pub enum DetStage { Composed, } +/// A page read at or above this confidence is taken as upright and +/// `auto_orient` does not run its orientation decision on it. Between the +/// lowest upright page PdfCraft measured (0.806, all capitals) and the highest +/// turned one (0.474, upside down). +const ORIENT_TRUST_ABOVE: f32 = 0.7; + +/// A page's recognition confidence: the line confidences weighted by their +/// characters. `None` when no line has text and a confidence. +fn page_confidence(out: &OcrOutput) -> Option { + let (mut sum, mut n) = (0f64, 0usize); + for line in out.lines() { + if let Some(c) = line.confidence { + let k = line.text.chars().filter(|c| !c.is_whitespace()).count(); + sum += f64::from(c) * k as f64; + n += k; + } + } + (n > 0).then(|| (sum / n as f64) as f32) +} + /// Minimum short side the mobile-det input is scaled UP to; larger images pass /// through untouched (capped at `FFAI_DET_MAX_SIDE`). See /// `image::mobiledet_input` for why this is a floor and not a ceiling — the @@ -775,10 +795,22 @@ impl OcrEngine for CraftCrnn { // its boxes are mapped back so the caller's coordinates never change // meaning underneath them. if opts.auto_orient { - let o = self.detect_orientation(img)?; let inner = OcrOptions { auto_orient: false, ..opts.clone() }; + // Read the page as given FIRST, and decide orientation only when + // that read is unconvincing. PdfCraft measured the unconditional + // decision at 2.2-2.3x the time of every page, upright ones + // included (the decision is a detection pass plus two dozen crop + // reads, and an upright page was then read in full anyway), while + // the page's own confidence already separates the cases: upright + // pages 0.967-0.997 (0.806 for a page of capitals), pages turned + // 90 / 180 / 270 degrees 0.000 / 0.474 / 0.267, a blank page none. + let first = self.recognize(img, &inner)?; + if page_confidence(&first).is_some_and(|c| c >= ORIENT_TRUST_ABOVE) { + return Ok(first); + } + let o = self.detect_orientation(img)?; if o.turns == 0 { - return self.recognize(img, &inner); + return Ok(first); } let upright = crate::orient::rotate(img, o.turns); let mut out = self.recognize(&upright, &inner)?; diff --git a/crates/ffai-core/src/engine.rs b/crates/ffai-core/src/engine.rs index e1478990..1d7b944f 100644 --- a/crates/ffai-core/src/engine.rs +++ b/crates/ffai-core/src/engine.rs @@ -269,7 +269,9 @@ pub struct OcrOptions { /// way. Scanners store portrait forms sideways and phones photograph in /// any orientation; read sideways, a page silently comes back as a few /// low-confidence lines. Returned boxes stay in the INPUT image's - /// coordinates. Off by default: it costs an extra detection pass. + /// coordinates. Off by default. An engine may read the page as given + /// first and decide orientation only when that read is unconvincing, so + /// on an upright page it can cost nothing beyond the read itself. /// /// Engines that do not implement it must return an error when it is set. pub auto_orient: bool, From 059e9c34c19eeda918613b9b680188de9e686033 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 15:26:31 -0700 Subject: [PATCH 07/25] feat(core): threads::available_threads / affinity_pool honour the process affinity mask std::thread::available_parallelism ignores the affinity mask on Windows (24 under `start /affinity 4`), so rayon's global pool and candle's gemm fan-out ran two dozen threads on a process pinned to one core. PdfCraft measured sizing the pool to the mask (ABBA, identical output): 8 cores 0.927x time (10/10) and peak 1959 -> 1240 MB; 1 core neutral time and 1285 -> 747 MB. It could not read the mask itself: that is an FFI call, and it forbids unsafe. available_threads() is that call (GetProcessAffinityMask, one SAFETY-noted unsafe line) behind a safe function, falling back to available_parallelism elsewhere; affinity_pool() builds a rayon pool of that size to install the engine in. windows-sys 0.61 was already in the tree. Checked: 1 under mask 0x4, 8 under 0x3FC, 24 unmasked. pdfcraft_bench reports it and takes BENCH_POOL=1. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + .../ffai-carmenta/examples/pdfcraft_bench.rs | 10 +++ crates/ffai-core/Cargo.toml | 5 ++ crates/ffai-core/src/lib.rs | 1 + crates/ffai-core/src/threads.rs | 68 +++++++++++++++++++ 5 files changed, 85 insertions(+) create mode 100644 crates/ffai-core/src/threads.rs diff --git a/Cargo.lock b/Cargo.lock index 70c0047a..43aca379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1320,6 +1320,7 @@ dependencies = [ "candle-core", "rayon", "thiserror 2.0.19", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs index a7244f5e..6b7b7c9c 100644 --- a/crates/ffai-carmenta/examples/pdfcraft_bench.rs +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -39,6 +39,16 @@ fn esc(s: &str) -> String { } fn main() { + eprintln!("available_threads {}", ffai_core::threads::available_threads()); + if std::env::var("BENCH_POOL").is_ok_and(|v| v == "1") { + let pool = ffai_core::threads::affinity_pool().expect("thread pool"); + pool.install(run); + } else { + run(); + } +} + +fn run() { let mut args = std::env::args().skip(1); let root = args.next().expect("usage: pdfcraft_bench ..."); let pages: Vec = args.collect(); diff --git a/crates/ffai-core/Cargo.toml b/crates/ffai-core/Cargo.toml index 506ba3fa..35542b8f 100644 --- a/crates/ffai-core/Cargo.toml +++ b/crates/ffai-core/Cargo.toml @@ -24,6 +24,11 @@ candle-core = { workspace = true } rayon = { workspace = true } thiserror = { workspace = true } +# `threads::available_threads` reads the process affinity mask, which std does +# not on Windows. Already in the tree (0.61) through rayon/std consumers. +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_Threading"] } + [features] # GPU backends forward straight to candle. Note: today these pull C/CUDA/Metal # tooling — the pure-Rust replacement path (cool-japan/oxicuda) is tracked in diff --git a/crates/ffai-core/src/lib.rs b/crates/ffai-core/src/lib.rs index e9cd7ad0..d85983f1 100644 --- a/crates/ffai-core/src/lib.rs +++ b/crates/ffai-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod error; pub mod fastmath; pub mod fastops; pub mod registry; +pub mod threads; pub mod types; /// The shared tensor framework (Hugging Face candle). diff --git a/crates/ffai-core/src/threads.rs b/crates/ffai-core/src/threads.rs new file mode 100644 index 00000000..67dbfa10 --- /dev/null +++ b/crates/ffai-core/src/threads.rs @@ -0,0 +1,68 @@ +//! How many threads this process may actually run on. +//! +//! `std::thread::available_parallelism` ignores the process affinity mask on +//! Windows: under `start /affinity 4` (one core) it still reports every +//! logical CPU. rayon sizes its global pool the same way, and candle hands +//! gemm `Parallelism::Rayon(physical cores)` on every matmul, so a process +//! pinned to one core ran two dozen threads on it. +//! +//! PdfCraft measured what sizing the pool to the mask is worth (ABBA, output +//! identical): on 8 cores, 0.927x the time (10/10 rounds) and peak memory +//! 1 959 -> 1 240 MB; on 1 core, time neutral and memory 1 285 -> 747 MB. It +//! could not do it itself, because reading the mask is an FFI call and it +//! forbids `unsafe`. This module is that one call, behind a safe function. +//! +//! Use [`affinity_pool`] and run the engine inside it: +//! +//! ```no_run +//! let pool = ffai_core::threads::affinity_pool().expect("thread pool"); +//! pool.install(|| { +//! // engine.recognize(&image, &options) — every rayon task the engine and +//! // candle spawn now runs on this pool's threads. +//! }); +//! ``` + +/// The number of CPUs this process may run on: the affinity mask on Windows, +/// `available_parallelism` elsewhere (which already honours affinity and +/// cgroup quotas on Linux). At least 1. +#[must_use] +pub fn available_threads() -> usize { + #[cfg(windows)] + if let Some(n) = windows_affinity() { + return n; + } + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) +} + +#[cfg(windows)] +fn windows_affinity() -> Option { + use windows_sys::Win32::System::Threading::{GetCurrentProcess, GetProcessAffinityMask}; + let (mut process, mut system) = (0usize, 0usize); + // SAFETY: `GetCurrentProcess` returns a pseudo-handle that is always valid + // for the calling process and needs no closing; both out-pointers point at + // live, writable `usize`s for the duration of the call. + #[allow(unsafe_code)] + let ok = unsafe { GetProcessAffinityMask(GetCurrentProcess(), &raw mut process, &raw mut system) }; + // The mask covers the process's processor group (at most 64 CPUs). + (ok != 0 && process != 0).then(|| process.count_ones() as usize) +} + +/// A rayon pool with [`available_threads`] threads. +/// +/// # Errors +/// When rayon cannot spawn the threads. +pub fn affinity_pool() -> Result { + rayon::ThreadPoolBuilder::new().num_threads(available_threads()).build() +} + +#[cfg(test)] +mod tests { + #[test] + fn at_least_one_and_at_most_the_machine() { + let n = super::available_threads(); + assert!(n >= 1); + // On Windows the mask can only narrow the count, never widen it. + #[cfg(windows)] + assert!(n <= std::thread::available_parallelism().map_or(usize::MAX, std::num::NonZeroUsize::get)); + } +} From 3688bd8164f475c0cd035eaa38d8cb032f07f958 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 15:26:32 -0700 Subject: [PATCH 08/25] docs(carmenta): state the SVTR charset's Latin gap in the README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowercase French/German/Spanish read 1.9 / 0.9 / 0.1 % CER on PdfCraft's rendered pages, but the 18 383-character head cannot emit À Â Ê Ë Ì Î Ï Ñ Ò Ô Õ Ø Ù Ú Û Ý, î ï ÿ œ or « » ¿ ¡ ‘ ™ ‰ (checked against charset.txt): a page of accented capitals read 24 % CER. OcrOptions::charset is the documented way to keep such a page from emitting CJK look-alikes. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index c98f6a85..fa5f683d 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -85,6 +85,15 @@ additionally be routed to dedicated structure models (`FFAI_ROUTE=1`, experimental: pure-Rust ports of PP-DocLayout-S, SLANet-plus and PP-FormulaNet-S, each matched against its reference runtime). +**Latin beyond English is partial, and the gap is in the charset, not the +model's skill.** On rendered pages lowercase French reads 1.9 % CER, German +0.9 %, Spanish 0.1 % (PdfCraft's corpus). But the head's 18 383 characters do +not include `À Â Ê Ë Ì Î Ï Ñ Ò Ô Õ Ø Ù Ú Û Ý`, `î ï ÿ œ` or `« » ¿ ¡ ‘ ™ ‰`, so +those can never be emitted: a page of accented capitals read 24 % CER +(`ESPAÑA` → `EsPAnA`). `OcrOptions::charset` restricts decoding to what a +caller can store, and is how to keep a sideways or capitals-heavy page from +emitting CJK look-alikes. + Measured by [OmniDocBench](https://github.com/opendatalab/OmniDocBench)'s **own evaluator** — the benchmark PaddleOCR-VL and Baidu's Unlimited-OCR state their records on — over all 1 651 pages of v1.6, English and Chinese, zero From ed7b78d36189a0ed7d9146515efa25277ae8ddee Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 15:28:44 -0700 Subject: [PATCH 09/25] style: clippy pedantic on the new code (doc backticks, identity map, collapsible if) Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/deskew.rs | 2 +- crates/ffai-carmenta/src/svtr.rs | 9 +++++---- crates/ffai-core/src/threads.rs | 2 +- crates/ffai-models/src/lib.rs | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/ffai-carmenta/src/deskew.rs b/crates/ffai-carmenta/src/deskew.rs index a1004897..76005f33 100644 --- a/crates/ffai-carmenta/src/deskew.rs +++ b/crates/ffai-carmenta/src/deskew.rs @@ -135,7 +135,7 @@ fn paper_colour(img: &ImageBuffer) -> [u8; 4] { return out; } let mut idx: Vec = Vec::with_capacity(2 * (w + h)); - idx.extend((0..w).map(|x| x)); + idx.extend(0..w); idx.extend((0..w).map(|x| (h - 1) * w + x)); idx.extend((0..h).map(|y| y * w)); idx.extend((0..h).map(|y| y * w + w - 1)); diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index e9dfc1bc..3de88af9 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -762,10 +762,11 @@ pub fn ctc_greedy_emits( open = true; } kept.push(conf[i]); - } else if k != 0 && open { - if let Some(e) = emits.last_mut() { - e.t1 = i; - } + } else if k != 0 + && open + && let Some(e) = emits.last_mut() + { + e.t1 = i; } prev = k; } diff --git a/crates/ffai-core/src/threads.rs b/crates/ffai-core/src/threads.rs index 67dbfa10..dc68620c 100644 --- a/crates/ffai-core/src/threads.rs +++ b/crates/ffai-core/src/threads.rs @@ -6,7 +6,7 @@ //! gemm `Parallelism::Rayon(physical cores)` on every matmul, so a process //! pinned to one core ran two dozen threads on it. //! -//! PdfCraft measured what sizing the pool to the mask is worth (ABBA, output +//! `PdfCraft` measured what sizing the pool to the mask is worth (ABBA, output //! identical): on 8 cores, 0.927x the time (10/10 rounds) and peak memory //! 1 959 -> 1 240 MB; on 1 core, time neutral and memory 1 285 -> 747 MB. It //! could not do it itself, because reading the mask is an FFI call and it diff --git a/crates/ffai-models/src/lib.rs b/crates/ffai-models/src/lib.rs index ff4e994d..97e91be1 100644 --- a/crates/ffai-models/src/lib.rs +++ b/crates/ffai-models/src/lib.rs @@ -310,7 +310,7 @@ pub fn cache_dir() -> PathBuf { /// /// Written out rather than taken from `dirs`, whose `dirs-sys` pulls /// `option-ext`, an MPL-2.0 crate: one call site was enough to turn a -/// permissive-only `cargo deny` policy red for every consumer (PdfCraft, +/// permissive-only `cargo deny` policy red for every consumer (`PdfCraft`, /// 2026-10-30). fn os_cache_dir() -> Option { let abs = |v: std::ffi::OsString| Some(PathBuf::from(v)).filter(|p| p.is_absolute()); From 47d6f5c80572e980db583e3854041ac66205b466 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 16:08:07 -0700 Subject: [PATCH 10/25] feat(carmenta): read table bands across their rows (ruled table 77% -> 0% CER) The recursive cut read PdfCraft's 13-row ruled table as columns -- its gutters are the widest gaps on the page -- so every word came back right in an order scoring 77% CER (ocrs, reading rows: 7%). boxes::grid_order runs on the FINISHED line order (after the cut, the sparse gate and the 8.157 probe reorder, any of which scattered rows re-ordered earlier) and re-emits a table band row-major into the slots its lines held, so nothing outside a band moves. A band is 4+ consecutive rows that are all: - grid rows: 2+ cells, every gap >= the wider neighbour (1.5x for 2 cells); - columns: the same cell count, cell j overlapping the previous row's cell j; - short: median cell width/height <= 8. FFAI_ORDER_GRID=0 disables. Gated on OmniDocBench (315 pages, one binary, FFAI_ORDER_GRID=0 vs on). The first two versions were REFUTED there: gap + alignment alone changed 29 and then 13 pages, 12 worse (omni-0137 3.07% -> 14.15%): three-column newspapers whose justified lines split into fragments, and two prose columns either side of a photograph, which pass every geometric test except cell shape. With the short-cell condition: 1 page changed, better (omni-0005 83.47 -> 83.37%), macro 22.682% -> 22.681%. PdfCraft's corpus: table-300 77.04% -> 0.00%, no other page changed. README: OmniDocBench pages now read at 1.4 s median / 2.7 s p90 per page on 24 threads (was measured at ~9-17 s/page). Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/README.md | 8 +- crates/ffai-carmenta/src/boxes.rs | 172 ++++++++++++++++++++++++++++- crates/ffai-carmenta/src/engine.rs | 19 ++++ 3 files changed, 194 insertions(+), 5 deletions(-) diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index fa5f683d..b819245d 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -138,8 +138,12 @@ So the remaining loss is mostly **coverage and characters**, not layout: the reading-order and detection stages this crate spent its campaign on are down to a combined 4 % of remaining error. -All of this on CPU at ~9–17 s/page — roughly 9× the reference VLM's measured -throughput on the same machine, from megabytes of weights against gigabytes. +All of this on CPU, from megabytes of weights against gigabytes. It was +measured at ~9–17 s/page, roughly 9× the reference VLM's throughput on the +same machine; since the depthwise-convolution and fused-epilogue work +(2026-10-30; recognizer bit-identical, text and boxes identical on PdfCraft's +42-page corpus) the same 315 OmniDocBench pages read at a +**1.4 s median, 2.7 s p90** per page on a 24-thread i7-14650HX. ## Forms, scanned filings and PDFs (0.11) diff --git a/crates/ffai-carmenta/src/boxes.rs b/crates/ffai-carmenta/src/boxes.rs index 28d7d246..b06a7220 100644 --- a/crates/ffai-carmenta/src/boxes.rs +++ b/crates/ffai-carmenta/src/boxes.rs @@ -178,6 +178,34 @@ pub fn line_bbox(line: &[DetBox]) -> DetBox { mod tests { use super::*; + fn db(x0: usize, y0: usize, x1: usize, y1: usize) -> Vec { + vec![DetBox { x0, y0, x1, y1, score: 1.0 }] + } + + /// A 3-column, 4-row table emitted column by column comes back row by + /// row; two columns of prose emitted column by column stay as they are. + #[test] + fn tables_read_across_and_prose_columns_do_not() { + let mut table = Vec::new(); + for col in [100usize, 800, 1400] { + for row in 0..4 { + table.push(db(col, 300 + row * 80, col + 100, 330 + row * 80)); + } + } + let bbs: Vec = table.iter().map(|l| l[0]).collect(); + let xs: Vec = grid_order(&bbs).iter().map(|&i| bbs[i].x0).collect(); + assert_eq!(&xs[..6], &[100, 800, 1400, 100, 800, 1400]); + + let mut prose = Vec::new(); + for col in [100usize, 1160] { + for row in 0..6 { + prose.push(db(col, 300 + row * 50, col + 1000, 330 + row * 50)); + } + } + let bbs: Vec = prose.iter().map(|l| l[0]).collect(); + assert_eq!(grid_order(&bbs), (0..bbs.len()).collect::>()); + } + #[test] fn two_words_one_line_group_and_order() { // Two boxes on one row (out of x order), one on a lower row. @@ -455,16 +483,154 @@ pub fn order_reading(lines: Vec>, page_w: usize) -> Vec> // (§8.119, the metric `ffai-bench` reports), and the two losses are // a CJK page we cannot read either way and a 344-character contents // page where +3.5 pp is twelve characters. - if std::env::var("FFAI_ORDER_GATE").as_deref() != Ok("0") + let out = if std::env::var("FFAI_ORDER_GATE").as_deref() != Ok("0") && sparse_scatter(&out) > env_f32("FFAI_ORDER_GATE_T", ORDER_GATE_T) { - return xy_cut(out, page_w, 0); - } + xy_cut(out, page_w, 0) + } else { + out + }; + // Table bands are re-read across their rows at the END of the + // pipeline, after every reorder (`grid_order`). out } } } +/// Lines sharing a row: y-centres within half the median line height. +/// Each row is a list of indices into `lines`, sorted left to right; rows are +/// top to bottom. +fn rows_of(bbs: &[DetBox]) -> Vec> { + let mut hs: Vec = bbs.iter().map(|b| b.y1.saturating_sub(b.y0)).collect(); + hs.sort_unstable(); + let tol = hs.get(hs.len() / 2).copied().unwrap_or(0) as f32 * 0.5; + let cy = |i: usize| (bbs[i].y0 + bbs[i].y1) as f32 / 2.0; + let mut order: Vec = (0..bbs.len()).collect(); + order.sort_by(|&a, &b| cy(a).total_cmp(&cy(b))); + let mut rows: Vec> = Vec::new(); + for i in order { + match rows.last_mut() { + Some(r) if (cy(i) - cy(r[0])).abs() <= tol => r.push(i), + _ => rows.push(vec![i]), + } + } + for r in &mut rows { + r.sort_by_key(|&i| bbs[i].x0); + } + rows +} + +/// Read a TABLE row by row, leaving everything else where the cut put it. +/// +/// The recursive cut reads a ruled table as columns: its gutters are the +/// widest gaps on the page, so it splits there first. PdfCraft's 13-row table +/// came back with every word right and 77 % CER in row order (ocrs, reading +/// rows, 7 %). A column of PROSE beside another lines up row for row as well, +/// so alignment alone cannot be the test. Width can: a table's gutters are +/// wide against the cells either side of them (the 13-row table: cells about +/// 100 px, gutters 260-700 px), while a prose gutter is a sliver of the lines +/// it separates (60 px against 1000). +/// +/// So a GRID ROW is one of two or more cells where every gap between +/// neighbours is at least [`GRID_GAP_3`] times the wider of the two +/// ([`GRID_GAP_2`] for a row of just two), and [`GRID_MIN_ROWS`] or more +/// consecutive grid rows whose cells line up as columns, and are short +/// ([`GRID_MAX_ASPECT`]), form a table band. The +/// band's lines are re-emitted row-major INTO THE SLOTS they already held in +/// the order, so nothing outside a band moves. It runs on the FINISHED line +/// order — after the cut, the sparse-page gate and the §8.157 probe reorder, +/// any of which would otherwise scatter the rows again. +/// +/// `bbs` are the lines' boxes in reading order; the result is the new order as +/// indices into them (the identity when there is no table). +/// `FFAI_ORDER_GRID=0` disables. +#[must_use] +pub fn grid_order(bbs: &[DetBox]) -> Vec { + let identity: Vec = (0..bbs.len()).collect(); + if bbs.len() < 2 * GRID_MIN_ROWS || std::env::var("FFAI_ORDER_GRID").as_deref() == Ok("0") { + return identity; + } + let rows = rows_of(bbs); + let is_grid = |r: &Vec| { + // Two cells need the wider margin: a short last line of one prose + // column beside a full line of the next has a gap about one line + // wide. + let k = if r.len() >= 3 { GRID_GAP_3 } else { GRID_GAP_2 }; + r.len() >= 2 + && r.windows(2).all(|p| { + let (a, b) = (&bbs[p[0]], &bbs[p[1]]); + let gap = b.x0.saturating_sub(a.x1) as f32; + let wider = (a.x1 - a.x0).max(b.x1 - b.x0) as f32; + gap >= k * wider + }) + }; + // Bands: maximal runs of consecutive grid rows that also form COLUMNS — + // the same number of cells in every row, and each row's cell `j` + // overlapping the previous row's cell `j` horizontally. Prose fails this + // even where a few rows pass the gap test (OmniDocBench's three-column + // newspaper pages, whose justified lines the detector splits into + // fragments with wide gaps between them). + let continues = |prev: &Vec, r: &Vec| { + prev.len() == r.len() + && prev.iter().zip(r).all(|(&a, &b)| bbs[a].x0 < bbs[b].x1 && bbs[b].x0 < bbs[a].x1) + }; + // Table cells are SHORT: the median cell is at most [`GRID_MAX_ASPECT`] + // times wider than tall (the 13-row table: about 3.3). Two prose columns + // with a photograph between them pass every test above — wide gap, + // consistent columns (OmniDocBench omni-0124..0152, newspaper pages) — + // but their cells are whole lines, about 15 times wider than tall. + let short_cells = |band: &[Vec]| { + let mut aspects: Vec = band + .iter() + .flatten() + .map(|&k| (bbs[k].x1 - bbs[k].x0) as f32 / (bbs[k].y1 - bbs[k].y0).max(1) as f32) + .collect(); + aspects.sort_by(f32::total_cmp); + aspects.get(aspects.len() / 2).is_some_and(|&a| a <= GRID_MAX_ASPECT) + }; + let mut bands: Vec<&[Vec]> = Vec::new(); + let mut start: Option = None; + for i in 0..=rows.len() { + let extends = i < rows.len() + && is_grid(&rows[i]) + && start.is_some_and(|s| i > s && continues(&rows[i - 1], &rows[i])); + if extends { + continue; + } + if let Some(s) = start.take() + && i - s >= GRID_MIN_ROWS + && short_cells(&rows[s..i]) + { + bands.push(&rows[s..i]); + } + if i < rows.len() && is_grid(&rows[i]) { + start = Some(i); + } + } + if bands.is_empty() { + return identity; + } + let mut perm = identity; + for band in bands { + let row_major: Vec = band.iter().flatten().copied().collect(); + let mut slots = row_major.clone(); + slots.sort_unstable(); // the positions the band's lines held + for (slot, line) in slots.into_iter().zip(row_major) { + perm[slot] = line; + } + } + perm +} + +/// A gap at least this many times the wider neighbouring cell is a table +/// gutter, in a row of two cells and in a row of three or more. +const GRID_GAP_2: f32 = 1.5; +const GRID_GAP_3: f32 = 1.0; +/// Consecutive grid rows that make a table band. +const GRID_MIN_ROWS: usize = 4; +/// Median width/height of a table band's cells, at most. +const GRID_MAX_ASPECT: f32 = 8.0; + /// Mean vertical step between consecutive emitted lines, over the ink extent. /// /// §8.153 decomposed ordering cost into 1.90 pp of SELECTOR REGRET and 2.90 pp diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index 4eab24ff..c45e4e72 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -1411,6 +1411,25 @@ impl OcrEngine for CraftCrnn { }; let out_lines = crate::suppress::probe_reorder(out_lines, &probe_stats, lowconf, w as f32, h as f32); + // Table bands read across their rows, on the finished order. + let out_lines = { + let bbs: Vec = out_lines + .iter() + .map(|l| { + let b = l.bbox.unwrap_or(BoundingBox { x: 0.0, y: 0.0, width: 0.0, height: 0.0 }); + boxes::DetBox { + x0: b.x.max(0.0) as usize, + y0: b.y.max(0.0) as usize, + x1: (b.x + b.width).max(0.0) as usize, + y1: (b.y + b.height).max(0.0) as usize, + score: 1.0, + } + }) + .collect(); + let perm = boxes::grid_order(&bbs); + let mut slots: Vec> = out_lines.into_iter().map(Some).collect(); + perm.into_iter().filter_map(|i| slots[i].take()).collect::>() + }; // §48: REGION ROUTING. Runs last, on the finished sequence, so the // reading order §29–§43 earned is spliced into rather than rebuilt. From 8b1000b6034e902286e8cd0aae6b74d702830dd1 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 17:12:42 -0700 Subject: [PATCH 11/25] fix(carmenta): cap the detector floor's upscale at 4x; record the refuted per-page-type cap #13 (PdfCraft): its 344x108 "HELLO WORLD" test image (28 px blocky glyphs) read `HELL` + `WORLO`. The 736 px floor enlarged it 6.8x, the box scores sank to 0.41-0.64 against the 0.6 threshold, and the final O fell outside its box. The floor exists for small TEXT, not small images: capping its factor at 4x (FFAI_DET_MAX_UP, 0 = uncapped) gives one box at 0.91 and `HELLO`. The `D` -> `O` that remains is the recognizer's own reading of a blocky 5x7 D (it differs from the O by two corner dots), not geometry. Inert elsewhere: carmenta-form's 240 single-line clips (4.9x before) identical, 0.407% CER; carmenta-capture (3.68x) below the cap; PdfCraft's 42 pages byte-identical. #11 (PdfCraft): a lower detector cap for rendered pages. Measured 1280 vs 736 and refuted, now documented at DET_MAX_SHORT_DEFAULT: content::classify calls all 32 carmenta-doc scans Rendered, and 736 loses on 14 of them (up to +18.4 pp); on rendered pages the ruled table goes 0 -> 32.7% and capitals 24.1 -> 31.0%; OmniDocBench is a wash (65 worse, 67 better); a median-text-height signal does not separate the losses (9.1-11.9 px vs neutral at 8.7 px). 1280 stays. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/image.rs | 52 ++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/crates/ffai-carmenta/src/image.rs b/crates/ffai-carmenta/src/image.rs index 9cff96a8..56a80092 100644 --- a/crates/ffai-carmenta/src/image.rs +++ b/crates/ffai-carmenta/src/image.rs @@ -654,8 +654,58 @@ fn det_max_short() -> usize { /// `carmenta-capture` is unaffected by construction: all 108 clips are 620x200, /// short side 200, so the FLOOR fires and the ceiling never can. The LIVE record /// is untouched. +/// +/// # A cap by page type: refuted (2026-10-30) +/// +/// After the depthwise work made detection cheap enough that the cap matters +/// again (one 300 dpi page, one thread: detection 0.88 s at 1280, 0.47 s at +/// 960, 0.28 s at 736), PdfCraft asked for a lower cap on RENDERED pages, +/// where 736 had looked neutral (0.15 -> 0.18 % CER). Measured with +/// mobiledet-svtr, 1280 vs 736: +/// +/// * the page-type signal does not exist: `content::classify` puts all 32 +/// `carmenta-doc` scans in `Rendered`, beside 40 of PdfCraft's 42 rendered +/// pages — and on those scans 736 is worse on 14 of 32, better on none, by +/// up to +18.4 pp (doc-007); +/// * on rendered pages it is not neutral either: PdfCraft's ruled table +/// 0.00 -> 32.70 %, its capitals page 24.1 -> 31.0 %; OmniDocBench is a wash +/// (65 pages worse, 67 better); +/// * text size does not rescue it: with the median word height scaled to the +/// 736 input as the signal, the scan regressions sit at 9.1-11.9 px while +/// neutral pages sit at 8.7 px. No threshold separates them. +/// +/// So 1280 stays, for every page. [`set_det_max_short`] remains for callers +/// trading accuracy for speed knowingly. pub const DET_MAX_SHORT_DEFAULT: usize = 1280; +/// Largest factor the floor may scale an image UP by (`FFAI_DET_MAX_UP`; +/// unset or 0 = no cap). See [`DET_MAX_UP_DEFAULT`]. +fn det_max_up() -> f32 { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| { + std::env::var("FFAI_DET_MAX_UP") + .ok() + .and_then(|s| s.parse::().ok()) + .map_or(DET_MAX_UP_DEFAULT, |v| if v > 0.0 { v } else { f32::INFINITY }) + }) +} + +/// Default cap on the floor's upscale factor: **4x**. +/// +/// The floor ([`mobiledet_input`]'s `min_side`, 736) exists for SMALL text — +/// receipts with ~10 px glyphs need enlarging before DBNet sees them. Applied +/// to a small IMAGE of normal-sized text it over-enlarges instead: PdfCraft's +/// 344x108 "HELLO WORLD" (28 px glyphs) reached the detector 6.8x larger, the +/// box scores sank to 0.41-0.64 against the 0.6 threshold, and the final `O` +/// fell out of its box (`HELL` + `WORLO`; ocrs read it whole). At 3x or 4x it +/// is one box at 0.91 and reads `HELLO`. +/// +/// Inert elsewhere, measured: `carmenta-form`'s 240 single-line clips (short +/// side 150, 4.9x before) read identically, 0.407 % CER either way; +/// `carmenta-capture` (620x200, 3.68x) is under the cap by construction, so +/// the LIVE record is untouched. +pub const DET_MAX_UP_DEFAULT: f32 = 4.0; + pub fn mobiledet_input( img: &ImageBuffer, min_side: usize, @@ -676,7 +726,7 @@ pub fn mobiledet_input( let short = w.min(h) as f32; let max_short = det_max_short().max(min_side); let ratio = if (short as usize) < min_side { - min_side as f32 / short + (min_side as f32 / short).min(det_max_up()) } else if (short as usize) > max_short { // The CEILING, and the one this policy was missing — see the section // below. Exclusive with the floor: `max_short` is clamped up to From a09ef5799c1bf9dc2d3e903846a18009e69ede01 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 17:12:45 -0700 Subject: [PATCH 12/25] tools: stage the mobiledet-svtr weights for a versioned host (uploads nothing) PdfCraft finding #12: the ppocrv5 manifests pin hashes but name no hf_repo, so no crates.io user can fetch the document engine's weights. ffai-models already downloads by file name from a manifest's hf_repo; the repo itself is missing, and publishing is the owner's call. This copies exactly the files the engine loads from the cache, verifies each against the embedded manifest, writes a model card with the PaddleOCR Apache-2.0 attribution and a LICENSE, and prints the two upload commands and the hf_repo line to add. Co-Authored-By: Claude Opus 5.5 --- tools/carmenta_stage_weights.py | 119 ++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tools/carmenta_stage_weights.py diff --git a/tools/carmenta_stage_weights.py b/tools/carmenta_stage_weights.py new file mode 100644 index 00000000..3df36efe --- /dev/null +++ b/tools/carmenta_stage_weights.py @@ -0,0 +1,119 @@ +"""Stage the mobiledet-svtr weights for a versioned public host. Uploads NOTHING. + +Why: the `ppocrv5-mobile-det` and `ppocrv5-mobile-rec` manifests carry SHA-256 +but no `hf_repo`, so `fetch` cannot download them and a crates.io user has no +way to get the document engine's weights (PdfCraft, 2026-10-30, finding #12). +`ffai-models` already downloads from a Hugging Face repo named in a manifest's +`hf_repo`, by file name, verifying each file against its pinned hash; what is +missing is the repo. Publishing weights is an owner's decision, so this stops at +a staging directory and prints the two commands that finish the job. + + python -I tools/carmenta_stage_weights.py OUT_DIR [HF_REPO] + +OUT_DIR gets exactly the files the engine loads, each verified against the +crate's embedded manifest, plus a model card and the upstream licence notice. +HF_REPO (default `Remade-With-Rust/carmenta-ppocrv5-mobile`) is only used in the +printed commands and manifest lines. + +The weights are PaddlePaddle's PP-OCRv5 mobile detector and recognizer +(Apache-2.0), converted by `carmenta_mobiledet_prepare.py` + +`carmenta_mobiledet_fuse.py` and `carmenta_svtr_prepare.py`. Redistribution +under Apache-2.0 requires the licence and attribution, which the card carries. +""" +import hashlib +import os +import re +import shutil +import sys +from pathlib import Path + +MANIFESTS = Path("crates/ffai-carmenta/models") +CACHE = Path(os.environ.get("LOCALAPPDATA", "")) / "ffai" / "models" +# model -> the files the engine actually loads (engine.rs `fetch` calls). +FILES = { + "ppocrv5-mobile-det": ["det-fused.safetensors"], + "ppocrv5-mobile-rec": ["rec.safetensors", "charset.txt"], +} + +CARD = """--- +license: apache-2.0 +tags: [ocr, text-detection, text-recognition, safetensors] +--- + +# Carmenta: PP-OCRv5 mobile (detection + SVTR recognition), safetensors + +Weights for the `mobiledet-svtr` document engine of +[ffai-carmenta](https://crates.io/crates/ffai-carmenta), the pure-Rust OCR +engine of [FFAI](https://github.com/Remade-With-Rust/FFAI). + +| file | model | sha256 | +|---|---|---| +{rows} + +These are **PaddlePaddle's PP-OCRv5 mobile** text detector (DBNet / PP-LCNetV3) +and text recognizer (SVTR, 18 383 characters + blank), released by PaddleOCR +under the Apache License 2.0. They were converted to safetensors (the detector +with its re-parameterised branches and BatchNorms fused into plain +convolutions) and checked against PaddlePaddle's own exported programs. No +weights were retrained. Copyright for the original models remains with the +PaddlePaddle Authors; see `LICENSE`. + +Use: `ffai_carmenta::engine::CraftCrnn::offline(RecStage::Svtr, +DetStage::MobileDet, root)` with `root//`, or let the `fetch` +feature download them by the hashes pinned in the crate's manifests. +""" + + +def sha256(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def pinned(model: str) -> dict: + """file name -> sha256 from the crate's embedded manifest.""" + text = (MANIFESTS / f"{model}.toml").read_text(encoding="utf8") + return dict(re.findall(r'name = "([^"]+)"\s*\nsha256 = "([0-9a-f]{64})"', text)) + + +def main(): + if len(sys.argv) < 2: + sys.exit(__doc__) + out = Path(sys.argv[1]) + repo = sys.argv[2] if len(sys.argv) > 2 else "Remade-With-Rust/carmenta-ppocrv5-mobile" + out.mkdir(parents=True, exist_ok=True) + rows = [] + for model, files in FILES.items(): + pins = pinned(model) + for name in files: + src = CACHE / model / name + want = pins.get(name) + if want is None: + sys.exit(f"{model}.toml pins no `{name}`") + got = sha256(src) + if got != want: + sys.exit(f"{src}: sha256 {got} does not match the manifest's {want}; refusing to stage") + shutil.copy2(src, out / name) + rows.append(f"| `{name}` | {model} | `{want}` |") + print(f"staged {name} ({src.stat().st_size:,} B), sha256 verified") + (out / "README.md").write_text(CARD.format(rows="\n".join(rows)), encoding="utf8") + (out / "LICENSE").write_text( + "The model weights in this repository are derived from PaddleOCR's PP-OCRv5\n" + "mobile detection and recognition models, Copyright (c) PaddlePaddle Authors,\n" + "licensed under the Apache License, Version 2.0:\n" + "https://www.apache.org/licenses/LICENSE-2.0\n" + "Source: https://github.com/PaddlePaddle/PaddleOCR\n", + encoding="utf8", + ) + print(f"\nStaged in {out}. Nothing was uploaded. To publish (the owner's decision):\n") + print(f" huggingface-cli repo create {repo.split('/', 1)[1]} --organization {repo.split('/', 1)[0]} --type model") + print(f" huggingface-cli upload {repo} {out} . --commit-message \"PP-OCRv5 mobile det+rec for ffai-carmenta\"") + print("\nthen add this line under `license = ...` in BOTH manifests") + print("(crates/ffai-carmenta/models/ppocrv5-mobile-{det,rec}.toml and the copies in models/):\n") + print(f' hf_repo = "{repo}"') + + +if __name__ == "__main__": + main() From a6d6fb2672de729a68b5a33afeb42da40d8bdb6c Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 17:34:22 -0700 Subject: [PATCH 13/25] docs(carmenta): pdfcraft_bench describes its corpus instead of naming a local one; documents BENCH_POOL Co-Authored-By: Claude Opus 5.5 --- .../ffai-carmenta/examples/pdfcraft_bench.rs | 6 +- crates/ffai-diana/examples/alloc_ab.rs | 56 ++++++++++++++++++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs index 6b7b7c9c..df92890c 100644 --- a/crates/ffai-carmenta/examples/pdfcraft_bench.rs +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -2,8 +2,8 @@ //! write one JSON line per page in the shape PdfCraft's `score.py` reads: //! `{"page", "ms", "lines": [[[text, [l, t, r, b]], ...], ...]}`. //! -//! It exists so a caller's corpus (artcraft's pdfcraft-ocr pages, rendered -//! with ground truth by construction) can be scored against THIS tree, not +//! It exists so a caller's corpus — PdfCraft's 49 rendered pages, 42 of them +//! with ground truth by construction — can be scored against THIS tree, not //! only against the release on crates.io. A line with word geometry emits one //! entry per word; a line without it emits the whole line as one entry with //! the line box, which the scorer's CER is indifferent to. @@ -11,7 +11,7 @@ //! Usage: `pdfcraft_bench ...` //! Env: `BENCH_REC=crnn` for mobiledet-crnn, `BENCH_ORIENT=1` for //! `auto_orient`, `BENCH_DESKEW=1` for `deskew`, `BENCH_LANG=en,fr` for -//! `languages`. +//! `languages`, `BENCH_POOL=1` to run inside `ffai_core::threads::affinity_pool`. #[global_allocator] static GLOBAL: rusty_alloc_api::RustyAlloc = rusty_alloc_api::RustyAlloc; diff --git a/crates/ffai-diana/examples/alloc_ab.rs b/crates/ffai-diana/examples/alloc_ab.rs index 4ef1c307..52785e38 100644 --- a/crates/ffai-diana/examples/alloc_ab.rs +++ b/crates/ffai-diana/examples/alloc_ab.rs @@ -76,11 +76,19 @@ fn main() -> Result<(), Box> { let reps: usize = std::env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(200); let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).parent().and_then(|p| p.parent()).unwrap(); - let img = ffai_media::load_image(&root.join("corpora/clips/diana-coco/coco-032.png"))?; let engine = ffai_diana::engine::Yolo26::build(&tier, ffai_diana::image::Geometry::Rect, root.join("models")); let opts = DetectOptions { confidence: 0.25, ..Default::default() }; + // `alloc_ab corpus`: every `coco-*.png` in the Diana corpus, + // so the allocator sees the spread of image content and detection counts + // rather than one image's allocation pattern repeated. + if std::env::args().nth(3).as_deref() == Some("corpus") { + return corpus(&engine, &opts, &tier, reps, &root.join("corpora/clips/diana-coco")); + } + + let img = ffai_media::load_image(&root.join("corpora/clips/diana-coco/coco-032.png"))?; + // Warm: first call pays lazy weight materialisation and the allocator's // own arena bring-up, neither of which is what we are comparing. let warm = engine.detect(&img, &opts)?; @@ -111,3 +119,49 @@ fn main() -> Result<(), Box> { ); Ok(()) } + +/// Corpus mode: one timed sample is one full PASS over every image, so the +/// sample carries the corpus's whole allocation mix. Images are decoded once, +/// before timing, so the decoder's own allocations are not in the number. +fn corpus( + engine: &ffai_diana::engine::Yolo26, + opts: &DetectOptions, + tier: &str, + passes: usize, + dir: &std::path::Path, +) -> Result<(), Box> { + let mut paths: Vec<_> = std::fs::read_dir(dir)? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.extension().is_some_and(|x| x == "png") + && p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("coco-")) + }) + .collect(); + paths.sort(); + let imgs = paths.iter().map(|p| ffai_media::load_image(p)).collect::, _>>()?; + + // Warm pass, which also takes the work-parity checksum over the corpus. + let (mut n_det, mut sum) = (0usize, 0f64); + for img in &imgs { + for d in &engine.detect(img, opts)?.detections { + n_det += 1; + sum += f64::from(d.x0) + f64::from(d.y0) + f64::from(d.x1) + f64::from(d.y1) + f64::from(d.confidence); + } + } + + let mut times = Vec::with_capacity(passes); + for _ in 0..passes { + let t = Instant::now(); + for img in &imgs { + engine.detect(img, opts)?; + } + times.push(t.elapsed().as_secs_f64() * 1e3); + } + times.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let (min, med) = (times[0], times[times.len() / 2]); + println!( + "ARM={ARM} tier={tier} images={} passes={passes} min={min:.3} med={med:.3} ndet={n_det} sum={sum:.6}", + imgs.len() + ); + Ok(()) +} From 17ced6710c1179cad2ea9199a0f86bfb107d8242 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 17:49:36 -0700 Subject: [PATCH 14/25] docs(carmenta): record the refuted vertical crop pad (fixes a 5x7 D, wrecks every corpus) FFAI_MDET_PAD_Y=0.3 read the blocky 5x7 D correctly and took PdfCraft's 42 pages from 0.25% to 9.93% CER (ruled table 0 -> 55%), carmenta-form 0.41 -> 2.53% (73/240 worse), carmenta-doc 30.1 -> 48.5% (32/32 worse). Default stays 0; the 5x7 D -> O is recorded as a known misread of a blocky font. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/engine.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index c45e4e72..b72fd5a1 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -88,6 +88,13 @@ fn mobiledet_min_side() -> usize { /// Crop pads for the mobile-det path, as fractions of line height. Zero by /// default because DB's unclip has already expanded the box; sweepable because /// "already expanded" is a claim a gate should get to test. +/// +/// Tested 2026-10-30 and refuted: `FFAI_MDET_PAD_Y=0.3` made PdfCraft's +/// blocky 5x7 "HELLO WORLD" read its `D` correctly instead of as `O`, and wrecked +/// everything else — PdfCraft's 42 pages 0.25 % -> 9.93 % CER (its ruled table +/// 0 -> 55 %), `carmenta-form` 0.41 -> 2.53 % (73 of 240 worse), `carmenta-doc` +/// 30.1 -> 48.5 % (all 32 worse). The extra rows are the neighbouring lines. +/// The 5x7 `D` stays a known misread of a blocky font. fn mobiledet_pads() -> (f32, f32) { static V: OnceLock<(f32, f32)> = OnceLock::new(); *V.get_or_init(|| { From 3aa9ab60ca6898fa7498902a38a281cb54c73ca1 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 19:02:20 -0700 Subject: [PATCH 15/25] feat(carmenta): publish the mobiledet-svtr weights; hf_repo in both ppocrv5 manifests PdfCraft finding #12: the document engine's weights had pinned hashes and no host, so no crates.io user could fetch them. They are now public at huggingface.co/remade-with-rust/carmenta-ppocrv5-mobile (first commit 637f0fc): det-fused.safetensors, rec.safetensors and charset.txt, with a model card and the PaddleOCR Apache-2.0 notice. Each file was downloaded back anonymously and matches the sha256 its manifest pins. ppocrv5-mobile-det and -rec now name the repo (crate and repository copies, identical), so `fetch` downloads by file name and verifies as it already does; `offline` users can download the same three files. The staging tool's default repo is the lowercase organisation name; the README gains the host. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/README.md | 1 + crates/ffai-carmenta/models/ppocrv5-mobile-det.toml | 3 +++ crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml | 3 +++ models/ppocrv5-mobile-det.toml | 3 +++ models/ppocrv5-mobile-rec.toml | 3 +++ tools/carmenta_stage_weights.py | 4 ++-- 6 files changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index b819245d..62eba060 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -181,6 +181,7 @@ What else 0.11 adds, each measured: | erase form rules | `OcrOptions::remove_rules` | CRNN on underlined values: 13-20 % → **70 %**. **Not for SVTR**: on strikethroughs it goes 77-87 % → 73 %. Opt-in | | full-width → ASCII on Latin lines | on by default (`FFAI_NO_FOLD=1` disables) | OmniDocBench, 316 pages: fires on 35 of 34 852 lines; **5 pages better, 1 worse** (a full-width comma in the ground truth itself) | | offline weights, fail fast | `CraftCrnn::offline(rec, det, dir)`, `preload()` | no cache, no network; a missing file names its exact path | +| document-engine weights, hosted | `fetch` feature, or download for `offline` | [`remade-with-rust/carmenta-ppocrv5-mobile`](https://huggingface.co/remade-with-rust/carmenta-ppocrv5-mobile): the three files `mobiledet-svtr` loads, each verified against the sha256 pinned in its manifest | Two measured non-results, recorded so nobody has to re-run them: diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml index bba4797e..fc9491c2 100644 --- a/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml @@ -7,6 +7,9 @@ name = "ppocrv5-mobile-det" task = "ocr" description = "PP-OCRv5 mobile det (DBNet/PP-LCNetV3) — staged detector lineage" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "det.safetensors" diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml index c9b07bb1..669064b4 100644 --- a/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml @@ -24,6 +24,9 @@ name = "ppocrv5-mobile-rec" task = "ocr" description = "PP-OCRv5 mobile rec (SVTR, 18385-class CTC) — opt-in via engine name mobiledet-svtr" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "rec.safetensors" diff --git a/models/ppocrv5-mobile-det.toml b/models/ppocrv5-mobile-det.toml index bba4797e..fc9491c2 100644 --- a/models/ppocrv5-mobile-det.toml +++ b/models/ppocrv5-mobile-det.toml @@ -7,6 +7,9 @@ name = "ppocrv5-mobile-det" task = "ocr" description = "PP-OCRv5 mobile det (DBNet/PP-LCNetV3) — staged detector lineage" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "det.safetensors" diff --git a/models/ppocrv5-mobile-rec.toml b/models/ppocrv5-mobile-rec.toml index c9b07bb1..669064b4 100644 --- a/models/ppocrv5-mobile-rec.toml +++ b/models/ppocrv5-mobile-rec.toml @@ -24,6 +24,9 @@ name = "ppocrv5-mobile-rec" task = "ocr" description = "PP-OCRv5 mobile rec (SVTR, 18385-class CTC) — opt-in via engine name mobiledet-svtr" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "rec.safetensors" diff --git a/tools/carmenta_stage_weights.py b/tools/carmenta_stage_weights.py index 3df36efe..82878295 100644 --- a/tools/carmenta_stage_weights.py +++ b/tools/carmenta_stage_weights.py @@ -12,7 +12,7 @@ OUT_DIR gets exactly the files the engine loads, each verified against the crate's embedded manifest, plus a model card and the upstream licence notice. -HF_REPO (default `Remade-With-Rust/carmenta-ppocrv5-mobile`) is only used in the +HF_REPO (default `remade-with-rust/carmenta-ppocrv5-mobile`) is only used in the printed commands and manifest lines. The weights are PaddlePaddle's PP-OCRv5 mobile detector and recognizer @@ -82,7 +82,7 @@ def main(): if len(sys.argv) < 2: sys.exit(__doc__) out = Path(sys.argv[1]) - repo = sys.argv[2] if len(sys.argv) > 2 else "Remade-With-Rust/carmenta-ppocrv5-mobile" + repo = sys.argv[2] if len(sys.argv) > 2 else "remade-with-rust/carmenta-ppocrv5-mobile" out.mkdir(parents=True, exist_ok=True) rows = [] for model, files in FILES.items(): From 9329febf4ae1f2e275c4994e5e568fb95f07e091 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 30 Oct 2026 20:39:40 -0700 Subject: [PATCH 16/25] fix(models): fetch only the files an engine opens, as offline resolution already did The first download from the published PP-OCRv5 repo failed: the cache path called ModelManifest::fetch(), which resolves EVERY declared file, and the detector manifest also lists det.safetensors and det-train.safetensors for provenance -- files no engine opens and the repo does not carry. Offline resolution already asked only for the needed files (resolve_only_in). ModelManifest::fetch_only(needed) is the download-path twin, with the same checksum verification and the same refusal of undeclared names; fetch() is fetch_only(all). Carmenta's resolver uses it for every source. Checked end to end: an empty FFAI_CACHE/HF_HOME, `ocr_text mobiledet-svtr` downloaded exactly det-fused.safetensors, rec.safetensors and charset.txt from remade-with-rust/carmenta-ppocrv5-mobile @ 637f0fc, verified them and read the page correctly. Unit test for the undeclared-name refusal. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/engine.rs | 12 +++++---- crates/ffai-models/src/lib.rs | 43 +++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index b72fd5a1..986008a9 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -542,14 +542,16 @@ struct Resolver<'a> { } impl Resolver<'_> { - /// Resolve the files this loader will open. Offline resolution asks for - /// exactly those, so a deployment never has to ship files a manifest lists - /// only for provenance (a training checkpoint, say). The cache path keeps - /// its existing behaviour of resolving every declared file. + /// Resolve the files this loader will open, and only those, whichever the + /// source: a deployment never has to ship, and a hosted repo never has to + /// carry, files a manifest lists only for provenance (a training + /// checkpoint, say). The cache path used to resolve every declared file, + /// and the first download from the published PP-OCRv5 repo failed on + /// `det.safetensors`, which no engine opens. fn fetch(&self, needed: &[&str]) -> Result { match self.source { Source::Offline(root) => self.manifest.resolve_only_in(root, needed), - _ => self.manifest.fetch(), + _ => self.manifest.fetch_only(needed), } } } diff --git a/crates/ffai-models/src/lib.rs b/crates/ffai-models/src/lib.rs index 97e91be1..3b961be6 100644 --- a/crates/ffai-models/src/lib.rs +++ b/crates/ffai-models/src/lib.rs @@ -154,8 +154,29 @@ impl ModelManifest { /// still network I/O the first time, which is why benchmarks warm the /// cache outside any timed region (see docs/benchmarking.md). pub fn fetch(&self) -> Result { + let all: Vec<&str> = self.files.iter().map(|f| f.name.as_str()).collect(); + self.fetch_only(&all) + } + + /// [`Self::fetch`], for just the named files — the cache-and-download twin + /// of [`Self::resolve_only_in`]. + /// + /// A manifest can list files no engine loads (`ppocrv5-mobile-det` lists + /// its training checkpoint, the source of the fused weights), and a hosted + /// repo need not carry them. Fetching every declared file made the first + /// download of the document engine fail on a file it never opens. A name + /// the manifest does not declare is an error: it has no checksum. + pub fn fetch_only(&self, needed: &[&str]) -> Result { + for name in needed { + if !self.files.iter().any(|f| f.name == *name) { + return Err(Error::Model(format!( + "model `{}` declares no file `{name}` — it has no checksum to verify against", + self.name + ))); + } + } let mut files = BTreeMap::new(); - for file in &self.files { + for file in self.files.iter().filter(|f| needed.contains(&f.name.as_str())) { if let Some(path) = self.local_path(&file.name) { verify_checksum(&path, file)?; files.insert(file.name.clone(), path); @@ -451,4 +472,24 @@ mod tests { assert!(err.contains("declares no file `nope.bin`"), "{err}"); let _ = std::fs::remove_dir_all(&root); } + + /// The download path refuses an undeclared name before touching the + /// network or the cache, exactly as the offline path does. + #[test] + fn fetch_only_refuses_undeclared_names_before_any_lookup() { + let m = ModelManifest::from_toml( + r#" + name = "tiny-test" + task = "ocr" + license = "MIT" + hf_repo = "example/does-not-matter" + + [[files]] + name = "w.bin" + "#, + ) + .unwrap(); + let err = m.fetch_only(&["nope.bin"]).unwrap_err().to_string(); + assert!(err.contains("declares no file `nope.bin`"), "{err}"); + } } From 70f08d0b574a81cd76338d385c36f1171ba486c0 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 02:47:19 -0700 Subject: [PATCH 17/25] chore: allow unsafe_code at each site, so path and git dependents build quietly The crates set `unsafe_code = "warn"`, and a crate pulled in by path or git rev is not lint-capped the way a registry crate is: PdfCraft saw 18 warnings from its pinned build. Each site now carries its own allow, as threads.rs and fastops.rs already did: the SIMD kernels and their dispatch arms in fastmath, conv3x3 and dwconv, the Win32 trim in release_load_arena, and the two mmapped safetensors loads. Allows sit on fns, statements and lets, never on a tail expression. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/conv3x3.rs | 5 +++++ crates/ffai-carmenta/src/dwconv.rs | 2 ++ crates/ffai-carmenta/src/engine.rs | 1 + crates/ffai-carmenta/src/svtr.rs | 1 + crates/ffai-core/src/fastmath.rs | 18 ++++++++++++++++++ crates/ffai-core/src/lib.rs | 1 + 6 files changed, 28 insertions(+) diff --git a/crates/ffai-carmenta/src/conv3x3.rs b/crates/ffai-carmenta/src/conv3x3.rs index 730a5d48..983cfb5a 100644 --- a/crates/ffai-carmenta/src/conv3x3.rs +++ b/crates/ffai-carmenta/src/conv3x3.rs @@ -112,6 +112,7 @@ pub(crate) fn conv3x3_scalar( /// tail. Requires AVX2 + FMA, checked by the caller. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] pub(crate) unsafe fn conv3x3_avx2( in_p: &[f32], w: &[f32], @@ -239,6 +240,7 @@ fn force_scalar() -> bool { /// gives: "differs from candle" says nothing about which is right. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] pub(crate) unsafe fn conv3x3_simd128( in_p: &[f32], w: &[f32], @@ -354,6 +356,7 @@ impl CustomOp2 for Conv3x3Op { // SAFETY: `in_p` is `pad`'s output for exactly these dims, `wt` has // `c_out*c_in*9` elements (the dims4 above plus the caller's 3x3 // guard), and `out` has `c_out*h*w`. + #[allow(unsafe_code)] unsafe { conv3x3_avx2(&in_p, wt, c_in, c_out, h, w, &mut out) }; return Ok((CpuStorage::F32(out), (1, c_out, h, w).into())); } @@ -371,6 +374,7 @@ impl CustomOp2 for Conv3x3Op { // SAFETY: identical contract to the AVX2 twin — `in_p` is `pad`'s // output for exactly these dims, `wt` has `c_out*c_in*9` elements, // and `out` has `c_out*h*wd`. + #[allow(unsafe_code)] unsafe { conv3x3_simd128(&in_p, wt, c_in, c_out, h, w, &mut out) }; return Ok((CpuStorage::F32(out), (1, c_out, h, w).into())); } @@ -473,6 +477,7 @@ mod tests { let mut b = vec![0f32; co * h * w]; conv3x3_scalar(&in_p, &wt, ci, co, h, w, &mut a); // SAFETY: shapes match `pad`'s output exactly. + #[allow(unsafe_code)] unsafe { conv3x3_avx2(&in_p, &wt, ci, co, h, w, &mut b) }; let scale = a.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-9); let err = a.iter().zip(&b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale; diff --git a/crates/ffai-carmenta/src/dwconv.rs b/crates/ffai-carmenta/src/dwconv.rs index 741aaea8..b530e149 100644 --- a/crates/ffai-carmenta/src/dwconv.rs +++ b/crates/ffai-carmenta/src/dwconv.rs @@ -126,6 +126,7 @@ fn plane( #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] #[allow(clippy::too_many_arguments)] +#[allow(unsafe_code)] unsafe fn plane_fma( inp: &[f32], h: usize, @@ -200,6 +201,7 @@ impl CustomOp2 for DwOp { #[cfg(target_arch = "x86_64")] if fma { // SAFETY: `has_fma` confirmed AVX2 and FMA at run time. + #[allow(unsafe_code)] unsafe { plane_fma(x, h, w, k, kh, kw, self.stride, self.pad, o, wo) }; return; } diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index 986008a9..70c6508b 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -465,6 +465,7 @@ fn load_models(source: &Source, rec: RecStage, det: DetStage) -> Result .map(|m| Resolver { manifest: m, source }) .ok_or_else(|| Error::Model(format!("no model manifest named `{name}` in {searched}"))) }; + #[allow(unsafe_code)] let load_vb = |file: std::path::PathBuf| { unsafe { VarBuilder::from_mmaped_safetensors(&[file], DType::F32, &device) } .map_err(image::candle_err) diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index 3de88af9..59559c76 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -852,6 +852,7 @@ use std::sync::{PoisonError, RwLock}; /// Load from a safetensors file produced by `tools/carmenta_svtr_prepare.py`. pub fn load(path: &std::path::Path, device: &Device, n_class: usize) -> Result { + #[allow(unsafe_code)] let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[path], DType::F32, device)? }; Svtr::new(vb, n_class) } diff --git a/crates/ffai-core/src/fastmath.rs b/crates/ffai-core/src/fastmath.rs index c85026d4..7085692e 100644 --- a/crates/ffai-core/src/fastmath.rs +++ b/crates/ffai-core/src/fastmath.rs @@ -854,6 +854,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { { if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { // SAFETY: both features probed immediately above. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_avx2(row, max) }; } } @@ -867,6 +868,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { // stops LLVM cold no matter what the baseline is. // // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_neon(row, max) }; } #[cfg(target_arch = "wasm32")] @@ -879,6 +881,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { // requirement rather than an upgrade. // // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_simd128(row, max) }; } #[allow(unreachable_code)] @@ -910,6 +913,7 @@ pub fn exp_sub_sum_scalar(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_avx2(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1001,6 +1005,7 @@ unsafe fn exp_sub_sum_avx2(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_neon(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1080,6 +1085,7 @@ unsafe fn exp_sub_sum_neon(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_simd128(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1155,17 +1161,20 @@ pub fn max_f32(xs: &[f32]) -> f32 { { if is_x86_feature_detected!("avx2") { // SAFETY: probed immediately above. + #[allow(unsafe_code)] return unsafe { max_f32_avx2(xs) }; } } #[cfg(target_arch = "aarch64")] { // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { max_f32_neon(xs) }; } #[cfg(target_arch = "wasm32")] { // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { max_f32_simd128(xs) }; } #[allow(unreachable_code)] @@ -1186,6 +1195,7 @@ pub fn max_f32_scalar(xs: &[f32]) -> f32 { /// Caller must have verified `avx2`. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] +#[allow(unsafe_code)] unsafe fn max_f32_avx2(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1222,6 +1232,7 @@ unsafe fn max_f32_avx2(xs: &[f32]) -> f32 { /// `neon` is a compile-time guarantee on `aarch64` here. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn max_f32_neon(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1251,6 +1262,7 @@ unsafe fn max_f32_neon(xs: &[f32]) -> f32 { /// `simd128` is a compile-time guarantee on `wasm32` here. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn max_f32_simd128(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1297,6 +1309,7 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { { if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { // SAFETY: probed immediately above. + #[allow(unsafe_code)] unsafe { gelu_tanh_avx2(xs) }; return; } @@ -1304,12 +1317,14 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { #[cfg(target_arch = "aarch64")] { // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] unsafe { gelu_tanh_neon(xs) }; return; } #[cfg(target_arch = "wasm32")] { // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] unsafe { gelu_tanh_simd128(xs) }; return; } @@ -1335,6 +1350,7 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_avx2(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1412,6 +1428,7 @@ unsafe fn gelu_tanh_avx2(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_neon(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1482,6 +1499,7 @@ unsafe fn gelu_tanh_neon(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_simd128(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: diff --git a/crates/ffai-core/src/lib.rs b/crates/ffai-core/src/lib.rs index d85983f1..017f6932 100644 --- a/crates/ffai-core/src/lib.rs +++ b/crates/ffai-core/src/lib.rs @@ -72,6 +72,7 @@ pub const fn best_device() -> candle::Device { // Linux and break on Windows - so the lint is allowed rather than obeyed. // Caught only when CI first ran on Linux; it never fired on the author's box. #[allow(clippy::missing_const_for_fn)] +#[allow(unsafe_code)] pub fn release_load_arena() { #[cfg(windows)] { From 65052a3e6f86c24f13438ed7566577c9c699fd19 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 02:47:37 -0700 Subject: [PATCH 18/25] feat(core)!: OcrOutput reports the turn auto_orient applied; rotated_corners takes it Boxes come back in the input's coordinates, so a page read at 90 and one read at 270 degrees gave the same boxes and the reading direction was lost. `OcrOutput::turns` (0-3 clockwise quarter turns) now says which, set when the turned read is kept. With auto_orient and deskew both on, `skew_degrees` is measured on the upright page while the boxes are mapped back to the input, so `rotated_corners` met a tall box with a near-horizontal angle and clamped the line's width to 0. It now takes `turns`: on the input the rectangle runs at `degrees - 90 * turns`, which the bounds solve handles as it is. Tested for turns 1-3 at 2 degrees against the exact point map. auto_orient also compares the two reads now: the turned one is kept only when its page confidence beats the first, so a low-confidence upright page the sampled decision misjudged keeps its own read. PdfCraft's 42 pages read byte-identically (text and boxes) with orient and deskew on. Findings 1-3 from PdfCraft's rerun. BREAKING CHANGE: `OcrOutput` gains `turns` and, with the unreleased `skew_degrees` and `OcrOptions::deskew`, breaks struct literals and exhaustive destructuring downstream; `deskew::rotated_corners` takes a third argument. Not `#[non_exhaustive]`: that would forbid the `..Default::default()` construction callers use. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/deskew.rs | 59 ++++++++++++++++++++++++++---- crates/ffai-carmenta/src/engine.rs | 11 +++++- crates/ffai-carmenta/src/forms.rs | 1 + crates/ffai-carmenta/src/live.rs | 1 + crates/ffai-core/src/types.rs | 12 ++++++ 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/crates/ffai-carmenta/src/deskew.rs b/crates/ffai-carmenta/src/deskew.rs index 76005f33..a3620c70 100644 --- a/crates/ffai-carmenta/src/deskew.rs +++ b/crates/ffai-carmenta/src/deskew.rs @@ -19,7 +19,8 @@ //! turned by the skew, and the reported box is its axis-aligned bounds. //! `OcrOutput::skew_degrees` carries the angle, and [`rotated_corners`] //! recovers the turned rectangle from a reported box, which is what a text -//! layer drawn along the page's slope needs. +//! layer drawn along the page's slope needs. With `auto_orient` the skew is +//! measured on the upright page, so it also takes `OcrOutput::turns`. use ffai_core::types::{BoundingBox, ImageBuffer}; @@ -148,16 +149,20 @@ fn paper_colour(img: &ImageBuffer) -> [u8; 4] { } /// The four corners, on the input page, of the turned rectangle whose -/// axis-aligned bounds are `b`, for a page skewed `degrees` (as -/// `OcrOutput::skew_degrees` reports it). In order: top-left, top-right, -/// bottom-right, bottom-left of the straightened rectangle. +/// axis-aligned bounds are `b`, for a page skewed `degrees` and turned +/// upright by `turns` (as `OcrOutput::skew_degrees` and `OcrOutput::turns` +/// report them). In order: top-left, top-right, bottom-right, bottom-left of +/// the rectangle as it reads upright. /// /// A `W x H` rectangle turned by `a` has bounds `w = W|cos a| + H|sin a|` and /// `h = W|sin a| + H|cos a|`, which this solves for `W` and `H`; the centre is -/// shared. Exact up to the clamping at the page edge. +/// shared. Exact up to the clamping at the page edge. On the input page the +/// upright rectangle runs at `degrees - 90 * turns`: the skew is measured on +/// the upright page, and the input is that page turned back counterclockwise. #[must_use] -pub fn rotated_corners(b: BoundingBox, degrees: f32) -> [(f32, f32); 4] { - let (s, c) = degrees.to_radians().sin_cos(); +pub fn rotated_corners(b: BoundingBox, degrees: f32, turns: u8) -> [(f32, f32); 4] { + let a = degrees - 90.0 * f32::from(turns % 4); + let (s, c) = a.to_radians().sin_cos(); let (sa, ca) = (s.abs(), c.abs()); let det = ca * ca - sa * sa; let (cx, cy) = (b.x + b.width / 2.0, b.y + b.height / 2.0); @@ -186,7 +191,7 @@ mod tests { // A line on the straightened page, mapped back and recovered. let line = BoundingBox { x: 400.0, y: 900.0, width: 1200.0, height: 40.0 }; let on_input = skew.back_box(line, 2000.0, 3000.0); - let corners = rotated_corners(on_input, 2.0); + let corners = rotated_corners(on_input, 2.0, 0); let want = [ (line.x, line.y), (line.x + line.width, line.y), @@ -199,6 +204,44 @@ mod tests { } } + /// `auto_orient` and `deskew` together: the page is turned upright, + /// straightened there, and its boxes reach the caller unrotated (tall, for + /// an odd turn) while the angle stays the upright page's. Without `turns` + /// the solve met a tall box with a near-horizontal angle and clamped the + /// line's width to 0. + #[test] + fn rotated_corners_follows_a_turned_page() { + let (w, h) = (800u32, 1200u32); + let img = ImageBuffer { width: w, height: h, format: PixelFormat::Gray8, data: vec![255; (w * h) as usize] }; + let (wf, hf) = (w as f32, h as f32); + for turns in 1..4u8 { + let upright = crate::orient::rotate(&img, turns); + let (uw, uh) = (upright.width as f32, upright.height as f32); + let (straight, skew) = straighten(&upright, 2.0); + let (sw, sh) = (straight.width as f32, straight.height as f32); + let line = BoundingBox { x: 0.15 * sw, y: 0.4 * sh, width: 0.6 * sw, height: 30.0 }; + // A point of the straightened page, on the input: a zero-size box + // maps exactly, boxes being edge coordinates. + let to_input = |p: (f32, f32)| { + let (x, y) = skew.back(p); + let b = crate::orient::unrotate_box(BoundingBox { x, y, width: 0.0, height: 0.0 }, turns, wf, hf); + (b.x, b.y) + }; + let on_input = crate::orient::unrotate_box(skew.back_box(line, uw, uh), turns, wf, hf); + let corners = rotated_corners(on_input, 2.0, turns); + let want = [ + (line.x, line.y), + (line.x + line.width, line.y), + (line.x + line.width, line.y + line.height), + (line.x, line.y + line.height), + ] + .map(to_input); + for (g, w) in corners.iter().zip(&want) { + assert!((g.0 - w.0).abs() < 0.05 && (g.1 - w.1).abs() < 0.05, "turns {turns}: {g:?} vs {w:?}"); + } + } + } + #[test] fn straighten_keeps_the_paper_colour() { let img = ImageBuffer { width: 64, height: 32, format: PixelFormat::Rgb8, data: vec![200; 64 * 32 * 3] }; diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index 70c6508b..0979cab8 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -824,6 +824,13 @@ impl OcrEngine for CraftCrnn { } let upright = crate::orient::rotate(img, o.turns); let mut out = self.recognize(&upright, &inner)?; + // The decision samples two dozen crops; the two full reads are the + // better witness. A low-confidence upright page the sample misjudged + // keeps its own read. (`None`, no text, orders below any `Some`.) + if page_confidence(&out) <= page_confidence(&first) { + return Ok(first); + } + out.turns = o.turns; let (w, h) = (img.width as f32, img.height as f32); for line in out.blocks.iter_mut().flat_map(|b| b.lines.iter_mut()) { if let Some(b) = line.bbox { @@ -941,7 +948,7 @@ impl OcrEngine for CraftCrnn { confidence, }); } - return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], skew_degrees: None }); + return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], skew_degrees: None, turns: 0 }); } // ---- detect ---- @@ -1495,7 +1502,7 @@ impl OcrEngine for CraftCrnn { // v1: one block per page — paragraph segmentation is the DOCUMENT // milestone's work, and inventing it early would be unearned. - Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }], skew_degrees: None }) + Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }], skew_degrees: None, turns: 0 }) } } diff --git a/crates/ffai-carmenta/src/forms.rs b/crates/ffai-carmenta/src/forms.rs index 5649ad54..2464b851 100644 --- a/crates/ffai-carmenta/src/forms.rs +++ b/crates/ffai-carmenta/src/forms.rs @@ -154,6 +154,7 @@ mod tests { OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], skew_degrees: None, + turns: 0, } } diff --git a/crates/ffai-carmenta/src/live.rs b/crates/ffai-carmenta/src/live.rs index 2670af28..81fdb629 100644 --- a/crates/ffai-carmenta/src/live.rs +++ b/crates/ffai-carmenta/src/live.rs @@ -437,6 +437,7 @@ impl LiveSession { OcrOutput { blocks: self.bands.iter().flat_map(|b| b.cached.clone()).collect(), skew_degrees: None, + turns: 0, } } else { let full = self.engine.recognize(img, &self.opts)?; diff --git a/crates/ffai-core/src/types.rs b/crates/ffai-core/src/types.rs index 7fd33f89..019179d9 100644 --- a/crates/ffai-core/src/types.rs +++ b/crates/ffai-core/src/types.rs @@ -364,7 +364,18 @@ pub struct OcrOutput { /// rectangle turned by this angle, so the rectangle itself — what a text /// layer drawn along the page's slope wants — is recoverable from the box /// and the angle (`ffai_carmenta::deskew::rotated_corners`). + /// + /// On a page also turned upright (`turns` non-zero) the angle is measured + /// on the upright page, not on the input. pub skew_degrees: Option, + /// Clockwise quarter turns (0-3) the engine applied to the input to read + /// it upright (`OcrOptions::auto_orient`); 0 when it was read as given. + /// + /// Boxes stay in the INPUT image's coordinates either way, so this is + /// what gives each box its reading direction: at 1 turn the text in a box + /// runs bottom to top, at 2 right to left upside down, at 3 top to + /// bottom. A 90-degree and a 270-degree page give the same boxes. + pub turns: u8, } impl OcrOutput { @@ -399,6 +410,7 @@ impl OcrOutput { bbox: None, }], skew_degrees: None, + turns: 0, } } From 16813c28d0c22ce20c0c999132bc0a72749ffbe8 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 02:47:38 -0700 Subject: [PATCH 19/25] fix(carmenta): a formula splice keeps the word boxes it does not touch route.rs cleared a line's words whenever an inline formula was spliced into its text, which dropped every box on the line for one token. Now the words wholly outside the splice keep theirs, and the words it touches become one word holding the spliced token, boxed by their union. Words that cannot be found in the text in order are still cleared. It happened on 0 of 1,019 lines in PdfCraft's corpus, but form routing can reach it. Co-Authored-By: Claude Opus 5.5 --- crates/ffai-carmenta/src/route.rs | 101 +++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 7 deletions(-) diff --git a/crates/ffai-carmenta/src/route.rs b/crates/ffai-carmenta/src/route.rs index 3c6588e8..29f2dc0d 100644 --- a/crates/ffai-carmenta/src/route.rs +++ b/crates/ffai-carmenta/src/route.rs @@ -30,7 +30,7 @@ use crate::doclayout::{DocLayout, Region}; use crate::formula::FormulaModel; use crate::table::TableModel; use ffai_core::error::Result; -use ffai_core::types::{BoundingBox, ImageBuffer, OcrLine}; +use ffai_core::types::{BoundingBox, ImageBuffer, OcrLine, OcrWord}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; @@ -191,6 +191,47 @@ fn crop(img: &ImageBuffer, x0: usize, y0: usize, x1: usize, y1: usize) -> ImageB ImageBuffer { width: cw as u32, height: ch as u32, format: img.format, data } } +/// A line's word boxes after its `text[b0..b1]` was spliced into `new_text`. +/// Words wholly outside the splice keep their boxes; the words it touches +/// become one word holding the spliced token, boxed by their union. Empty when +/// the words cannot be found in `text` in order, or none touches the splice: +/// a box that no longer matches its text is worse than no box. +fn splice_words(words: &[OcrWord], text: &str, b0: usize, b1: usize, new_text: &str) -> Vec { + let (mut kept, mut touched) = (Vec::with_capacity(words.len()), Vec::new()); + let mut merged_at = None; + let (mut s, mut e, mut cursor) = (b0, b1, 0); + for w in words { + let Some(at) = text.get(cursor..).and_then(|t| t.find(&w.text)) else { + return Vec::new(); + }; + let (w0, w1) = (cursor + at, cursor + at + w.text.len()); + cursor = w1; + if w1 <= b0 || w0 >= b1 { + kept.push(w.clone()); + } else { + merged_at.get_or_insert(kept.len()); + (s, e) = (s.min(w0), e.max(w1)); + touched.push(w); + } + } + let Some(i) = merged_at else { + return Vec::new(); + }; + // `e` is an offset into `text`; the splice moved everything after `b1`. + let end = (e + new_text.len()).checked_sub(text.len()); + let Some(token) = end.and_then(|end| new_text.get(s..end)) else { + return Vec::new(); + }; + let bbox = touched.iter().filter_map(|w| w.bbox).reduce(|a, b| { + let (x0, y0) = (a.x.min(b.x), a.y.min(b.y)); + let (x1, y1) = ((a.x + a.width).max(b.x + b.width), (a.y + a.height).max(b.y + b.height)); + BoundingBox { x: x0, y: y0, width: x1 - x0, height: y1 - y0 } + }); + let confidence = touched.iter().filter_map(|w| w.confidence).reduce(f32::min); + kept.insert(i, OcrWord { text: token.to_string(), bbox, confidence }); + kept +} + /// Fraction of `l` that lies inside `r`. fn covered(l: &BoundingBox, r: &Region) -> f32 { let (lx1, ly1) = (l.x + l.width, l.y + l.height); @@ -450,10 +491,9 @@ pub fn apply( g.score, g.x0, g.y0, rw, rh, latex.len() ); } + // Only the words the splice touched lose their own boxes. + out[hi].words = splice_words(&out[hi].words, &out[hi].text, b0, b1, &t); out[hi].text = t; - // The splice changes the text, so the decoder's word boxes no - // longer describe it. - out[hi].words.clear(); spliced.push(g as *const Region); } } @@ -611,10 +651,9 @@ pub fn apply( latex.len() ); } + // Only the words the splice touched lose their own boxes. + out[hi].words = splice_words(&out[hi].words, &out[hi].text, b0, b1, &t); out[hi].text = t; - // The splice changes the text, so the decoder's word boxes no - // longer describe it. - out[hi].words.clear(); } if dbg { eprintln!( @@ -713,3 +752,51 @@ fn bind_cells( } texts } + +#[cfg(test)] +mod tests { + use super::*; + + fn word(text: &str, x: f32) -> OcrWord { + OcrWord { + text: text.into(), + bbox: Some(BoundingBox { x, y: 10.0, width: 20.0, height: 12.0 }), + confidence: Some(0.9), + } + } + + #[test] + fn splice_keeps_the_words_it_does_not_touch() { + let text = "area x2+y2 of the disc"; + let words = [word("area", 0.0), word("x2+y2", 30.0), word("of", 60.0), word("the", 90.0), word("disc", 120.0)]; + // "2+y" (bytes 6..9) becomes LaTeX, inside the second word. + let mut t = text.to_string(); + t.replace_range(6..9, "$2+y$"); + let got = splice_words(&words, text, 6, 9, &t); + let texts: Vec<&str> = got.iter().map(|w| w.text.as_str()).collect(); + assert_eq!(texts, ["area", "x$2+y$2", "of", "the", "disc"]); + assert_eq!(got[0], words[0]); + assert_eq!(got[4], words[4]); + assert_eq!(got[1].bbox, words[1].bbox); + } + + #[test] + fn splice_over_several_words_merges_their_boxes() { + let text = "so a + b = c holds"; + let words = [word("so", 0.0), word("a", 30.0), word("+", 60.0), word("b", 90.0), word("=", 120.0), word("c", 150.0), word("holds", 180.0)]; + let mut t = text.to_string(); + t.replace_range(3..12, "$a+b=c$"); + let got = splice_words(&words, text, 3, 12, &t); + let texts: Vec<&str> = got.iter().map(|w| w.text.as_str()).collect(); + assert_eq!(texts, ["so", "$a+b=c$", "holds"]); + let b = got[1].bbox.unwrap(); + assert!((b.x - 30.0).abs() < 1e-6 && (b.width - 140.0).abs() < 1e-6, "{b:?}"); + } + + #[test] + fn splice_clears_words_that_do_not_match_the_text() { + let text = "alpha beta"; + let words = [word("alpha", 0.0), word("gamma", 30.0)]; + assert!(splice_words(&words, text, 0, 3, "$al$pha beta").is_empty()); + } +} From 0c7236418501ac27deaa14dd7758c6bacee04999 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 02:47:38 -0700 Subject: [PATCH 20/25] fix(bench): pdfcraft_bench reads the affinity mask after the load A harness that pins the process after spawning it (pinvs does) can lose a race against the top of main, and BENCH_POOL=1 then built a pool for the whole machine. The mask is read after preload(), and the work row prints the pool the pages actually ran on. Co-Authored-By: Claude Opus 5.5 --- .../ffai-carmenta/examples/pdfcraft_bench.rs | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs index df92890c..7840be3c 100644 --- a/crates/ffai-carmenta/examples/pdfcraft_bench.rs +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -39,16 +39,6 @@ fn esc(s: &str) -> String { } fn main() { - eprintln!("available_threads {}", ffai_core::threads::available_threads()); - if std::env::var("BENCH_POOL").is_ok_and(|v| v == "1") { - let pool = ffai_core::threads::affinity_pool().expect("thread pool"); - pool.install(run); - } else { - run(); - } -} - -fn run() { let mut args = std::env::args().skip(1); let root = args.next().expect("usage: pdfcraft_bench ..."); let pages: Vec = args.collect(); @@ -60,6 +50,19 @@ fn run() { let t = Instant::now(); engine.preload().expect("preload"); eprintln!("load {:.1} ms", t.elapsed().as_secs_f64() * 1e3); + // The mask is read only now, after the load: a harness that pins the + // process after spawning it (pinvs does) can lose a race against the top + // of `main`, and the pool would be built for the whole machine. + eprintln!("available_threads {}", ffai_core::threads::available_threads()); + if std::env::var("BENCH_POOL").is_ok_and(|v| v == "1") { + let pool = ffai_core::threads::affinity_pool().expect("thread pool"); + pool.install(|| run(&engine, &pages)); + } else { + run(&engine, &pages); + } +} + +fn run(engine: &CraftCrnn, pages: &[String]) { let opts = OcrOptions { auto_orient: std::env::var("BENCH_ORIENT").is_ok_and(|v| v == "1"), deskew: std::env::var("BENCH_DESKEW").is_ok_and(|v| v == "1"), @@ -70,7 +73,7 @@ fn run() { }; let all = Instant::now(); let (mut words_total, mut lines_with_words, mut lines_total) = (0usize, 0usize, 0usize); - for p in &pages { + for p in pages { let img = ffai_media::load_image(Path::new(p)).expect("load image"); let t = Instant::now(); let out = engine.recognize(&img, &opts).expect("recognize"); @@ -116,10 +119,12 @@ fn run() { s.push_str("]}"); println!("{s}"); } + // The pool the pages actually ran on, beside the time it produced. eprintln!( - "pages {} total {:.1} ms; lines {lines_total}, with words {lines_with_words}, words {words_total}", + "pages {} total {:.1} ms on {} threads; lines {lines_total}, with words {lines_with_words}, words {words_total}", pages.len(), - all.elapsed().as_secs_f64() * 1e3 + all.elapsed().as_secs_f64() * 1e3, + rayon::current_num_threads() ); if std::env::var_os("FFAI_PROFILE").is_some() { eprint!("{}", ffai_carmenta::profile::profile().report()); From ac5062b82614c8ebcf946d4a9cc70e2d1e6ae274 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 07:45:49 -0700 Subject: [PATCH 21/25] fix(deps): rustls 0.23.46, rustls-webpki 0.103.16, webpki-root-certs 1.0.11; dated zip ignores cargo-deny started failing on master and here on nine new advisories. Six have semver-compatible fixes and are taken in the lockfile (RUSTSEC-2026-0346, -0347, -0348, -0349 in rustls; -0382 in rustls-webpki; -0383 in webpki-root-certs). The other three (RUSTSEC-2026-0367, -0368, -0369) are zip 8.6 extraction bugs fixed only in zip 9, which candle-core 0.11 cannot take (it pins ^8.6). They live in ZipArchive::extract / ZipStreamReader::extract; candle opens archives and reads entries by name and never extracts, and nothing here depends on zip directly. Ignored with a dated, scoped justification and a 2026-11-15 review, per deny.toml's policy. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 14 +++++++------- deny.toml | 9 +++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43aca379..2db864cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3865,9 +3865,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -3930,9 +3930,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4553,7 +4553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -5370,9 +5370,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] diff --git a/deny.toml b/deny.toml index b5296f7e..bd3fc0bc 100644 --- a/deny.toml +++ b/deny.toml @@ -25,6 +25,15 @@ ignore = [ # Not present in any shipped crate's normal dependency graph. # Review 2026-11-15. Replacement candidate: skrifa. "RUSTSEC-2026-0192", + # zip 8.6.0 - three extraction advisories: SUID/SGID bits preserved, symlink + # targets outside the destination, permissions applied through symlinks. Fixed + # only in zip 9, and candle-core 0.11 pins `^8.6` (npy/pickle). All three are in + # `ZipArchive::extract` / `ZipStreamReader::extract`; candle opens archives and + # reads entries by name and never extracts, and no crate here depends on zip + # directly. Added 2026-10-31. Review 2026-11-15, or when candle moves to zip 9. + "RUSTSEC-2026-0367", + "RUSTSEC-2026-0368", + "RUSTSEC-2026-0369", # RESOLVED 2026-08-19, ignores REMOVED rather than left to rot. pyo3 went # 0.23.5 -> 0.29.2, so RUSTSEC-2025-0020 (PyString::from_object buffer # overflow) and RUSTSEC-2026-0177 (missing Sync bound on From ee628f93e5d239723d286040a496b3d84bede46f Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 09:10:31 -0700 Subject: [PATCH 22/25] chore(release): ffai-core 0.9.0, ffai-carmenta 0.13.0, and every dependent past crates.io ffai-core's OcrOptions/OcrOutput gained fields (deskew, skew_degrees, turns), a breaking change, so each published crate that depends on it gets a new version requiring core 0.9. Republishing only core and carmenta would leave a caller with, for example, ffai-media built on core 0.8 beside carmenta on core 0.9: two ImageBuffer types that do not unify. ffai-core 0.9.0, ffai-carmenta 0.13.0, ffai-models 0.7.0, ffai-media 0.7.0, ffai-mercury 1.1.0, ffai-diana 0.9.0, ffai-argus 0.9.0 (workspace.package, with ffai-demo), ffai-bench 0.8.0, ffai-cli 0.9.0, ffai-argus-wasm 0.3.0, ffai-carmenta-wasm 0.4.0, ffai-mercury-wasm 0.3.0, ffai-wasm 0.3.0 Each crate's CHANGELOG has the entry, and the install snippets in the argus, carmenta, mercury, media and diana READMEs name the new versions (mercury's still said 0.7). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 28 +++++++++++++------------- Cargo.toml | 18 ++++++++--------- crates/ffai-argus-wasm/CHANGELOG.md | 6 ++++++ crates/ffai-argus-wasm/Cargo.toml | 4 ++-- crates/ffai-argus/CHANGELOG.md | 6 ++++++ crates/ffai-argus/Cargo.toml | 2 +- crates/ffai-argus/README.md | 4 ++-- crates/ffai-bench/CHANGELOG.md | 6 ++++++ crates/ffai-bench/Cargo.toml | 2 +- crates/ffai-carmenta-wasm/CHANGELOG.md | 6 ++++++ crates/ffai-carmenta-wasm/Cargo.toml | 4 ++-- crates/ffai-carmenta/CHANGELOG.md | 22 ++++++++++++++++++++ crates/ffai-carmenta/Cargo.toml | 4 ++-- crates/ffai-carmenta/README.md | 6 +++--- crates/ffai-cli/CHANGELOG.md | 6 ++++++ crates/ffai-cli/Cargo.toml | 2 +- crates/ffai-core/CHANGELOG.md | 13 ++++++++++++ crates/ffai-core/Cargo.toml | 2 +- crates/ffai-diana/CHANGELOG.md | 6 ++++++ crates/ffai-diana/Cargo.toml | 4 ++-- crates/ffai-diana/README.md | 6 +++--- crates/ffai-media/CHANGELOG.md | 6 ++++++ crates/ffai-media/Cargo.toml | 2 +- crates/ffai-media/README.md | 2 +- crates/ffai-mercury-wasm/CHANGELOG.md | 6 ++++++ crates/ffai-mercury-wasm/Cargo.toml | 4 ++-- crates/ffai-mercury/CHANGELOG.md | 6 ++++++ crates/ffai-mercury/Cargo.toml | 4 ++-- crates/ffai-mercury/README.md | 8 ++++---- crates/ffai-models/CHANGELOG.md | 15 ++++++++++++++ crates/ffai-models/Cargo.toml | 2 +- crates/ffai-wasm/CHANGELOG.md | 6 ++++++ crates/ffai-wasm/Cargo.toml | 2 +- 33 files changed, 165 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2db864cf..f4dbf991 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1215,7 +1215,7 @@ dependencies = [ [[package]] name = "ffai-argus" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "candle-nn", @@ -1233,7 +1233,7 @@ dependencies = [ [[package]] name = "ffai-argus-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-argus", "ffai-core", @@ -1245,7 +1245,7 @@ dependencies = [ [[package]] name = "ffai-bench" -version = "0.7.7" +version = "0.8.0" dependencies = [ "clap", "csv", @@ -1263,7 +1263,7 @@ dependencies = [ [[package]] name = "ffai-carmenta" -version = "0.12.0" +version = "0.13.0" dependencies = [ "candle-core", "candle-nn", @@ -1281,7 +1281,7 @@ dependencies = [ [[package]] name = "ffai-carmenta-wasm" -version = "0.3.0" +version = "0.4.0" dependencies = [ "ffai-carmenta", "ffai-core", @@ -1296,7 +1296,7 @@ dependencies = [ [[package]] name = "ffai-cli" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "clap", @@ -1315,7 +1315,7 @@ dependencies = [ [[package]] name = "ffai-core" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "rayon", @@ -1325,7 +1325,7 @@ dependencies = [ [[package]] name = "ffai-demo" -version = "0.8.0" +version = "0.9.0" dependencies = [ "ffai-argus", "ffai-bench", @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "ffai-diana" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "candle-nn", @@ -1358,7 +1358,7 @@ dependencies = [ [[package]] name = "ffai-media" -version = "0.6.8" +version = "0.7.0" dependencies = [ "fax", "ffai-core", @@ -1380,7 +1380,7 @@ dependencies = [ [[package]] name = "ffai-mercury" -version = "1.0.5" +version = "1.1.0" dependencies = [ "candle-nn", "candle-transformers", @@ -1399,7 +1399,7 @@ dependencies = [ [[package]] name = "ffai-mercury-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-core", "ffai-mercury", @@ -1410,7 +1410,7 @@ dependencies = [ [[package]] name = "ffai-models" -version = "0.6.6" +version = "0.7.0" dependencies = [ "ffai-core", "hf-hub", @@ -1432,7 +1432,7 @@ dependencies = [ [[package]] name = "ffai-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-core", "ffai-diana", diff --git a/Cargo.toml b/Cargo.toml index 93d43aff..4b9e253c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = ["crates/*"] exclude = ["demo-ui"] [workspace.package] -version = "0.8.0" +version = "0.9.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" @@ -16,14 +16,14 @@ authors = ["Remade With Rust"] [workspace.dependencies] # ---- internal crates ---- -ffai-core = { path = "crates/ffai-core", version = "0.8.0" } -ffai-media = { path = "crates/ffai-media", version = "0.6.8" } -ffai-models = { path = "crates/ffai-models", version = "0.6.6" } -ffai-mercury = { path = "crates/ffai-mercury", version = "1.0.5" } -ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.12.0" } -ffai-diana = { path = "crates/ffai-diana", version = "0.8.0" } -ffai-argus = { path = "crates/ffai-argus", version = "0.8.0" } -ffai-bench = { path = "crates/ffai-bench", version = "0.7.7" } +ffai-core = { path = "crates/ffai-core", version = "0.9.0" } +ffai-media = { path = "crates/ffai-media", version = "0.7.0" } +ffai-models = { path = "crates/ffai-models", version = "0.7.0" } +ffai-mercury = { path = "crates/ffai-mercury", version = "1.1.0" } +ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.13.0" } +ffai-diana = { path = "crates/ffai-diana", version = "0.9.0" } +ffai-argus = { path = "crates/ffai-argus", version = "0.9.0" } +ffai-bench = { path = "crates/ffai-bench", version = "0.8.0" } # ---- tensor spine ---- # Candle is FFai's PyTorch: every engine shares its Tensor/Device types so diff --git a/crates/ffai-argus-wasm/CHANGELOG.md b/crates/ffai-argus-wasm/CHANGELOG.md index 3dcc3ec5..1e0cb0c3 100644 --- a/crates/ffai-argus-wasm/CHANGELOG.md +++ b/crates/ffai-argus-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-wasm-v0.2.0...ffai-argus-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-wasm-v0.1.3...ffai-argus-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-argus-wasm/Cargo.toml b/crates/ffai-argus-wasm/Cargo.toml index eb94129e..560a0d70 100644 --- a/crates/ffai-argus-wasm/Cargo.toml +++ b/crates/ffai-argus-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-argus-wasm" description = "Argus in the browser — pure-Rust SmolVLM image captioning compiled to WebAssembly, no ONNX runtime, no Python" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -25,7 +25,7 @@ ffai-core = { workspace = true } # **0.7.2, not 0.7.1** — see the note in `ffai-mercury-wasm`. `SmolVlm::from_bytes` # and `ArgusBytes` first exist in 0.7.2, and a tarball verification resolves # this from crates.io rather than from the path. -ffai-argus = { path = "../ffai-argus", version = "0.8.0", default-features = false } +ffai-argus = { path = "../ffai-argus", version = "0.9.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-argus/CHANGELOG.md b/crates/ffai-argus/CHANGELOG.md index b7d1e9ab..3afa0e36 100644 --- a/crates/ffai-argus/CHANGELOG.md +++ b/crates/ffai-argus/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-v0.8.0...ffai-argus-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-v0.7.5...ffai-argus-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-argus/Cargo.toml b/crates/ffai-argus/Cargo.toml index 53b4ce57..3b673e24 100644 --- a/crates/ffai-argus/Cargo.toml +++ b/crates/ffai-argus/Cargo.toml @@ -19,7 +19,7 @@ ffai-core = { workspace = true } # their own weights; they must not inherit fetch just because Argus is linked. # Not `workspace = true`: cargo forbids overriding default-features on an # inherited dep. -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # The tokenizer is the ENGINE's business now, not just the step-4 test's: the # assembled prompt has to become ids and the generated ids have to become text. tokenizers = { workspace = true } diff --git a/crates/ffai-argus/README.md b/crates/ffai-argus/README.md index b6357dc9..41d22d87 100644 --- a/crates/ffai-argus/README.md +++ b/crates/ffai-argus/README.md @@ -407,9 +407,9 @@ or in `Cargo.toml`: ```toml [dependencies] -ffai-argus = "0.6" # the engine +ffai-argus = "0.9" # the engine ffai-core = "0.7" # the VlmEngine trait, the registry, the cost counters -ffai-media = "0.6" # decoding an image/video file is the CALLER's job +ffai-media = "0.7" # decoding an image/video file is the CALLER's job ``` `ffai-media` is separate on purpose: the engine takes an already-decoded diff --git a/crates/ffai-bench/CHANGELOG.md b/crates/ffai-bench/CHANGELOG.md index 38e26aa8..fa9f84a4 100644 --- a/crates/ffai-bench/CHANGELOG.md +++ b/crates/ffai-bench/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-bench-v0.7.7...ffai-bench-v0.8.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.7.7](https://github.com/Remade-With-Rust/FFAI/compare/ffai-bench-v0.7.6...ffai-bench-v0.7.7) - 2026-09-25 ### Added diff --git a/crates/ffai-bench/Cargo.toml b/crates/ffai-bench/Cargo.toml index bc42549c..6e3847de 100644 --- a/crates/ffai-bench/Cargo.toml +++ b/crates/ffai-bench/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai's analyzer — one call to compare any engine against world # the runner records a clip failure and continues instead of aborting the # whole corpus, and decode is wrapped so a panicking dependency costs one # page rather than the run. -version = "0.7.7" +version = "0.8.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-carmenta-wasm/CHANGELOG.md b/crates/ffai-carmenta-wasm/CHANGELOG.md index 0b3ce2f6..76e23af3 100644 --- a/crates/ffai-carmenta-wasm/CHANGELOG.md +++ b/crates/ffai-carmenta-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-wasm-v0.3.0...ffai-carmenta-wasm-v0.4.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-wasm-v0.2.3...ffai-carmenta-wasm-v0.3.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-carmenta-wasm/Cargo.toml b/crates/ffai-carmenta-wasm/Cargo.toml index 7d49ea15..7542603f 100644 --- a/crates/ffai-carmenta-wasm/Cargo.toml +++ b/crates/ffai-carmenta-wasm/Cargo.toml @@ -5,7 +5,7 @@ description = "Carmenta in the browser — pure-Rust OCR compiled to WebAssembly # slice. The JS signature is unchanged — it is still a `Uint8Array` — but the # Rust signature is, so this is a breaking change for a Rust consumer even # though a browser sees nothing. Adds `setDetMaxShort` and `linearMemoryBytes`. -version = "0.3.0" +version = "0.4.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -26,7 +26,7 @@ ffai-core = { workspace = true } # release calls `reqwest::blocking` — a module that does not exist on wasm32. # Nothing in a browser can fetch weights through the model cache anyway: the # caller supplies bytes. -ffai-carmenta = { path = "../ffai-carmenta", version = "0.12.0", default-features = false } +ffai-carmenta = { path = "../ffai-carmenta", version = "0.13.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/ffai-carmenta/CHANGELOG.md b/crates/ffai-carmenta/CHANGELOG.md index 7cafa965..d6258707 100644 --- a/crates/ffai-carmenta/CHANGELOG.md +++ b/crates/ffai-carmenta/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.13.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-v0.12.0...ffai-carmenta-v0.13.0) - 2026-10-31 + +### Added + +- `OcrOptions::deskew` for the mobile detector: the angle comes from the detector's own text regions, and a page at least 0.3 degrees off is rotated once and read +- Word boxes for `mobiledet-svtr`, from the CTC emission frames snapped to ink +- `auto_orient` reports the turn it applied in `OcrOutput::turns` +- Ruled table bands are read row-major (`boxes::grid_order`) +- Hosted weights: the ppocrv5 manifests name `remade-with-rust/carmenta-ppocrv5-mobile`, so `fetch` downloads them sha256-pinned + +### Changed + +- **BREAKING:** `deskew::rotated_corners` takes the page's `turns`; requires `ffai-core` 0.9 +- 4.6-6.0x faster than 0.12 on a clean 300 dpi page (measured independently by PdfCraft): a direct depthwise conv, fused epilogues, and 1x1 and transposed convs as matmuls, with output bit-identical to candle where gated +- `auto_orient` reads the page first and decides orientation only when the read is unconvincing, and keeps the turned read only when it is more confident +- The detector's floor upscale is capped at 4x +- Drops the unused `candle-transformers` and `tokenizers` dependencies + +### Fixed + +- An inline formula spliced into a line keeps the word boxes it does not touch + ## [0.12.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-v0.11.0...ffai-carmenta-v0.12.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-carmenta/Cargo.toml b/crates/ffai-carmenta/Cargo.toml index 63aa2895..d8cc86f5 100644 --- a/crates/ffai-carmenta/Cargo.toml +++ b/crates/ffai-carmenta/Cargo.toml @@ -38,7 +38,7 @@ description = "Carmenta — pure-Rust OCR: documents, screens, and a change-gate # (cord +0.0033 with a +0.1928 clip, doc +0.0395 with a +0.2920 clip) — the same # failure the 544x960 arm produced, smaller. 1280 is the only arm that improves # both with no badly-regressing clip. The table lives on `DET_MAX_SHORT_DEFAULT`. -version = "0.12.0" +version = "0.13.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -50,7 +50,7 @@ categories = ["computer-vision", "multimedia::images", "science"] [dependencies] ffai-core = { workspace = true } -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep candle-core = { workspace = true } candle-nn = { workspace = true } serde = { workspace = true } diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index 62eba060..69a8fc19 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -39,9 +39,9 @@ cargo add ffai-carmenta ffai-core ffai-media ```toml [dependencies] -ffai-carmenta = "0.11" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-carmenta = "0.13" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Recognize diff --git a/crates/ffai-cli/CHANGELOG.md b/crates/ffai-cli/CHANGELOG.md index ad523af9..b94f87d1 100644 --- a/crates/ffai-cli/CHANGELOG.md +++ b/crates/ffai-cli/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-cli-v0.8.0...ffai-cli-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-cli-v0.7.0...ffai-cli-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-cli/Cargo.toml b/crates/ffai-cli/Cargo.toml index f0ae1f6e..456aed8c 100644 --- a/crates/ffai-cli/Cargo.toml +++ b/crates/ffai-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "ffai-cli" description = "The ffai binary — a thin shell over the FFai library crates" # follows its dependencies. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-core/CHANGELOG.md b/crates/ffai-core/CHANGELOG.md index 76449f90..9c444a9d 100644 --- a/crates/ffai-core/CHANGELOG.md +++ b/crates/ffai-core/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-core-v0.8.0...ffai-core-v0.9.0) - 2026-10-31 + +### Added + +- `OcrOptions::deskew`, and `OcrOutput::skew_degrees` reporting the angle a straightened page ran at +- `OcrOutput::turns`: the clockwise quarter turns `auto_orient` applied, which gives each box its reading direction +- `threads::available_threads` and `threads::affinity_pool`, which honour the process affinity mask on Windows + +### Changed + +- **BREAKING:** `OcrOptions` gains `deskew` and `OcrOutput` gains `skew_degrees` and `turns`, which breaks construction by struct literal and exhaustive destructuring +- `unsafe_code` is allowed at each site, so path and git dependents build without warnings + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-core-v0.7.2...ffai-core-v0.8.0) - 2026-09-25 ### Changed diff --git a/crates/ffai-core/Cargo.toml b/crates/ffai-core/Cargo.toml index 35542b8f..b381a538 100644 --- a/crates/ffai-core/Cargo.toml +++ b/crates/ffai-core/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai shared types, engine traits, and the engine registry" # published, so the crates.io copy of 0.6.1 cannot build ffai-diana — a gap # invisible locally, where path dependencies mask it, and caught only by # `cargo publish --dry-run`. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-diana/CHANGELOG.md b/crates/ffai-diana/CHANGELOG.md index fbbb74dc..2729e7f6 100644 --- a/crates/ffai-diana/CHANGELOG.md +++ b/crates/ffai-diana/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-diana-v0.8.0...ffai-diana-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-diana-v0.7.8...ffai-diana-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-diana/Cargo.toml b/crates/ffai-diana/Cargo.toml index 3eb26d29..476d6d59 100644 --- a/crates/ffai-diana/Cargo.toml +++ b/crates/ffai-diana/Cargo.toml @@ -3,7 +3,7 @@ name = "ffai-diana" description = "Diana — FFai's detection component (YOLO26 on candle, pure Rust)" # 0.7.3 is a documentation-only republish: README restructured to the shared # Remade With Rust page layout. No code, API or output change. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -35,7 +35,7 @@ ffai-core = { workspace = true } # Declared directly rather than inherited: `default-features = false` cannot # override a workspace dependency's defaults, and setting it at the workspace # level would silently strip `fetch` from every other crate that expects it. -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } candle-core = { workspace = true } candle-nn = { workspace = true } serde = { workspace = true } diff --git a/crates/ffai-diana/README.md b/crates/ffai-diana/README.md index 18d14040..86765497 100644 --- a/crates/ffai-diana/README.md +++ b/crates/ffai-diana/README.md @@ -33,9 +33,9 @@ cargo add ffai-diana ffai-core ffai-media ```toml [dependencies] -ffai-diana = "0.7" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-diana = "0.9" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Detect diff --git a/crates/ffai-media/CHANGELOG.md b/crates/ffai-media/CHANGELOG.md index d2769e84..43888383 100644 --- a/crates/ffai-media/CHANGELOG.md +++ b/crates/ffai-media/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-media-v0.6.8...ffai-media-v0.7.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.6.8](https://github.com/Remade-With-Rust/FFAI/compare/ffai-media-v0.6.7...ffai-media-v0.6.8) - 2026-09-25 ### Added diff --git a/crates/ffai-media/Cargo.toml b/crates/ffai-media/Cargo.toml index bf61c57a..b21956e7 100644 --- a/crates/ffai-media/Cargo.toml +++ b/crates/ffai-media/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai media ingest/egress — demux, decode, resample (rff-backed) # rusty_jpeg 0.1 -> 0.2 picks up the progressive-JPEG fix. A dependency # change with no API change, so a patch bump -- but a load-bearing one: # 0.6.1 cannot decode 35 of OmniDocBench's 236 pages. -version = "0.6.8" +version = "0.7.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-media/README.md b/crates/ffai-media/README.md index 271468b8..d764da92 100644 --- a/crates/ffai-media/README.md +++ b/crates/ffai-media/README.md @@ -23,7 +23,7 @@ Both dispatch on the bytes' magic, not the file name: 169 of OmniDocBench's Engl ## PDF pages as a viewer shows them (`pdf` feature) ```toml -ffai-media = { version = "0.6", features = ["pdf"] } +ffai-media = { version = "0.7", features = ["pdf"] } ``` ```rust diff --git a/crates/ffai-mercury-wasm/CHANGELOG.md b/crates/ffai-mercury-wasm/CHANGELOG.md index e683ecc4..f846f1f2 100644 --- a/crates/ffai-mercury-wasm/CHANGELOG.md +++ b/crates/ffai-mercury-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-wasm-v0.2.0...ffai-mercury-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-wasm-v0.1.3...ffai-mercury-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-mercury-wasm/Cargo.toml b/crates/ffai-mercury-wasm/Cargo.toml index ef334ced..84dfbe6b 100644 --- a/crates/ffai-mercury-wasm/Cargo.toml +++ b/crates/ffai-mercury-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-mercury-wasm" description = "Mercury in the browser — pure-Rust Whisper speech recognition compiled to WebAssembly, no whisper.cpp, no ONNX runtime" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -28,7 +28,7 @@ ffai-core = { workspace = true } # the constructor this crate is built on, first exists in 1.0.2. A `^1.0.1` # requirement resolves happily in the workspace and then fails to compile the # moment it is packaged, which is exactly how it failed. -ffai-mercury = { path = "../ffai-mercury", version = "1.0.5", default-features = false } +ffai-mercury = { path = "../ffai-mercury", version = "1.1.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-mercury/CHANGELOG.md b/crates/ffai-mercury/CHANGELOG.md index 0e3b882b..eba2f9c9 100644 --- a/crates/ffai-mercury/CHANGELOG.md +++ b/crates/ffai-mercury/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-v1.0.5...ffai-mercury-v1.1.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [1.0.5](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-v1.0.4...ffai-mercury-v1.0.5) - 2026-09-25 ### Other diff --git a/crates/ffai-mercury/Cargo.toml b/crates/ffai-mercury/Cargo.toml index f5262cf2..31835295 100644 --- a/crates/ffai-mercury/Cargo.toml +++ b/crates/ffai-mercury/Cargo.toml @@ -6,7 +6,7 @@ description = "Mercury — FFai's voice component: pure-Rust ASR (Whisper/Whispe # to 0.6.1 -- the API is purely additive, but this crate advertises byte-stable # determinism, so a changed waveform is a compatibility break for anyone who # pinned bytes. Its dependencies stay at the published 0.6.1. -version = "1.0.5" +version = "1.1.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -18,7 +18,7 @@ categories = ["multimedia::audio", "science", "accessibility"] [dependencies] ffai-core = { workspace = true } -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep candle-nn = { workspace = true } candle-transformers = { workspace = true } tokenizers = { workspace = true } diff --git a/crates/ffai-mercury/README.md b/crates/ffai-mercury/README.md index bf567988..dbf08c7c 100644 --- a/crates/ffai-mercury/README.md +++ b/crates/ffai-mercury/README.md @@ -11,9 +11,9 @@ Mercury is [FFai](https://github.com/Remade-With-Rust/FFAI)'s voice component. S ```toml [dependencies] -ffai-mercury = "0.7" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-mercury = "1.1" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Transcribe @@ -258,7 +258,7 @@ MIT OR Apache-2.0. Model weights carry their own licenses, surfaced at selection first use. It is also the single largest thing in the dependency graph: ```toml -ffai-mercury = { version = "0.7", default-features = false } # ship your own weights +ffai-mercury = { version = "1.1", default-features = false } # ship your own weights ``` | build | dependencies | C crypto | diff --git a/crates/ffai-models/CHANGELOG.md b/crates/ffai-models/CHANGELOG.md index 8394846a..120f5070 100644 --- a/crates/ffai-models/CHANGELOG.md +++ b/crates/ffai-models/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-models-v0.6.6...ffai-models-v0.7.0) - 2026-10-31 + +### Added + +- `ModelManifest::fetch_only`: resolve and download only the named files + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. +- Drops `dirs` (and with it the MPL `option-ext`) + +### Fixed + +- An engine fetching from the hub downloads only the files it opens, as offline resolution already did + ## [0.6.6](https://github.com/Remade-With-Rust/FFAI/compare/ffai-models-v0.6.5...ffai-models-v0.6.6) - 2026-09-25 ### Added diff --git a/crates/ffai-models/Cargo.toml b/crates/ffai-models/Cargo.toml index 166bb27e..37676f5a 100644 --- a/crates/ffai-models/Cargo.toml +++ b/crates/ffai-models/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai model weight management — manifests, cache, licensing surf # excludes the 0.7.0 every other crate in this release now needs. Left at # 0.6.2 it would put TWO ffai-core versions in every downstream tree and the # shared `ffai_core` types would stop being the same types. -version = "0.6.6" +version = "0.7.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-wasm/CHANGELOG.md b/crates/ffai-wasm/CHANGELOG.md index ae702cdb..abf76fb5 100644 --- a/crates/ffai-wasm/CHANGELOG.md +++ b/crates/ffai-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-wasm-v0.2.0...ffai-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-wasm-v0.1.5...ffai-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-wasm/Cargo.toml b/crates/ffai-wasm/Cargo.toml index 4124f6ac..f5b8a16f 100644 --- a/crates/ffai-wasm/Cargo.toml +++ b/crates/ffai-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-wasm" description = "Diana in the browser — YOLO26 object detection compiled to WebAssembly, pure Rust, no ONNX runtime" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true From fa2a79e5d181a4adf2d01820dc6b711020962934 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 09:11:55 -0700 Subject: [PATCH 23/25] fix(deps): quinn-proto 0.11.19, webpki-roots 1.0.11; cargo-audit carries the zip ignores cargo-audit scans more of the tree than cargo-deny and reported six more: RUSTSEC-2026-0338 to -0342 in quinn-proto and -0381 in webpki-roots, all with semver-compatible fixes, taken in the lockfile. It also has its own ignore list, which did not exist, so it still failed on the three zip 8.6 extraction advisories deny.toml accepts (RUSTSEC-2026-0367/0368/0369; candle-core never extracts, and zip 9 is out of its reach). .cargo/audit.toml now carries the same three, dated and pointing at deny.toml for the justification. Co-Authored-By: Claude Opus 5.5 --- .cargo/audit.toml | 12 ++++++++++++ Cargo.lock | 10 +++++----- 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 .cargo/audit.toml diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 00000000..f8d5415a --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,12 @@ +# cargo-audit's ignores. deny.toml carries the same advisories with the full +# justification; keep the two lists in step (CI runs both). +[advisories] +ignore = [ + # zip 8.6.0 extraction advisories, fixed only in zip 9, which candle-core + # 0.11 cannot take (it pins ^8.6). All three are in ZipArchive::extract / + # ZipStreamReader::extract; candle reads entries by name and never extracts. + # Added 2026-10-31. Review 2026-11-15, or when candle moves to zip 9. + "RUSTSEC-2026-0367", + "RUSTSEC-2026-0368", + "RUSTSEC-2026-0369", +] diff --git a/Cargo.lock b/Cargo.lock index f4dbf991..1aab8510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3426,9 +3426,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -5383,14 +5383,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] From 833939f2bf05475be64b04ee5b3455eedf26a320 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 10:49:42 -0700 Subject: [PATCH 24/25] fix(supply-chain): cargo-vet passes again cargo-vet now refuses an unversioned policy for a crate that also has a third-party version. candle-metal-kernels 0.11.0 is in the lockfile twice, from crates.io via candle-nn and from our candle fork. The policy is now keyed to the git revision, since only the fork is first-party. Master has the same lockfile; this step was masked there by the cargo-audit failure. The quinn-proto and webpki-roots exemptions move to the security-fix versions in the lockfile (0.11.19, 1.0.11), which is the repo's convention for dependency deltas. imports.lock is the refresh cargo-vet produced. Co-Authored-By: Claude Opus 5.5 --- supply-chain/config.toml | 6 ++-- supply-chain/imports.lock | 75 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 65b3650c..bd2f583f 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -37,7 +37,7 @@ audit-as-crates-io = true [policy.candle-kernels] audit-as-crates-io = true -[policy.candle-metal-kernels] +[policy."candle-metal-kernels:0.11.0@git:fa366d84d12c5732cbc34c973fe92b8f480bfdbe"] audit-as-crates-io = true [policy.candle-ug] @@ -852,7 +852,7 @@ version = "0.11.11" criteria = "safe-to-deploy" [[exemptions.quinn-proto]] -version = "0.11.16" +version = "0.11.19" criteria = "safe-to-deploy" [[exemptions.r-efi]] @@ -1304,7 +1304,7 @@ version = "0.26.11" criteria = "safe-to-deploy" [[exemptions.webpki-roots]] -version = "1.0.9" +version = "1.0.11" criteria = "safe-to-deploy" [[exemptions.weezl]] diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 20231e98..5b152f10 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -9,6 +9,10 @@ audited_as = "0.7.4" version = "0.8.0" audited_as = "0.7.5" +[[unpublished.ffai-argus]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-argus-wasm]] version = "0.1.3" audited_as = "0.1.2" @@ -17,10 +21,18 @@ audited_as = "0.1.2" version = "0.2.0" audited_as = "0.1.3" +[[unpublished.ffai-argus-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[unpublished.ffai-bench]] version = "0.7.7" audited_as = "0.7.6" +[[unpublished.ffai-bench]] +version = "0.8.0" +audited_as = "0.7.7" + [[unpublished.ffai-carmenta]] version = "0.11.0" audited_as = "0.10.2" @@ -29,6 +41,10 @@ audited_as = "0.10.2" version = "0.12.0" audited_as = "0.11.0" +[[unpublished.ffai-carmenta]] +version = "0.13.0" +audited_as = "0.12.0" + [[unpublished.ffai-carmenta-wasm]] version = "0.2.3" audited_as = "0.2.2" @@ -37,6 +53,10 @@ audited_as = "0.2.2" version = "0.3.0" audited_as = "0.2.3" +[[unpublished.ffai-carmenta-wasm]] +version = "0.4.0" +audited_as = "0.3.0" + [[unpublished.ffai-cli]] version = "0.7.0" audited_as = "0.6.9" @@ -45,10 +65,18 @@ audited_as = "0.6.9" version = "0.8.0" audited_as = "0.7.0" +[[unpublished.ffai-cli]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-core]] version = "0.8.0" audited_as = "0.7.2" +[[unpublished.ffai-core]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-diana]] version = "0.7.8" audited_as = "0.7.7" @@ -57,14 +85,26 @@ audited_as = "0.7.7" version = "0.8.0" audited_as = "0.7.8" +[[unpublished.ffai-diana]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-media]] version = "0.6.8" audited_as = "0.6.7" +[[unpublished.ffai-media]] +version = "0.7.0" +audited_as = "0.6.8" + [[unpublished.ffai-mercury]] version = "1.0.5" audited_as = "1.0.4" +[[unpublished.ffai-mercury]] +version = "1.1.0" +audited_as = "1.0.5" + [[unpublished.ffai-mercury-wasm]] version = "0.1.3" audited_as = "0.1.2" @@ -73,10 +113,18 @@ audited_as = "0.1.2" version = "0.2.0" audited_as = "0.1.3" +[[unpublished.ffai-mercury-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[unpublished.ffai-models]] version = "0.6.6" audited_as = "0.6.5" +[[unpublished.ffai-models]] +version = "0.7.0" +audited_as = "0.6.6" + [[unpublished.ffai-wasm]] version = "0.1.5" audited_as = "0.1.4" @@ -85,6 +133,10 @@ audited_as = "0.1.4" version = "0.2.0" audited_as = "0.1.5" +[[unpublished.ffai-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[publisher.aho-corasick]] version = "1.1.4" when = "2025-10-28" @@ -1526,6 +1578,17 @@ criteria = "safe-to-deploy" delta = "0.3.0 -> 0.4.0" notes = "Minor updates for a new `rand` crate version, nothing awry." +[[audits.bytecode-alliance.audits.rustls]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +delta = "0.23.45 -> 0.23.46" + +[[audits.bytecode-alliance.audits.rustls-webpki]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.103.15 -> 0.103.16" +notes = "Updated with new certificates" + [[audits.bytecode-alliance.audits.semver]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -1591,6 +1654,18 @@ who = "Pat Hickey " criteria = "safe-to-deploy" version = "0.3.0" +[[audits.bytecode-alliance.audits.webpki-root-certs]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +version = "1.0.4" +notes = "Purely a data crate." + +[[audits.bytecode-alliance.audits.webpki-root-certs]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "1.0.4 -> 1.0.11" +notes = "Standard root updates" + [[audits.bytecode-alliance.audits.xattr]] who = "Andrew Brown " criteria = "safe-to-deploy" From 2635b6b91695ed744261d9217f3396115da3fed4 Mon Sep 17 00:00:00 2001 From: Tim Date: Sat, 31 Oct 2026 11:18:35 -0700 Subject: [PATCH 25/25] fix(supply-chain): cargo-vet policy for the serde_json fork cargo-vet 0.10.3 requires an explicit audit-as-crates-io policy for a git package whose version matches a published crate. serde_json 1.0.151 from rusty_json_turbo is house code, so it is declared first-party. Co-Authored-By: Claude Opus 5.5 --- supply-chain/config.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index bd2f583f..e0cbace4 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -88,6 +88,9 @@ audit-as-crates-io = true [policy.rusty_json_turbo-accel] audit-as-crates-io = true +[policy."serde_json:1.0.151@git:77a58ed4c8e281f66e2d6b3a5b9ccd979ebec05b"] +audit-as-crates-io = false + [policy.tokenizers] audit-as-crates-io = true