diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..f8d5415 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,12 @@ +# cargo-audit's ignores. deny.toml carries the same advisories with the full +# justification; keep the two lists in step (CI runs both). +[advisories] +ignore = [ + # zip 8.6.0 extraction advisories, fixed only in zip 9, which candle-core + # 0.11 cannot take (it pins ^8.6). All three are in ZipArchive::extract / + # ZipStreamReader::extract; candle reads entries by name and never extracts. + # Added 2026-10-31. Review 2026-11-15, or when candle moves to zip 9. + "RUSTSEC-2026-0367", + "RUSTSEC-2026-0368", + "RUSTSEC-2026-0369", +] diff --git a/Cargo.lock b/Cargo.lock index 0bdc333..1aab851 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,7 +454,7 @@ dependencies = [ "byteorder", "candle-core", "candle-nn", - "fancy-regex 0.18.0", + "fancy-regex", "num-traits", "rand 0.9.5", "rayon", @@ -1181,17 +1181,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -[[package]] -name = "fancy-regex" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fancy-regex" version = "0.18.0" @@ -1226,7 +1215,7 @@ dependencies = [ [[package]] name = "ffai-argus" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "candle-nn", @@ -1244,7 +1233,7 @@ dependencies = [ [[package]] name = "ffai-argus-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-argus", "ffai-core", @@ -1256,7 +1245,7 @@ dependencies = [ [[package]] name = "ffai-bench" -version = "0.7.7" +version = "0.8.0" dependencies = [ "clap", "csv", @@ -1274,28 +1263,25 @@ dependencies = [ [[package]] name = "ffai-carmenta" -version = "0.12.0" +version = "0.13.0" dependencies = [ "candle-core", "candle-nn", - "candle-transformers", "ffai-bench", "ffai-core", "ffai-media", "ffai-models", - "getrandom 0.3.4", "mimalloc", "png", "rayon", "rusty_alloc-api", "serde", "serde_json", - "tokenizers 0.23.2", ] [[package]] name = "ffai-carmenta-wasm" -version = "0.3.0" +version = "0.4.0" dependencies = [ "ffai-carmenta", "ffai-core", @@ -1310,7 +1296,7 @@ dependencies = [ [[package]] name = "ffai-cli" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "clap", @@ -1329,16 +1315,17 @@ dependencies = [ [[package]] name = "ffai-core" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "rayon", "thiserror 2.0.19", + "windows-sys 0.61.2", ] [[package]] name = "ffai-demo" -version = "0.8.0" +version = "0.9.0" dependencies = [ "ffai-argus", "ffai-bench", @@ -1351,7 +1338,7 @@ dependencies = [ [[package]] name = "ffai-diana" -version = "0.8.0" +version = "0.9.0" dependencies = [ "candle-core", "candle-nn", @@ -1371,7 +1358,7 @@ dependencies = [ [[package]] name = "ffai-media" -version = "0.6.8" +version = "0.7.0" dependencies = [ "fax", "ffai-core", @@ -1393,7 +1380,7 @@ dependencies = [ [[package]] name = "ffai-mercury" -version = "1.0.5" +version = "1.1.0" dependencies = [ "candle-nn", "candle-transformers", @@ -1412,7 +1399,7 @@ dependencies = [ [[package]] name = "ffai-mercury-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-core", "ffai-mercury", @@ -1423,9 +1410,8 @@ dependencies = [ [[package]] name = "ffai-models" -version = "0.6.6" +version = "0.7.0" dependencies = [ - "dirs", "ffai-core", "hf-hub", "serde", @@ -1446,7 +1432,7 @@ dependencies = [ [[package]] name = "ffai-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ffai-core", "ffai-diana", @@ -3440,9 +3426,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -3879,9 +3865,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -3944,9 +3930,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4567,7 +4553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4720,7 +4706,6 @@ dependencies = [ "dary_heap", "derive_builder", "esaxx-rs", - "fancy-regex 0.17.0", "getrandom 0.3.4", "itertools", "log", @@ -5385,9 +5370,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] @@ -5398,14 +5383,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index d63caa2..4b9e253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = ["crates/*"] exclude = ["demo-ui"] [workspace.package] -version = "0.8.0" +version = "0.9.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" @@ -16,14 +16,14 @@ authors = ["Remade With Rust"] [workspace.dependencies] # ---- internal crates ---- -ffai-core = { path = "crates/ffai-core", version = "0.8.0" } -ffai-media = { path = "crates/ffai-media", version = "0.6.8" } -ffai-models = { path = "crates/ffai-models", version = "0.6.6" } -ffai-mercury = { path = "crates/ffai-mercury", version = "1.0.5" } -ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.12.0" } -ffai-diana = { path = "crates/ffai-diana", version = "0.8.0" } -ffai-argus = { path = "crates/ffai-argus", version = "0.8.0" } -ffai-bench = { path = "crates/ffai-bench", version = "0.7.7" } +ffai-core = { path = "crates/ffai-core", version = "0.9.0" } +ffai-media = { path = "crates/ffai-media", version = "0.7.0" } +ffai-models = { path = "crates/ffai-models", version = "0.7.0" } +ffai-mercury = { path = "crates/ffai-mercury", version = "1.1.0" } +ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.13.0" } +ffai-diana = { path = "crates/ffai-diana", version = "0.9.0" } +ffai-argus = { path = "crates/ffai-argus", version = "0.9.0" } +ffai-bench = { path = "crates/ffai-bench", version = "0.8.0" } # ---- tensor spine ---- # Candle is FFai's PyTorch: every engine shares its Tensor/Device types so @@ -46,7 +46,6 @@ toml = "0.8" anyhow = "1" clap = { version = "4", features = ["derive"] } hound = "3" -dirs = "6" sha2 = "0.10" serde_json = "1" regex = "1" diff --git a/crates/ffai-argus-wasm/CHANGELOG.md b/crates/ffai-argus-wasm/CHANGELOG.md index 3dcc3ec..1e0cb0c 100644 --- a/crates/ffai-argus-wasm/CHANGELOG.md +++ b/crates/ffai-argus-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-wasm-v0.2.0...ffai-argus-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-wasm-v0.1.3...ffai-argus-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-argus-wasm/Cargo.toml b/crates/ffai-argus-wasm/Cargo.toml index eb94129..560a0d7 100644 --- a/crates/ffai-argus-wasm/Cargo.toml +++ b/crates/ffai-argus-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-argus-wasm" description = "Argus in the browser — pure-Rust SmolVLM image captioning compiled to WebAssembly, no ONNX runtime, no Python" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -25,7 +25,7 @@ ffai-core = { workspace = true } # **0.7.2, not 0.7.1** — see the note in `ffai-mercury-wasm`. `SmolVlm::from_bytes` # and `ArgusBytes` first exist in 0.7.2, and a tarball verification resolves # this from crates.io rather than from the path. -ffai-argus = { path = "../ffai-argus", version = "0.8.0", default-features = false } +ffai-argus = { path = "../ffai-argus", version = "0.9.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-argus/CHANGELOG.md b/crates/ffai-argus/CHANGELOG.md index b7d1e9a..3afa0e3 100644 --- a/crates/ffai-argus/CHANGELOG.md +++ b/crates/ffai-argus/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-v0.8.0...ffai-argus-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-argus-v0.7.5...ffai-argus-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-argus/Cargo.toml b/crates/ffai-argus/Cargo.toml index 53b4ce5..3b673e2 100644 --- a/crates/ffai-argus/Cargo.toml +++ b/crates/ffai-argus/Cargo.toml @@ -19,7 +19,7 @@ ffai-core = { workspace = true } # their own weights; they must not inherit fetch just because Argus is linked. # Not `workspace = true`: cargo forbids overriding default-features on an # inherited dep. -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # The tokenizer is the ENGINE's business now, not just the step-4 test's: the # assembled prompt has to become ids and the generated ids have to become text. tokenizers = { workspace = true } diff --git a/crates/ffai-argus/README.md b/crates/ffai-argus/README.md index b6357dc..41d22d8 100644 --- a/crates/ffai-argus/README.md +++ b/crates/ffai-argus/README.md @@ -407,9 +407,9 @@ or in `Cargo.toml`: ```toml [dependencies] -ffai-argus = "0.6" # the engine +ffai-argus = "0.9" # the engine ffai-core = "0.7" # the VlmEngine trait, the registry, the cost counters -ffai-media = "0.6" # decoding an image/video file is the CALLER's job +ffai-media = "0.7" # decoding an image/video file is the CALLER's job ``` `ffai-media` is separate on purpose: the engine takes an already-decoded diff --git a/crates/ffai-bench/CHANGELOG.md b/crates/ffai-bench/CHANGELOG.md index 38e26aa..fa9f84a 100644 --- a/crates/ffai-bench/CHANGELOG.md +++ b/crates/ffai-bench/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-bench-v0.7.7...ffai-bench-v0.8.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.7.7](https://github.com/Remade-With-Rust/FFAI/compare/ffai-bench-v0.7.6...ffai-bench-v0.7.7) - 2026-09-25 ### Added diff --git a/crates/ffai-bench/Cargo.toml b/crates/ffai-bench/Cargo.toml index bc42549..6e3847d 100644 --- a/crates/ffai-bench/Cargo.toml +++ b/crates/ffai-bench/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai's analyzer — one call to compare any engine against world # the runner records a clip failure and continues instead of aborting the # whole corpus, and decode is wrapped so a panicking dependency costs one # page rather than the run. -version = "0.7.7" +version = "0.8.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-carmenta-wasm/CHANGELOG.md b/crates/ffai-carmenta-wasm/CHANGELOG.md index 0b3ce2f..76e23af 100644 --- a/crates/ffai-carmenta-wasm/CHANGELOG.md +++ b/crates/ffai-carmenta-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-wasm-v0.3.0...ffai-carmenta-wasm-v0.4.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-wasm-v0.2.3...ffai-carmenta-wasm-v0.3.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-carmenta-wasm/Cargo.toml b/crates/ffai-carmenta-wasm/Cargo.toml index 7d49ea1..7542603 100644 --- a/crates/ffai-carmenta-wasm/Cargo.toml +++ b/crates/ffai-carmenta-wasm/Cargo.toml @@ -5,7 +5,7 @@ description = "Carmenta in the browser — pure-Rust OCR compiled to WebAssembly # slice. The JS signature is unchanged — it is still a `Uint8Array` — but the # Rust signature is, so this is a breaking change for a Rust consumer even # though a browser sees nothing. Adds `setDetMaxShort` and `linearMemoryBytes`. -version = "0.3.0" +version = "0.4.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -26,7 +26,7 @@ ffai-core = { workspace = true } # release calls `reqwest::blocking` — a module that does not exist on wasm32. # Nothing in a browser can fetch weights through the model cache anyway: the # caller supplies bytes. -ffai-carmenta = { path = "../ffai-carmenta", version = "0.12.0", default-features = false } +ffai-carmenta = { path = "../ffai-carmenta", version = "0.13.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/ffai-carmenta/CHANGELOG.md b/crates/ffai-carmenta/CHANGELOG.md index 7cafa96..d625870 100644 --- a/crates/ffai-carmenta/CHANGELOG.md +++ b/crates/ffai-carmenta/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.13.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-v0.12.0...ffai-carmenta-v0.13.0) - 2026-10-31 + +### Added + +- `OcrOptions::deskew` for the mobile detector: the angle comes from the detector's own text regions, and a page at least 0.3 degrees off is rotated once and read +- Word boxes for `mobiledet-svtr`, from the CTC emission frames snapped to ink +- `auto_orient` reports the turn it applied in `OcrOutput::turns` +- Ruled table bands are read row-major (`boxes::grid_order`) +- Hosted weights: the ppocrv5 manifests name `remade-with-rust/carmenta-ppocrv5-mobile`, so `fetch` downloads them sha256-pinned + +### Changed + +- **BREAKING:** `deskew::rotated_corners` takes the page's `turns`; requires `ffai-core` 0.9 +- 4.6-6.0x faster than 0.12 on a clean 300 dpi page (measured independently by PdfCraft): a direct depthwise conv, fused epilogues, and 1x1 and transposed convs as matmuls, with output bit-identical to candle where gated +- `auto_orient` reads the page first and decides orientation only when the read is unconvincing, and keeps the turned read only when it is more confident +- The detector's floor upscale is capped at 4x +- Drops the unused `candle-transformers` and `tokenizers` dependencies + +### Fixed + +- An inline formula spliced into a line keeps the word boxes it does not touch + ## [0.12.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-carmenta-v0.11.0...ffai-carmenta-v0.12.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-carmenta/Cargo.toml b/crates/ffai-carmenta/Cargo.toml index f7aad95..d8cc86f 100644 --- a/crates/ffai-carmenta/Cargo.toml +++ b/crates/ffai-carmenta/Cargo.toml @@ -38,7 +38,7 @@ description = "Carmenta — pure-Rust OCR: documents, screens, and a change-gate # (cord +0.0033 with a +0.1928 clip, doc +0.0395 with a +0.2920 clip) — the same # failure the 544x960 arm produced, smaller. 1280 is the only arm that improves # both with no badly-regressing clip. The table lives on `DET_MAX_SHORT_DEFAULT`. -version = "0.12.0" +version = "0.13.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -50,10 +50,9 @@ categories = ["computer-vision", "multimedia::images", "science"] [dependencies] ffai-core = { workspace = true } -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep candle-core = { workspace = true } candle-nn = { workspace = true } -candle-transformers = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } png = { workspace = true } @@ -64,17 +63,14 @@ png = { workspace = true } # panicking in someone's browser. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rayon = { workspace = true } -tokenizers = { workspace = true } -[target.'cfg(target_arch = "wasm32")'.dependencies] -# getrandom 0.3 needs its `wasm_js` FEATURE in addition to the -# `--cfg getrandom_backend` that .cargo/config.toml supplies; it errors if -# either half is missing. Arrives here through tokenizers -> ahash. -getrandom = { version = "0.3", features = ["wasm_js"] } -# `onig` is Oniguruma, a C regex engine: there is no `cc` for wasm32, and a C -# dependency is not something this house ships anyway. `unstable_wasm` swaps it -# for fancy-regex, which is pure Rust. -tokenizers = { version = "0.23", default-features = false, features = ["unstable_wasm"] } +# `candle-transformers` and `tokenizers` used to be listed here and nothing in +# this crate used either (PdfCraft, 2026-10-30): an OCR-only consumer compiled +# all of candle-transformers, and `tokenizers` with Oniguruma, a C library, for +# nothing. candle-core 0.11 still pulls `tokenizers` + onig on native targets on +# its own (its `quantized::tokenizer`); in this workspace the root +# `[patch.crates-io]` points that at the onig-free fork, a crates.io consumer +# gets onig until candle makes it optional. [dev-dependencies] # Examples only, never the library. `ffai-cli` sets mimalloc as its global diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index fd80ff5..69a8fc1 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -39,9 +39,9 @@ cargo add ffai-carmenta ffai-core ffai-media ```toml [dependencies] -ffai-carmenta = "0.11" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-carmenta = "0.13" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Recognize @@ -85,6 +85,15 @@ additionally be routed to dedicated structure models (`FFAI_ROUTE=1`, experimental: pure-Rust ports of PP-DocLayout-S, SLANet-plus and PP-FormulaNet-S, each matched against its reference runtime). +**Latin beyond English is partial, and the gap is in the charset, not the +model's skill.** On rendered pages lowercase French reads 1.9 % CER, German +0.9 %, Spanish 0.1 % (PdfCraft's corpus). But the head's 18 383 characters do +not include `À Â Ê Ë Ì Î Ï Ñ Ò Ô Õ Ø Ù Ú Û Ý`, `î ï ÿ œ` or `« » ¿ ¡ ‘ ™ ‰`, so +those can never be emitted: a page of accented capitals read 24 % CER +(`ESPAÑA` → `EsPAnA`). `OcrOptions::charset` restricts decoding to what a +caller can store, and is how to keep a sideways or capitals-heavy page from +emitting CJK look-alikes. + Measured by [OmniDocBench](https://github.com/opendatalab/OmniDocBench)'s **own evaluator** — the benchmark PaddleOCR-VL and Baidu's Unlimited-OCR state their records on — over all 1 651 pages of v1.6, English and Chinese, zero @@ -129,8 +138,12 @@ So the remaining loss is mostly **coverage and characters**, not layout: the reading-order and detection stages this crate spent its campaign on are down to a combined 4 % of remaining error. -All of this on CPU at ~9–17 s/page — roughly 9× the reference VLM's measured -throughput on the same machine, from megabytes of weights against gigabytes. +All of this on CPU, from megabytes of weights against gigabytes. It was +measured at ~9–17 s/page, roughly 9× the reference VLM's throughput on the +same machine; since the depthwise-convolution and fused-epilogue work +(2026-10-30; recognizer bit-identical, text and boxes identical on PdfCraft's +42-page corpus) the same 315 OmniDocBench pages read at a +**1.4 s median, 2.7 s p90** per page on a 24-thread i7-14650HX. ## Forms, scanned filings and PDFs (0.11) @@ -158,7 +171,9 @@ What else 0.11 adds, each measured: | capability | API | measured | |---|---|---| -| orientation, 0/90/180/270 | `OcrOptions::auto_orient`, `detect_orientation` | **144 / 144** turns undone on document pages and single form lines; a decision costs **48 %** of one page read | +| orientation, 0/90/180/270 | `OcrOptions::auto_orient`, `detect_orientation` | **144 / 144** turns undone on document pages and single form lines. The page is read as given first and the decision runs only when that read's confidence is below 0.7: **1.004x** on upright pages (ABBA, 3/6), where the unconditional decision cost 2.2x (PdfCraft); turned pages 91 % → 0 % CER | +| deskew, small tilt | `OcrOptions::deskew`, `OcrOutput::skew_degrees` | angle from the detector's own text regions; pages 0.5-5° off read **0.10 %** CER (48.3 % without), a 1.5°-tilted phone photo **0.11 %** (61 %); square pages untouched (PdfCraft's corpus) | +| word boxes | `OcrLine::words` (mobiledet-svtr) | from the CTC frames, snapped to ink: **96 %** of ground-truth words at IoU ≥ 0.5 with the right text (PdfCraft's corpus) | | checkboxes, frame + marked/empty | `checkbox::find` | **100 %** recall and **100 %** state on 960 synthetic boxes (stroked and glyph boxes; X, overflowing tick, fill). Known false positives: the square ideograph `口`, and small square plot markers | | label → value pairs | `forms::pair_fields` | **220 / 221** correctly-read values paired with their label | | restrict a field's characters | `OcrOptions::charset` | forced characters show in the confidence: a prose page read as digits drops from 0.918 to 0.781 mean | @@ -166,6 +181,7 @@ What else 0.11 adds, each measured: | erase form rules | `OcrOptions::remove_rules` | CRNN on underlined values: 13-20 % → **70 %**. **Not for SVTR**: on strikethroughs it goes 77-87 % → 73 %. Opt-in | | full-width → ASCII on Latin lines | on by default (`FFAI_NO_FOLD=1` disables) | OmniDocBench, 316 pages: fires on 35 of 34 852 lines; **5 pages better, 1 worse** (a full-width comma in the ground truth itself) | | offline weights, fail fast | `CraftCrnn::offline(rec, det, dir)`, `preload()` | no cache, no network; a missing file names its exact path | +| document-engine weights, hosted | `fetch` feature, or download for `offline` | [`remade-with-rust/carmenta-ppocrv5-mobile`](https://huggingface.co/remade-with-rust/carmenta-ppocrv5-mobile): the three files `mobiledet-svtr` loads, each verified against the sha256 pinned in its manifest | Two measured non-results, recorded so nobody has to re-run them: @@ -248,7 +264,8 @@ turns it off. pipeline still trails PaddleOCR (20.9 % vs 15.6 % CER) even though our recognition stage beats theirs on identical crops — the diagnosed cause is tilt-sensitive line grouping, with deskew as the named fix. `auto_orient` -handles quarter turns, not small tilt: deskew is still open. +handles quarter turns and `deskew` small tilt (measured on rendered and +synthetic-photo pages; not yet re-run on the receipt corpus). ## Status: `experimental`, honestly diff --git a/crates/ffai-carmenta/examples/pdfcraft_bench.rs b/crates/ffai-carmenta/examples/pdfcraft_bench.rs new file mode 100644 index 0000000..7840be3 --- /dev/null +++ b/crates/ffai-carmenta/examples/pdfcraft_bench.rs @@ -0,0 +1,132 @@ +//! Read a list of pages with `mobiledet-svtr` from an OFFLINE weights root and +//! write one JSON line per page in the shape PdfCraft's `score.py` reads: +//! `{"page", "ms", "lines": [[[text, [l, t, r, b]], ...], ...]}`. +//! +//! It exists so a caller's corpus — PdfCraft's 49 rendered pages, 42 of them +//! with ground truth by construction — can be scored against THIS tree, not +//! only against the release on crates.io. A line with word geometry emits one +//! entry per word; a line without it emits the whole line as one entry with +//! the line box, which the scorer's CER is indifferent to. +//! +//! Usage: `pdfcraft_bench ...` +//! Env: `BENCH_REC=crnn` for mobiledet-crnn, `BENCH_ORIENT=1` for +//! `auto_orient`, `BENCH_DESKEW=1` for `deskew`, `BENCH_LANG=en,fr` for +//! `languages`, `BENCH_POOL=1` to run inside `ffai_core::threads::affinity_pool`. + +#[global_allocator] +static GLOBAL: rusty_alloc_api::RustyAlloc = rusty_alloc_api::RustyAlloc; + +use std::fmt::Write as _; +use std::path::Path; +use std::time::Instant; + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_core::engine::{OcrEngine, OcrOptions}; + +fn esc(s: &str) -> String { + let mut o = String::with_capacity(s.len() + 2); + for c in s.chars() { + match c { + '"' => o.push_str("\\\""), + '\\' => o.push_str("\\\\"), + c if (c as u32) < 0x20 => { + let _ = write!(o, "\\u{:04x}", c as u32); + } + c => o.push(c), + } + } + o +} + +fn main() { + let mut args = std::env::args().skip(1); + let root = args.next().expect("usage: pdfcraft_bench ..."); + let pages: Vec = args.collect(); + let rec = match std::env::var("BENCH_REC").as_deref() { + Ok("crnn") => RecStage::Crnn, + _ => RecStage::Svtr, + }; + let engine = CraftCrnn::offline(rec, DetStage::MobileDet, Path::new(&root)); + let t = Instant::now(); + engine.preload().expect("preload"); + eprintln!("load {:.1} ms", t.elapsed().as_secs_f64() * 1e3); + // The mask is read only now, after the load: a harness that pins the + // process after spawning it (pinvs does) can lose a race against the top + // of `main`, and the pool would be built for the whole machine. + eprintln!("available_threads {}", ffai_core::threads::available_threads()); + if std::env::var("BENCH_POOL").is_ok_and(|v| v == "1") { + let pool = ffai_core::threads::affinity_pool().expect("thread pool"); + pool.install(|| run(&engine, &pages)); + } else { + run(&engine, &pages); + } +} + +fn run(engine: &CraftCrnn, pages: &[String]) { + let opts = OcrOptions { + auto_orient: std::env::var("BENCH_ORIENT").is_ok_and(|v| v == "1"), + deskew: std::env::var("BENCH_DESKEW").is_ok_and(|v| v == "1"), + languages: std::env::var("BENCH_LANG") + .map(|v| v.split(',').map(str::to_string).collect()) + .unwrap_or_default(), + ..OcrOptions::default() + }; + let all = Instant::now(); + let (mut words_total, mut lines_with_words, mut lines_total) = (0usize, 0usize, 0usize); + for p in pages { + let img = ffai_media::load_image(Path::new(p)).expect("load image"); + let t = Instant::now(); + let out = engine.recognize(&img, &opts).expect("recognize"); + let ms = t.elapsed().as_secs_f64() * 1e3; + let name = Path::new(p).file_stem().unwrap().to_string_lossy().into_owned(); + let mut s = format!("{{\"page\":\"{}\",\"ms\":{ms:.1},\"lines\":[", esc(&name)); + for (li, line) in out.lines().enumerate() { + lines_total += 1; + if li > 0 { + s.push(','); + } + s.push('['); + let entries: Vec<(String, [f32; 4])> = if line.words.is_empty() { + let b = line.bbox.map_or([0.0; 4], |b| [b.x, b.y, b.x + b.width, b.y + b.height]); + vec![(line.text.clone(), b)] + } else { + lines_with_words += 1; + words_total += line.words.len(); + line.words + .iter() + .map(|w| { + let b = w.bbox.map_or([0.0; 4], |b| [b.x, b.y, b.x + b.width, b.y + b.height]); + (w.text.clone(), b) + }) + .collect() + }; + for (wi, (text, b)) in entries.iter().enumerate() { + if wi > 0 { + s.push(','); + } + let _ = write!( + s, + "[\"{}\",[{:.1},{:.1},{:.1},{:.1}]]", + esc(text), + b[0], + b[1], + b[2], + b[3] + ); + } + s.push(']'); + } + s.push_str("]}"); + println!("{s}"); + } + // The pool the pages actually ran on, beside the time it produced. + eprintln!( + "pages {} total {:.1} ms on {} threads; lines {lines_total}, with words {lines_with_words}, words {words_total}", + pages.len(), + all.elapsed().as_secs_f64() * 1e3, + rayon::current_num_threads() + ); + if std::env::var_os("FFAI_PROFILE").is_some() { + eprint!("{}", ffai_carmenta::profile::profile().report()); + } +} diff --git a/crates/ffai-carmenta/examples/svtr_logits.rs b/crates/ffai-carmenta/examples/svtr_logits.rs new file mode 100644 index 0000000..4f29b73 --- /dev/null +++ b/crates/ffai-carmenta/examples/svtr_logits.rs @@ -0,0 +1,45 @@ +//! Write SVTR's output probabilities for real line crops to a file, so two +//! builds — or one build under two environments — can be compared tensor for +//! tensor. +//! +//! This is the gate for changing how the recognizer COMPUTES without changing +//! what it computes. `svtr_oracle` holds the port against paddle, but its +//! input fixture is a binary that is not in every checkout; this needs only +//! the weights and an image. Run it once with `FFAI_SVTR_FUSED=0 +//! FFAI_DWCONV=0` (candle's own ops, the arm the oracle pinned at 8.857e-5) +//! and once without, then compare the files: the distance between them is +//! what the fast path spends of the oracle's 1e-4 bar. +//! +//! Usage: `svtr_logits ...` +//! The output is every crop's `(T, n_class)` probabilities as little-endian +//! f32, concatenated in argument order. + +use std::io::Write as _; +use std::path::Path; + +use candle_core::Device; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let usage = "usage: svtr_logits ..."; + let root = args.next().ok_or(usage)?; + let image = args.next().ok_or(usage)?; + let out = args.next().ok_or(usage)?; + let dev = Device::Cpu; + let rec = Path::new(&root).join("ppocrv5-mobile-rec"); + let charset = ffai_carmenta::svtr::load_charset(&rec.join("charset.txt"))?; + let m = ffai_carmenta::svtr::load(&rec.join("rec.safetensors"), &dev, charset.len() + 1)?; + let img = ffai_media::load_image(Path::new(&image))?; + let mut f = std::io::BufWriter::new(std::fs::File::create(&out)?); + for b in args { + let v: Vec = b.split(',').map(str::parse).collect::>()?; + let [x0, y0, x1, y1] = v[..] else { return Err(usage.into()) }; + let x = ffai_carmenta::svtr::svtr_input(&img, x0, y0, x1, y1, &dev)?; + let p = m.forward(&x)?.flatten_all()?.to_vec1::()?; + for v in &p { + f.write_all(&v.to_le_bytes())?; + } + eprintln!("{b}: {} values", p.len()); + } + Ok(()) +} diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml index bba4797..fc9491c 100644 --- a/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml @@ -7,6 +7,9 @@ name = "ppocrv5-mobile-det" task = "ocr" description = "PP-OCRv5 mobile det (DBNet/PP-LCNetV3) — staged detector lineage" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "det.safetensors" diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml index c9b07bb..669064b 100644 --- a/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml @@ -24,6 +24,9 @@ name = "ppocrv5-mobile-rec" task = "ocr" description = "PP-OCRv5 mobile rec (SVTR, 18385-class CTC) — opt-in via engine name mobiledet-svtr" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "rec.safetensors" diff --git a/crates/ffai-carmenta/src/boxes.rs b/crates/ffai-carmenta/src/boxes.rs index 28d7d24..b06a722 100644 --- a/crates/ffai-carmenta/src/boxes.rs +++ b/crates/ffai-carmenta/src/boxes.rs @@ -178,6 +178,34 @@ pub fn line_bbox(line: &[DetBox]) -> DetBox { mod tests { use super::*; + fn db(x0: usize, y0: usize, x1: usize, y1: usize) -> Vec { + vec![DetBox { x0, y0, x1, y1, score: 1.0 }] + } + + /// A 3-column, 4-row table emitted column by column comes back row by + /// row; two columns of prose emitted column by column stay as they are. + #[test] + fn tables_read_across_and_prose_columns_do_not() { + let mut table = Vec::new(); + for col in [100usize, 800, 1400] { + for row in 0..4 { + table.push(db(col, 300 + row * 80, col + 100, 330 + row * 80)); + } + } + let bbs: Vec = table.iter().map(|l| l[0]).collect(); + let xs: Vec = grid_order(&bbs).iter().map(|&i| bbs[i].x0).collect(); + assert_eq!(&xs[..6], &[100, 800, 1400, 100, 800, 1400]); + + let mut prose = Vec::new(); + for col in [100usize, 1160] { + for row in 0..6 { + prose.push(db(col, 300 + row * 50, col + 1000, 330 + row * 50)); + } + } + let bbs: Vec = prose.iter().map(|l| l[0]).collect(); + assert_eq!(grid_order(&bbs), (0..bbs.len()).collect::>()); + } + #[test] fn two_words_one_line_group_and_order() { // Two boxes on one row (out of x order), one on a lower row. @@ -455,16 +483,154 @@ pub fn order_reading(lines: Vec>, page_w: usize) -> Vec> // (§8.119, the metric `ffai-bench` reports), and the two losses are // a CJK page we cannot read either way and a 344-character contents // page where +3.5 pp is twelve characters. - if std::env::var("FFAI_ORDER_GATE").as_deref() != Ok("0") + let out = if std::env::var("FFAI_ORDER_GATE").as_deref() != Ok("0") && sparse_scatter(&out) > env_f32("FFAI_ORDER_GATE_T", ORDER_GATE_T) { - return xy_cut(out, page_w, 0); - } + xy_cut(out, page_w, 0) + } else { + out + }; + // Table bands are re-read across their rows at the END of the + // pipeline, after every reorder (`grid_order`). out } } } +/// Lines sharing a row: y-centres within half the median line height. +/// Each row is a list of indices into `lines`, sorted left to right; rows are +/// top to bottom. +fn rows_of(bbs: &[DetBox]) -> Vec> { + let mut hs: Vec = bbs.iter().map(|b| b.y1.saturating_sub(b.y0)).collect(); + hs.sort_unstable(); + let tol = hs.get(hs.len() / 2).copied().unwrap_or(0) as f32 * 0.5; + let cy = |i: usize| (bbs[i].y0 + bbs[i].y1) as f32 / 2.0; + let mut order: Vec = (0..bbs.len()).collect(); + order.sort_by(|&a, &b| cy(a).total_cmp(&cy(b))); + let mut rows: Vec> = Vec::new(); + for i in order { + match rows.last_mut() { + Some(r) if (cy(i) - cy(r[0])).abs() <= tol => r.push(i), + _ => rows.push(vec![i]), + } + } + for r in &mut rows { + r.sort_by_key(|&i| bbs[i].x0); + } + rows +} + +/// Read a TABLE row by row, leaving everything else where the cut put it. +/// +/// The recursive cut reads a ruled table as columns: its gutters are the +/// widest gaps on the page, so it splits there first. PdfCraft's 13-row table +/// came back with every word right and 77 % CER in row order (ocrs, reading +/// rows, 7 %). A column of PROSE beside another lines up row for row as well, +/// so alignment alone cannot be the test. Width can: a table's gutters are +/// wide against the cells either side of them (the 13-row table: cells about +/// 100 px, gutters 260-700 px), while a prose gutter is a sliver of the lines +/// it separates (60 px against 1000). +/// +/// So a GRID ROW is one of two or more cells where every gap between +/// neighbours is at least [`GRID_GAP_3`] times the wider of the two +/// ([`GRID_GAP_2`] for a row of just two), and [`GRID_MIN_ROWS`] or more +/// consecutive grid rows whose cells line up as columns, and are short +/// ([`GRID_MAX_ASPECT`]), form a table band. The +/// band's lines are re-emitted row-major INTO THE SLOTS they already held in +/// the order, so nothing outside a band moves. It runs on the FINISHED line +/// order — after the cut, the sparse-page gate and the §8.157 probe reorder, +/// any of which would otherwise scatter the rows again. +/// +/// `bbs` are the lines' boxes in reading order; the result is the new order as +/// indices into them (the identity when there is no table). +/// `FFAI_ORDER_GRID=0` disables. +#[must_use] +pub fn grid_order(bbs: &[DetBox]) -> Vec { + let identity: Vec = (0..bbs.len()).collect(); + if bbs.len() < 2 * GRID_MIN_ROWS || std::env::var("FFAI_ORDER_GRID").as_deref() == Ok("0") { + return identity; + } + let rows = rows_of(bbs); + let is_grid = |r: &Vec| { + // Two cells need the wider margin: a short last line of one prose + // column beside a full line of the next has a gap about one line + // wide. + let k = if r.len() >= 3 { GRID_GAP_3 } else { GRID_GAP_2 }; + r.len() >= 2 + && r.windows(2).all(|p| { + let (a, b) = (&bbs[p[0]], &bbs[p[1]]); + let gap = b.x0.saturating_sub(a.x1) as f32; + let wider = (a.x1 - a.x0).max(b.x1 - b.x0) as f32; + gap >= k * wider + }) + }; + // Bands: maximal runs of consecutive grid rows that also form COLUMNS — + // the same number of cells in every row, and each row's cell `j` + // overlapping the previous row's cell `j` horizontally. Prose fails this + // even where a few rows pass the gap test (OmniDocBench's three-column + // newspaper pages, whose justified lines the detector splits into + // fragments with wide gaps between them). + let continues = |prev: &Vec, r: &Vec| { + prev.len() == r.len() + && prev.iter().zip(r).all(|(&a, &b)| bbs[a].x0 < bbs[b].x1 && bbs[b].x0 < bbs[a].x1) + }; + // Table cells are SHORT: the median cell is at most [`GRID_MAX_ASPECT`] + // times wider than tall (the 13-row table: about 3.3). Two prose columns + // with a photograph between them pass every test above — wide gap, + // consistent columns (OmniDocBench omni-0124..0152, newspaper pages) — + // but their cells are whole lines, about 15 times wider than tall. + let short_cells = |band: &[Vec]| { + let mut aspects: Vec = band + .iter() + .flatten() + .map(|&k| (bbs[k].x1 - bbs[k].x0) as f32 / (bbs[k].y1 - bbs[k].y0).max(1) as f32) + .collect(); + aspects.sort_by(f32::total_cmp); + aspects.get(aspects.len() / 2).is_some_and(|&a| a <= GRID_MAX_ASPECT) + }; + let mut bands: Vec<&[Vec]> = Vec::new(); + let mut start: Option = None; + for i in 0..=rows.len() { + let extends = i < rows.len() + && is_grid(&rows[i]) + && start.is_some_and(|s| i > s && continues(&rows[i - 1], &rows[i])); + if extends { + continue; + } + if let Some(s) = start.take() + && i - s >= GRID_MIN_ROWS + && short_cells(&rows[s..i]) + { + bands.push(&rows[s..i]); + } + if i < rows.len() && is_grid(&rows[i]) { + start = Some(i); + } + } + if bands.is_empty() { + return identity; + } + let mut perm = identity; + for band in bands { + let row_major: Vec = band.iter().flatten().copied().collect(); + let mut slots = row_major.clone(); + slots.sort_unstable(); // the positions the band's lines held + for (slot, line) in slots.into_iter().zip(row_major) { + perm[slot] = line; + } + } + perm +} + +/// A gap at least this many times the wider neighbouring cell is a table +/// gutter, in a row of two cells and in a row of three or more. +const GRID_GAP_2: f32 = 1.5; +const GRID_GAP_3: f32 = 1.0; +/// Consecutive grid rows that make a table band. +const GRID_MIN_ROWS: usize = 4; +/// Median width/height of a table band's cells, at most. +const GRID_MAX_ASPECT: f32 = 8.0; + /// Mean vertical step between consecutive emitted lines, over the ink extent. /// /// §8.153 decomposed ordering cost into 1.90 pp of SELECTOR REGRET and 2.90 pp diff --git a/crates/ffai-carmenta/src/conv3x3.rs b/crates/ffai-carmenta/src/conv3x3.rs index 730a5d4..983cfb5 100644 --- a/crates/ffai-carmenta/src/conv3x3.rs +++ b/crates/ffai-carmenta/src/conv3x3.rs @@ -112,6 +112,7 @@ pub(crate) fn conv3x3_scalar( /// tail. Requires AVX2 + FMA, checked by the caller. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] pub(crate) unsafe fn conv3x3_avx2( in_p: &[f32], w: &[f32], @@ -239,6 +240,7 @@ fn force_scalar() -> bool { /// gives: "differs from candle" says nothing about which is right. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] pub(crate) unsafe fn conv3x3_simd128( in_p: &[f32], w: &[f32], @@ -354,6 +356,7 @@ impl CustomOp2 for Conv3x3Op { // SAFETY: `in_p` is `pad`'s output for exactly these dims, `wt` has // `c_out*c_in*9` elements (the dims4 above plus the caller's 3x3 // guard), and `out` has `c_out*h*w`. + #[allow(unsafe_code)] unsafe { conv3x3_avx2(&in_p, wt, c_in, c_out, h, w, &mut out) }; return Ok((CpuStorage::F32(out), (1, c_out, h, w).into())); } @@ -371,6 +374,7 @@ impl CustomOp2 for Conv3x3Op { // SAFETY: identical contract to the AVX2 twin — `in_p` is `pad`'s // output for exactly these dims, `wt` has `c_out*c_in*9` elements, // and `out` has `c_out*h*wd`. + #[allow(unsafe_code)] unsafe { conv3x3_simd128(&in_p, wt, c_in, c_out, h, w, &mut out) }; return Ok((CpuStorage::F32(out), (1, c_out, h, w).into())); } @@ -473,6 +477,7 @@ mod tests { let mut b = vec![0f32; co * h * w]; conv3x3_scalar(&in_p, &wt, ci, co, h, w, &mut a); // SAFETY: shapes match `pad`'s output exactly. + #[allow(unsafe_code)] unsafe { conv3x3_avx2(&in_p, &wt, ci, co, h, w, &mut b) }; let scale = a.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-9); let err = a.iter().zip(&b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale; diff --git a/crates/ffai-carmenta/src/deskew.rs b/crates/ffai-carmenta/src/deskew.rs new file mode 100644 index 0000000..a3620c7 --- /dev/null +++ b/crates/ffai-carmenta/src/deskew.rs @@ -0,0 +1,251 @@ +//! Straightening a slightly rotated page (`OcrOptions::deskew`). +//! +//! Detection and recognition both cut axis-aligned boxes. On a page scanned a +//! couple of degrees off square, a long line climbs or falls by more than its +//! own height across the page, so its crop takes in the lines above and below +//! and the recognizer reads noise: PdfCraft measured 0.41 % CER at 0.5 +//! degrees, 0.68 % at 1, 93.3 % at 2 and 98.8 % at 5 on rendered pages, and +//! 61 % on a phone photo tilted 1.5 degrees. +//! +//! The engine measures the skew from the detector's own regions +//! ([`crate::mobiledet::skew_degrees`]), and a page at least [`MIN_DEGREES`] +//! off is rotated back ONCE, read, and its boxes mapped back onto the input. +//! A square page is never resampled, so it reads exactly as before. +//! +//! # Coordinates +//! +//! Boxes stay in the INPUT image's coordinates, as with `auto_orient`. A +//! straightened line is a rectangle; on the input page it is that rectangle +//! turned by the skew, and the reported box is its axis-aligned bounds. +//! `OcrOutput::skew_degrees` carries the angle, and [`rotated_corners`] +//! recovers the turned rectangle from a reported box, which is what a text +//! layer drawn along the page's slope needs. With `auto_orient` the skew is +//! measured on the upright page, so it also takes `OcrOutput::turns`. + +use ffai_core::types::{BoundingBox, ImageBuffer}; + +use crate::par::prelude::*; + +/// Below this the page is read as it is. +pub const MIN_DEGREES: f32 = 0.3; + +/// A rotation about the image centres: the point `q` of the straightened image +/// shows the point `centre + R(angle) (q - straight_centre)` of the input, +/// `R` the rotation by `degrees` clockwise on screen (y down). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Skew { + pub degrees: f32, + pub centre: (f32, f32), + pub straight_centre: (f32, f32), +} + +impl Skew { + /// The input-image point that `(x, y)` of the straightened image shows. + #[must_use] + pub fn back(&self, (x, y): (f32, f32)) -> (f32, f32) { + let (s, c) = self.degrees.to_radians().sin_cos(); + let (dx, dy) = (x - self.straight_centre.0, y - self.straight_centre.1); + (self.centre.0 + c * dx - s * dy, self.centre.1 + s * dx + c * dy) + } + + /// A box of the straightened image as the axis-aligned bounds of its + /// turned rectangle on the input, clamped to the input's `w x h`. + #[must_use] + pub fn back_box(&self, b: BoundingBox, w: f32, h: f32) -> BoundingBox { + let corners = [ + (b.x, b.y), + (b.x + b.width, b.y), + (b.x + b.width, b.y + b.height), + (b.x, b.y + b.height), + ] + .map(|p| self.back(p)); + let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN); + for (x, y) in corners { + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x); + y1 = y1.max(y); + } + let (x0, y0) = (x0.clamp(0.0, w), y0.clamp(0.0, h)); + let (x1, y1) = (x1.clamp(0.0, w), y1.clamp(0.0, h)); + BoundingBox { x: x0, y: y0, width: (x1 - x0).max(0.0), height: (y1 - y0).max(0.0) } + } +} + +/// Rotate `img` by `-degrees` about its centre onto a canvas large enough to +/// hold all of it, so text the skew carried past an edge is not cut off. The +/// new corners are filled with the page's own paper colour (the median of its +/// border pixels), not black, which the detector would read as ink. +#[must_use] +pub fn straighten(img: &ImageBuffer, degrees: f32) -> (ImageBuffer, Skew) { + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let (s, c) = degrees.to_radians().sin_cos(); + let nw = (w as f32 * c.abs() + h as f32 * s.abs()).ceil() as usize; + let nh = (w as f32 * s.abs() + h as f32 * c.abs()).ceil() as usize; + let skew = Skew { + degrees, + centre: (w as f32 / 2.0, h as f32 / 2.0), + straight_centre: (nw as f32 / 2.0, nh as f32 / 2.0), + }; + let paper = paper_colour(img); + let mut out = vec![0u8; nw * nh * bpp]; + out.par_chunks_mut(nw * bpp).enumerate().for_each(|(qy, row)| { + for qx in 0..nw { + // Pixel centres: sample the input where this output centre lands. + let (px, py) = skew.back((qx as f32 + 0.5, qy as f32 + 0.5)); + let (sx, sy) = (px - 0.5, py - 0.5); + let dst = &mut row[qx * bpp..(qx + 1) * bpp]; + if sx < -1.0 || sy < -1.0 || sx > w as f32 || sy > h as f32 { + dst.copy_from_slice(&paper[..bpp]); + continue; + } + let (x0, y0) = (sx.floor(), sy.floor()); + let (fx, fy) = (sx - x0, sy - y0); + let tap = |x: f32, y: f32, ch: usize| -> f32 { + if x < 0.0 || y < 0.0 || x >= w as f32 || y >= h as f32 { + f32::from(paper[ch]) + } else { + f32::from(img.data[(y as usize * w + x as usize) * bpp + ch]) + } + }; + for (ch, d) in dst.iter_mut().enumerate() { + let top = tap(x0, y0, ch) * (1.0 - fx) + tap(x0 + 1.0, y0, ch) * fx; + let bot = tap(x0, y0 + 1.0, ch) * (1.0 - fx) + tap(x0 + 1.0, y0 + 1.0, ch) * fx; + *d = (top * (1.0 - fy) + bot * fy).round().clamp(0.0, 255.0) as u8; + } + } + }); + ( + ImageBuffer { + width: u32::try_from(nw).unwrap_or(u32::MAX), + height: u32::try_from(nh).unwrap_or(u32::MAX), + format: img.format, + data: out, + }, + skew, + ) +} + +/// Per-channel median of the image's border pixels (up to 4 channels). +fn paper_colour(img: &ImageBuffer) -> [u8; 4] { + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let mut out = [255u8; 4]; + if w == 0 || h == 0 { + return out; + } + let mut idx: Vec = Vec::with_capacity(2 * (w + h)); + idx.extend(0..w); + idx.extend((0..w).map(|x| (h - 1) * w + x)); + idx.extend((0..h).map(|y| y * w)); + idx.extend((0..h).map(|y| y * w + w - 1)); + for (ch, o) in out.iter_mut().enumerate().take(bpp) { + let mut v: Vec = idx.iter().map(|&i| img.data[i * bpp + ch]).collect(); + v.sort_unstable(); + *o = v[v.len() / 2]; + } + out +} + +/// The four corners, on the input page, of the turned rectangle whose +/// axis-aligned bounds are `b`, for a page skewed `degrees` and turned +/// upright by `turns` (as `OcrOutput::skew_degrees` and `OcrOutput::turns` +/// report them). In order: top-left, top-right, bottom-right, bottom-left of +/// the rectangle as it reads upright. +/// +/// A `W x H` rectangle turned by `a` has bounds `w = W|cos a| + H|sin a|` and +/// `h = W|sin a| + H|cos a|`, which this solves for `W` and `H`; the centre is +/// shared. Exact up to the clamping at the page edge. On the input page the +/// upright rectangle runs at `degrees - 90 * turns`: the skew is measured on +/// the upright page, and the input is that page turned back counterclockwise. +#[must_use] +pub fn rotated_corners(b: BoundingBox, degrees: f32, turns: u8) -> [(f32, f32); 4] { + let a = degrees - 90.0 * f32::from(turns % 4); + let (s, c) = a.to_radians().sin_cos(); + let (sa, ca) = (s.abs(), c.abs()); + let det = ca * ca - sa * sa; + let (cx, cy) = (b.x + b.width / 2.0, b.y + b.height / 2.0); + let (rw, rh) = if det.abs() < 1e-3 { + (b.width, b.height) + } else { + ( + ((b.width * ca - b.height * sa) / det).max(0.0), + ((b.height * ca - b.width * sa) / det).max(0.0), + ) + }; + [(-rw / 2.0, -rh / 2.0), (rw / 2.0, -rh / 2.0), (rw / 2.0, rh / 2.0), (-rw / 2.0, rh / 2.0)] + .map(|(dx, dy)| (cx + c * dx - s * dy, cy + s * dx + c * dy)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ffai_core::types::PixelFormat; + + #[test] + fn back_box_and_rotated_corners_invert() { + let img = ImageBuffer { width: 2000, height: 3000, format: PixelFormat::Gray8, data: vec![255; 6_000_000] }; + let (straight, skew) = straighten(&img, 2.0); + assert!(straight.width > 2000 && straight.height > 3000); + // A line on the straightened page, mapped back and recovered. + let line = BoundingBox { x: 400.0, y: 900.0, width: 1200.0, height: 40.0 }; + let on_input = skew.back_box(line, 2000.0, 3000.0); + let corners = rotated_corners(on_input, 2.0, 0); + let want = [ + (line.x, line.y), + (line.x + line.width, line.y), + (line.x + line.width, line.y + line.height), + (line.x, line.y + line.height), + ] + .map(|p| skew.back(p)); + for (g, w) in corners.iter().zip(&want) { + assert!((g.0 - w.0).abs() < 0.05 && (g.1 - w.1).abs() < 0.05, "{g:?} vs {w:?}"); + } + } + + /// `auto_orient` and `deskew` together: the page is turned upright, + /// straightened there, and its boxes reach the caller unrotated (tall, for + /// an odd turn) while the angle stays the upright page's. Without `turns` + /// the solve met a tall box with a near-horizontal angle and clamped the + /// line's width to 0. + #[test] + fn rotated_corners_follows_a_turned_page() { + let (w, h) = (800u32, 1200u32); + let img = ImageBuffer { width: w, height: h, format: PixelFormat::Gray8, data: vec![255; (w * h) as usize] }; + let (wf, hf) = (w as f32, h as f32); + for turns in 1..4u8 { + let upright = crate::orient::rotate(&img, turns); + let (uw, uh) = (upright.width as f32, upright.height as f32); + let (straight, skew) = straighten(&upright, 2.0); + let (sw, sh) = (straight.width as f32, straight.height as f32); + let line = BoundingBox { x: 0.15 * sw, y: 0.4 * sh, width: 0.6 * sw, height: 30.0 }; + // A point of the straightened page, on the input: a zero-size box + // maps exactly, boxes being edge coordinates. + let to_input = |p: (f32, f32)| { + let (x, y) = skew.back(p); + let b = crate::orient::unrotate_box(BoundingBox { x, y, width: 0.0, height: 0.0 }, turns, wf, hf); + (b.x, b.y) + }; + let on_input = crate::orient::unrotate_box(skew.back_box(line, uw, uh), turns, wf, hf); + let corners = rotated_corners(on_input, 2.0, turns); + let want = [ + (line.x, line.y), + (line.x + line.width, line.y), + (line.x + line.width, line.y + line.height), + (line.x, line.y + line.height), + ] + .map(to_input); + for (g, w) in corners.iter().zip(&want) { + assert!((g.0 - w.0).abs() < 0.05 && (g.1 - w.1).abs() < 0.05, "turns {turns}: {g:?} vs {w:?}"); + } + } + } + + #[test] + fn straighten_keeps_the_paper_colour() { + let img = ImageBuffer { width: 64, height: 32, format: PixelFormat::Rgb8, data: vec![200; 64 * 32 * 3] }; + let (s, _) = straighten(&img, 5.0); + assert!(s.data.iter().all(|&v| v == 200), "corners must be filled with paper, not black"); + } +} diff --git a/crates/ffai-carmenta/src/dwconv.rs b/crates/ffai-carmenta/src/dwconv.rs new file mode 100644 index 0000000..b530e14 --- /dev/null +++ b/crates/ffai-carmenta/src/dwconv.rs @@ -0,0 +1,300 @@ +//! Depthwise convolution, because candle's CPU backend has none. +//! +//! # Why this exists +//! +//! `Tensor::conv2d` with `groups > 1` does `chunk(groups, 1)`, one +//! `conv2d_single_group` per group, then `cat` (candle-core 0.11 +//! `conv.rs:262`). A depthwise layer has `groups == channels`, so SVTR's +//! 480-channel depthwise runs **480 separate single-channel convolutions**, +//! each through the full im2col machinery, plus 480 allocations and a 480-way +//! concatenation. ffai-diana measured the same mechanism at 72.9x slower per +//! FLOP than the dense convolution beside it (`ffai-diana/src/dwconv.rs`). +//! +//! SVTR has 14 depthwise layers per forward and runs one forward per text +//! line; PdfCraft measured recognition at ~300 ms a line, 97 % of the page. +//! +//! # What this does instead +//! +//! A direct stencil: one pass per output row, one AXPY per tap over the row, +//! parallel over `(batch, channel)` planes. Any kernel size, independent +//! strides and paddings per axis (SVTR uses `(2, 1)` and `(1, 2)`), so the +//! callers no longer compute a stride-1 output and throw half of it away. +//! +//! # Correctness +//! +//! On AVX2+FMA hardware the result is BIT-IDENTICAL to candle's grouped path: +//! see [`plane_fma`]. Elsewhere it is the same products summed in the same +//! order with a separate multiply and add, so it agrees to a tolerance. +//! `tests::matches_candle_grouped` holds both against candle on every shape +//! the two models use. + +use candle_core::{CpuStorage, CustomOp2, Layout, Result, Shape, Tensor}; + +use crate::par::prelude::*; + +struct DwOp { + stride: (usize, usize), + pad: (usize, usize), +} + +/// Output length of a strided, padded, valid convolution along one axis. +fn out_len(n: usize, k: usize, s: usize, p: usize) -> usize { + (n + 2 * p).saturating_sub(k) / s + 1 +} + +/// One output plane. `inp` is one `h x w` input plane, `wt` its `kh x kw` +/// kernel, `out` the `ho x wo` result (overwritten). `madd(w, x, acc)` is the +/// accumulation step: see [`plane_fma`] for why it is a parameter. +#[allow(clippy::too_many_arguments)] +#[inline(always)] +fn plane_with( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + (sh, sw): (usize, usize), + (ph, pw): (usize, usize), + out: &mut [f32], + wo: usize, + madd: impl Fn(f32, f32, f32) -> f32, +) { + out.fill(0.0); + for (oy, orow) in out.chunks_exact_mut(wo).enumerate() { + for ky in 0..kh { + // iy = oy*sh + ky - ph, skipped when it falls in the zero padding. + let Some(iy) = (oy * sh + ky).checked_sub(ph) else { continue }; + if iy >= h { + continue; + } + let irow = &inp[iy * w..(iy + 1) * w]; + for kx in 0..kw { + let wv = wt[ky * kw + kx]; + // Valid ox: 0 <= ox*sw + kx - pw < w. + let lo = pw.saturating_sub(kx).div_ceil(sw); + let hi = if w + pw > kx { (w + pw - kx - 1) / sw + 1 } else { 0 }; + let hi = hi.min(wo); + if lo >= hi { + continue; + } + if sw == 1 { + // Contiguous on both sides: the loop the compiler vectorises. + let src = &irow[lo + kx - pw..hi + kx - pw]; + for (o, &v) in orow[lo..hi].iter_mut().zip(src) { + *o = madd(wv, v, *o); + } + } else { + for ox in lo..hi { + orow[ox] = madd(wv, irow[ox * sw + kx - pw], orow[ox]); + } + } + } + } + } +} + +/// The portable plane: a separate multiply and add per tap. +#[allow(clippy::too_many_arguments)] +fn plane( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + stride: (usize, usize), + pad: (usize, usize), + out: &mut [f32], + wo: usize, +) { + plane_with(inp, h, w, wt, kh, kw, stride, pad, out, wo, |w, x, acc| acc + w * x); +} + +/// The same plane accumulated with FUSED multiply-adds, which is what makes it +/// agree with candle bit for bit rather than to a tolerance. +/// +/// Candle's grouped path im2cols each channel and hands the `kh * kw` taps to +/// gemm, whose microkernel accumulates `acc = fma(a, b, acc)` over them in +/// `(ky, kx)` order from zero. This walks the taps in that same order with the +/// same fused step, and a tap that lands in the padding is skipped where gemm +/// adds `w * 0` — which leaves the accumulator unchanged. `mul_add` is a libm +/// call unless the function is compiled with FMA, hence the target feature. +/// +/// # Safety +/// The CPU must support AVX2 and FMA (checked by the caller). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +#[allow(clippy::too_many_arguments)] +#[allow(unsafe_code)] +unsafe fn plane_fma( + inp: &[f32], + h: usize, + w: usize, + wt: &[f32], + kh: usize, + kw: usize, + stride: (usize, usize), + pad: (usize, usize), + out: &mut [f32], + wo: usize, +) { + plane_with(inp, h, w, wt, kh, kw, stride, pad, out, wo, |w, x, acc| w.mul_add(x, acc)); +} + +fn has_fma() -> bool { + #[cfg(target_arch = "x86_64")] + { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| { + std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") + }) + } + #[cfg(not(target_arch = "x86_64"))] + { + false + } +} + +impl CustomOp2 for DwOp { + fn name(&self) -> &'static str { + "ffai-carmenta-dwconv" + } + + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + ) -> Result<(CpuStorage, Shape)> { + let (CpuStorage::F32(inp), CpuStorage::F32(wt)) = (s1, s2) else { + candle_core::bail!("ffai-carmenta-dwconv: f32 only") + }; + let (n, c, h, w) = l1.shape().dims4()?; + let (wc, one, kh, kw) = l2.shape().dims4()?; + if wc != c || one != 1 { + candle_core::bail!( + "ffai-carmenta-dwconv: weight {:?} is not depthwise for {c} channels", + l2.shape() + ); + } + let inp = match l1.contiguous_offsets() { + Some((o, e)) => &inp[o..e], + None => candle_core::bail!("ffai-carmenta-dwconv: non-contiguous input"), + }; + let wt = match l2.contiguous_offsets() { + Some((o, e)) => &wt[o..e], + None => candle_core::bail!("ffai-carmenta-dwconv: non-contiguous weight"), + }; + let (sh, sw) = self.stride; + let (ph, pw) = self.pad; + if h + 2 * ph < kh || w + 2 * pw < kw || sh == 0 || sw == 0 { + candle_core::bail!("ffai-carmenta-dwconv: kernel larger than padded input"); + } + let (ho, wo) = (out_len(h, kh, sh, ph), out_len(w, kw, sw, pw)); + let mut out = vec![0f32; n * c * ho * wo]; + let fma = has_fma(); + out.par_chunks_mut(ho * wo).enumerate().for_each(|(i, o)| { + let ch = i % c; + let (x, k) = (&inp[i * h * w..(i + 1) * h * w], &wt[ch * kh * kw..(ch + 1) * kh * kw]); + #[cfg(target_arch = "x86_64")] + if fma { + // SAFETY: `has_fma` confirmed AVX2 and FMA at run time. + #[allow(unsafe_code)] + unsafe { plane_fma(x, h, w, k, kh, kw, self.stride, self.pad, o, wo) }; + return; + } + let _ = fma; + plane(x, h, w, k, kh, kw, self.stride, self.pad, o, wo); + }); + Ok((CpuStorage::F32(out), (n, c, ho, wo).into())) + } +} + +/// Depthwise convolution of `x` `(n, c, h, w)` with `weight` `(c, 1, kh, kw)`, +/// zero padding `pad = (ph, pw)` on both sides of each axis and per-axis +/// `stride`. No bias; callers add their own. +pub fn depthwise( + x: &Tensor, + weight: &Tensor, + stride: (usize, usize), + pad: (usize, usize), +) -> Result { + let x = x.contiguous()?; + let weight = weight.contiguous()?; + x.apply_op2_no_bwd(&weight, &DwOp { stride, pad }) +} + +/// Whether a conv with this weight and group count is one [`depthwise`] +/// handles: one group per channel, one input channel per group, f32, CPU. +/// `FFAI_DWCONV=0` sends every call back to candle's grouped path, so the +/// two arms stay one environment variable apart for an A/B. +#[must_use] +pub fn applies(x: &Tensor, weight: &Tensor, groups: usize) -> bool { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + let off = *OFF.get_or_init(|| std::env::var("FFAI_DWCONV").as_deref() == Ok("0")); + !off && groups > 1 + && weight.dims4().is_ok_and(|(oc, icg, _, _)| oc == groups && icg == 1) + && x.dims4().is_ok_and(|(_, c, _, _)| c == groups) + && x.dtype() == candle_core::DType::F32 + && x.device().is_cpu() +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + /// Candle's grouped conv with SVTR's emulation of asymmetric stride and + /// padding: pad, stride-1 conv, keep every `stride`-th row and column. + fn reference(x: &Tensor, w: &Tensor, stride: (usize, usize), pad: (usize, usize)) -> Tensor { + let c = x.dim(1).unwrap(); + let xp = x.pad_with_zeros(2, pad.0, pad.0).unwrap().pad_with_zeros(3, pad.1, pad.1).unwrap(); + let y = xp.conv2d(w, 0, 1, 1, c).unwrap(); + let pick = |y: Tensor, dim: usize, s: usize| { + if s == 1 { + return y; + } + let n = y.dim(dim).unwrap().div_ceil(s); + let idx: Vec = (0..n).map(|i| (i * s) as u32).collect(); + y.index_select(&Tensor::from_vec(idx, n, y.device()).unwrap(), dim).unwrap() + }; + pick(pick(y, 2, stride.0), 3, stride.1) + } + + #[test] + fn matches_candle_grouped() { + let d = Device::Cpu; + // (n, c, h, w, k, stride, pad): every depthwise shape in SVTR and + // MobileDet, plus odd sizes and a batch. + let cases = [ + (1, 16, 24, 161, 3, (1, 1), (1, 1)), + (1, 64, 24, 161, 3, (2, 1), (1, 1)), + (1, 128, 12, 161, 3, (1, 2), (1, 1)), + (1, 240, 12, 81, 5, (1, 1), (2, 2)), + (1, 240, 12, 81, 5, (2, 1), (2, 2)), + (1, 480, 6, 81, 5, (2, 1), (2, 2)), + (1, 32, 37, 53, 3, (2, 2), (1, 1)), + (1, 192, 19, 23, 5, (1, 1), (2, 2)), + (3, 48, 9, 17, 3, (1, 1), (1, 1)), + (2, 8, 5, 4, 5, (1, 1), (2, 2)), + ]; + for (n, c, h, w, k, stride, pad) in cases { + let x = Tensor::randn(0f32, 1.0, (n, c, h, w), &d).unwrap(); + let wt = Tensor::randn(0f32, 1.0, (c, 1, k, k), &d).unwrap(); + let got = depthwise(&x, &wt, stride, pad).unwrap(); + let want = reference(&x, &wt, stride, pad); + assert_eq!(got.dims(), want.dims(), "shape for {:?}", (n, c, h, w, k, stride, pad)); + let diff = (got.clone() - &want).unwrap().abs().unwrap().max_all().unwrap(); + let diff = diff.to_scalar::().unwrap(); + assert!(diff < 1e-4, "max abs diff {diff} for {:?}", (n, c, h, w, k, stride, pad)); + if has_fma() { + let (gv, wv) = (got.flatten_all().unwrap().to_vec1::().unwrap(), + want.flatten_all().unwrap().to_vec1::().unwrap()); + assert!(gv == wv, "FMA arm not bit-identical (max {diff}) for {:?}", + (n, c, h, w, k, stride, pad)); + } + } + } +} diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index b664868..0979cab 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -53,6 +53,26 @@ pub enum DetStage { Composed, } +/// A page read at or above this confidence is taken as upright and +/// `auto_orient` does not run its orientation decision on it. Between the +/// lowest upright page PdfCraft measured (0.806, all capitals) and the highest +/// turned one (0.474, upside down). +const ORIENT_TRUST_ABOVE: f32 = 0.7; + +/// A page's recognition confidence: the line confidences weighted by their +/// characters. `None` when no line has text and a confidence. +fn page_confidence(out: &OcrOutput) -> Option { + let (mut sum, mut n) = (0f64, 0usize); + for line in out.lines() { + if let Some(c) = line.confidence { + let k = line.text.chars().filter(|c| !c.is_whitespace()).count(); + sum += f64::from(c) * k as f64; + n += k; + } + } + (n > 0).then(|| (sum / n as f64) as f32) +} + /// Minimum short side the mobile-det input is scaled UP to; larger images pass /// through untouched (capped at `FFAI_DET_MAX_SIDE`). See /// `image::mobiledet_input` for why this is a floor and not a ceiling — the @@ -68,6 +88,13 @@ fn mobiledet_min_side() -> usize { /// Crop pads for the mobile-det path, as fractions of line height. Zero by /// default because DB's unclip has already expanded the box; sweepable because /// "already expanded" is a claim a gate should get to test. +/// +/// Tested 2026-10-30 and refuted: `FFAI_MDET_PAD_Y=0.3` made PdfCraft's +/// blocky 5x7 "HELLO WORLD" read its `D` correctly instead of as `O`, and wrecked +/// everything else — PdfCraft's 42 pages 0.25 % -> 9.93 % CER (its ruled table +/// 0 -> 55 %), `carmenta-form` 0.41 -> 2.53 % (73 of 240 worse), `carmenta-doc` +/// 30.1 -> 48.5 % (all 32 worse). The extra rows are the neighbouring lines. +/// The 5x7 `D` stays a known misread of a blocky font. fn mobiledet_pads() -> (f32, f32) { static V: OnceLock<(f32, f32)> = OnceLock::new(); *V.get_or_init(|| { @@ -186,6 +213,32 @@ impl CraftCrnn { Self::with_source(rec, det, Source::Offline(weights_root.to_path_buf())) } + /// Read `img` rotated back by `degrees` (see `crate::deskew`) and map the + /// result onto `img`'s own coordinates. + fn recognize_straightened(&self, img: &ImageBuffer, opts: &OcrOptions, degrees: f32) -> Result { + let (straight, sk) = crate::deskew::straighten(img, degrees); + let inner = OcrOptions { deskew: false, ..opts.clone() }; + let mut out = self.recognize(&straight, &inner)?; + let (w, h) = (img.width as f32, img.height as f32); + for block in &mut out.blocks { + if let Some(b) = block.bbox { + block.bbox = Some(sk.back_box(b, w, h)); + } + for line in &mut block.lines { + if let Some(b) = line.bbox { + line.bbox = Some(sk.back_box(b, w, h)); + } + for word in &mut line.words { + if let Some(b) = word.bbox { + word.bbox = Some(sk.back_box(b, w, h)); + } + } + } + } + out.skew_degrees = Some(degrees); + Ok(out) + } + /// Load every model now, returning the error instead of deferring it to /// the first `recognize`. For a batch job, a missing weight file should /// stop the job before page one, not after it has started writing output. @@ -412,6 +465,7 @@ fn load_models(source: &Source, rec: RecStage, det: DetStage) -> Result .map(|m| Resolver { manifest: m, source }) .ok_or_else(|| Error::Model(format!("no model manifest named `{name}` in {searched}"))) }; + #[allow(unsafe_code)] let load_vb = |file: std::path::PathBuf| { unsafe { VarBuilder::from_mmaped_safetensors(&[file], DType::F32, &device) } .map_err(image::candle_err) @@ -489,14 +543,16 @@ struct Resolver<'a> { } impl Resolver<'_> { - /// Resolve the files this loader will open. Offline resolution asks for - /// exactly those, so a deployment never has to ship files a manifest lists - /// only for provenance (a training checkpoint, say). The cache path keeps - /// its existing behaviour of resolving every declared file. + /// Resolve the files this loader will open, and only those, whichever the + /// source: a deployment never has to ship, and a hosted repo never has to + /// carry, files a manifest lists only for provenance (a training + /// checkpoint, say). The cache path used to resolve every declared file, + /// and the first download from the published PP-OCRv5 repo failed on + /// `det.safetensors`, which no engine opens. fn fetch(&self, needed: &[&str]) -> Result { match self.source { Source::Offline(root) => self.manifest.resolve_only_in(root, needed), - _ => self.manifest.fetch(), + _ => self.manifest.fetch_only(needed), } } } @@ -749,13 +805,32 @@ impl OcrEngine for CraftCrnn { // its boxes are mapped back so the caller's coordinates never change // meaning underneath them. if opts.auto_orient { - let o = self.detect_orientation(img)?; let inner = OcrOptions { auto_orient: false, ..opts.clone() }; + // Read the page as given FIRST, and decide orientation only when + // that read is unconvincing. PdfCraft measured the unconditional + // decision at 2.2-2.3x the time of every page, upright ones + // included (the decision is a detection pass plus two dozen crop + // reads, and an upright page was then read in full anyway), while + // the page's own confidence already separates the cases: upright + // pages 0.967-0.997 (0.806 for a page of capitals), pages turned + // 90 / 180 / 270 degrees 0.000 / 0.474 / 0.267, a blank page none. + let first = self.recognize(img, &inner)?; + if page_confidence(&first).is_some_and(|c| c >= ORIENT_TRUST_ABOVE) { + return Ok(first); + } + let o = self.detect_orientation(img)?; if o.turns == 0 { - return self.recognize(img, &inner); + return Ok(first); } let upright = crate::orient::rotate(img, o.turns); let mut out = self.recognize(&upright, &inner)?; + // The decision samples two dozen crops; the two full reads are the + // better witness. A low-confidence upright page the sample misjudged + // keeps its own read. (`None`, no text, orders below any `Some`.) + if page_confidence(&out) <= page_confidence(&first) { + return Ok(first); + } + out.turns = o.turns; let (w, h) = (img.width as f32, img.height as f32); for line in out.blocks.iter_mut().flat_map(|b| b.lines.iter_mut()) { if let Some(b) = line.bbox { @@ -774,7 +849,17 @@ impl OcrEngine for CraftCrnn { } return Ok(out); } + if opts.deskew && self.det != DetStage::MobileDet { + return Err(Error::Other( + "deskew is measured from the mobile detector's text regions; use a mobiledet-* \ + engine" + .into(), + )); + } let m = self.models()?; + // The caller's image, before rule removal: a page found skewed is + // straightened from THIS and read again from the top. + let caller_img = img; // Form rules come out BEFORE detection as well as recognition: a rule // joins the boxes DBNet draws, and the crop the recognizer reads is cut @@ -863,7 +948,7 @@ impl OcrEngine for CraftCrnn { confidence, }); } - return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }] }); + return Ok(OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], skew_degrees: None, turns: 0 }); } // ---- detect ---- @@ -872,6 +957,8 @@ impl OcrEngine for CraftCrnn { // `scale` is carried so the crop stage can map detector coordinates // back to image pixels via `to_img` below; mobile-det already emits // image coordinates, which is why its scale is the identity 2.0. + // Set by the mobile-det arm when `opts.deskew` asks for it. + let mut skew: Option = None; let (lines, region, affinity, mw, scale) = match self.det { DetStage::Craft => { // Detection sees COLOR (chroma contrast is real signal on @@ -937,6 +1024,9 @@ impl OcrEngine for CraftCrnn { } let _ = std::fs::write(format!("{dir}/{tag}.prob"), out); } + if opts.deskew { + skew = crate::mobiledet::skew_degrees(&flat, pw, ph, sx, sy); + } let mut b = crate::mobiledet::boxes_from_probability( &flat, pw, ph, sx, sy, unclip, ); @@ -1067,6 +1157,12 @@ impl OcrEngine for CraftCrnn { // components, so the photographic per-word path has nothing to iterate; // the ink-gap splitter supplies word boundaries inside each line. let det_is_line_level = self.det == DetStage::MobileDet; + // A page found skewed is straightened and read again from the top, + // its boxes mapped back onto the caller's image. Only skewed pages pay + // the second detection pass; a square one carries on with these boxes. + if let Some(deg) = skew.filter(|d| d.abs() >= crate::deskew::MIN_DEGREES) { + return self.recognize_straightened(caller_img, opts, deg); + } // Parallel across lines ONLY when there are enough to amortize the // pool contention with candle's internal rayon: measured, a 1-line // band strip pays 177 ms/line under par_iter vs 82 ms serial, while @@ -1094,6 +1190,8 @@ impl OcrEngine for CraftCrnn { let x1 = (to_img(lb.x1) + px) as usize; let y1 = (to_img(lb.y1) + py) as usize; + // Word geometry, where the recognizer can give it (SVTR). + let mut words: Vec = Vec::new(); let (text, confidence) = match self.rec { RecStage::Svtr => { let (svtr, charset) = @@ -1106,14 +1204,16 @@ impl OcrEngine for CraftCrnn { }; let probs = crate::profile::timed(|p| &p.rec_fwd, || svtr.forward(&crop)) .map_err(image::candle_err)?; - let (mut text, conf) = crate::profile::timed(|p| &p.decode, || match &mask { - Some(mk) => crate::svtr::ctc_greedy_masked(&probs, charset, mk), - None => crate::svtr::ctc_greedy(&probs, charset), + let (mut text, conf, emits, steps) = crate::profile::timed(|p| &p.decode, || { + crate::svtr::ctc_greedy_emits(&probs, charset, mask.as_deref()) }) .map_err(image::candle_err)?; // SVTR's multilingual charset can put a full-width // `0` or `,` into a Latin line; see `fold`. crate::fold::apply(&mut text); + // The crop exactly as `svtr_input` clamped it. + let crop = (x0, y0, x1.min(w), y1.min(h)); + words = crate::words::boxes(&emits, steps, &gray, w, crop, crate::fold::apply); (text, conf) } RecStage::Crnn => { @@ -1267,7 +1367,8 @@ impl OcrEngine for CraftCrnn { } Ok(Some(OcrLine { text, - words: Vec::new(), // line-level recognition; word detail is open work + // Empty for CRNN and the PARSeq path; see `crate::words`. + words, bbox: Some(BoundingBox { x: x0 as f32, y: y0 as f32, @@ -1295,6 +1396,16 @@ impl OcrEngine for CraftCrnn { // z = +1.50 over 16 ABBA-paired rounds, INSIDE a 17-27 % noise // floor. Reverted per revert-if-unproven; §8.100 carries the // numbers so it is not rebuilt. + // + // Re-measured 2026-10-30 after the depthwise/epilogue work removed + // the per-channel conv overhead, with a thread-local "inner ops + // serial" flag and SVTR's matmuls calling gemm with + // `Parallelism::None` inside this fan-out (bit-identical output): + // SLOWER, 1.066x at 24 threads (0/6 ABBA wins) and 1.040x at 8 + // (1/6), on 10 of PdfCraft's pages. Lines are uneven, and the + // inner fan-out is what lets the long ones finish with the rest. + // Serialising the LINES instead is worse still (55.6 s against + // 40.3 s on the 42-page corpus). Both refuted; keep this shape. lines.par_iter().map(&recognize_line).collect::>>()? } else { lines.iter().map(&recognize_line).collect::>>()? @@ -1317,6 +1428,25 @@ impl OcrEngine for CraftCrnn { }; let out_lines = crate::suppress::probe_reorder(out_lines, &probe_stats, lowconf, w as f32, h as f32); + // Table bands read across their rows, on the finished order. + let out_lines = { + let bbs: Vec = out_lines + .iter() + .map(|l| { + let b = l.bbox.unwrap_or(BoundingBox { x: 0.0, y: 0.0, width: 0.0, height: 0.0 }); + boxes::DetBox { + x0: b.x.max(0.0) as usize, + y0: b.y.max(0.0) as usize, + x1: (b.x + b.width).max(0.0) as usize, + y1: (b.y + b.height).max(0.0) as usize, + score: 1.0, + } + }) + .collect(); + let perm = boxes::grid_order(&bbs); + let mut slots: Vec> = out_lines.into_iter().map(Some).collect(); + perm.into_iter().filter_map(|i| slots[i].take()).collect::>() + }; // §48: REGION ROUTING. Runs last, on the finished sequence, so the // reading order §29–§43 earned is spliced into rather than rebuilt. @@ -1372,7 +1502,7 @@ impl OcrEngine for CraftCrnn { // v1: one block per page — paragraph segmentation is the DOCUMENT // milestone's work, and inventing it early would be unearned. - Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }] }) + Ok(OcrOutput { blocks: vec![OcrBlock { lines: out_lines, bbox: None }], skew_degrees: None, turns: 0 }) } } diff --git a/crates/ffai-carmenta/src/forms.rs b/crates/ffai-carmenta/src/forms.rs index 7284394..2464b85 100644 --- a/crates/ffai-carmenta/src/forms.rs +++ b/crates/ffai-carmenta/src/forms.rs @@ -153,6 +153,8 @@ mod tests { fn page(lines: Vec) -> OcrOutput { OcrOutput { blocks: vec![OcrBlock { lines, bbox: None }], + skew_degrees: None, + turns: 0, } } diff --git a/crates/ffai-carmenta/src/image.rs b/crates/ffai-carmenta/src/image.rs index 9cff96a..56a8009 100644 --- a/crates/ffai-carmenta/src/image.rs +++ b/crates/ffai-carmenta/src/image.rs @@ -654,8 +654,58 @@ fn det_max_short() -> usize { /// `carmenta-capture` is unaffected by construction: all 108 clips are 620x200, /// short side 200, so the FLOOR fires and the ceiling never can. The LIVE record /// is untouched. +/// +/// # A cap by page type: refuted (2026-10-30) +/// +/// After the depthwise work made detection cheap enough that the cap matters +/// again (one 300 dpi page, one thread: detection 0.88 s at 1280, 0.47 s at +/// 960, 0.28 s at 736), PdfCraft asked for a lower cap on RENDERED pages, +/// where 736 had looked neutral (0.15 -> 0.18 % CER). Measured with +/// mobiledet-svtr, 1280 vs 736: +/// +/// * the page-type signal does not exist: `content::classify` puts all 32 +/// `carmenta-doc` scans in `Rendered`, beside 40 of PdfCraft's 42 rendered +/// pages — and on those scans 736 is worse on 14 of 32, better on none, by +/// up to +18.4 pp (doc-007); +/// * on rendered pages it is not neutral either: PdfCraft's ruled table +/// 0.00 -> 32.70 %, its capitals page 24.1 -> 31.0 %; OmniDocBench is a wash +/// (65 pages worse, 67 better); +/// * text size does not rescue it: with the median word height scaled to the +/// 736 input as the signal, the scan regressions sit at 9.1-11.9 px while +/// neutral pages sit at 8.7 px. No threshold separates them. +/// +/// So 1280 stays, for every page. [`set_det_max_short`] remains for callers +/// trading accuracy for speed knowingly. pub const DET_MAX_SHORT_DEFAULT: usize = 1280; +/// Largest factor the floor may scale an image UP by (`FFAI_DET_MAX_UP`; +/// unset or 0 = no cap). See [`DET_MAX_UP_DEFAULT`]. +fn det_max_up() -> f32 { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| { + std::env::var("FFAI_DET_MAX_UP") + .ok() + .and_then(|s| s.parse::().ok()) + .map_or(DET_MAX_UP_DEFAULT, |v| if v > 0.0 { v } else { f32::INFINITY }) + }) +} + +/// Default cap on the floor's upscale factor: **4x**. +/// +/// The floor ([`mobiledet_input`]'s `min_side`, 736) exists for SMALL text — +/// receipts with ~10 px glyphs need enlarging before DBNet sees them. Applied +/// to a small IMAGE of normal-sized text it over-enlarges instead: PdfCraft's +/// 344x108 "HELLO WORLD" (28 px glyphs) reached the detector 6.8x larger, the +/// box scores sank to 0.41-0.64 against the 0.6 threshold, and the final `O` +/// fell out of its box (`HELL` + `WORLO`; ocrs read it whole). At 3x or 4x it +/// is one box at 0.91 and reads `HELLO`. +/// +/// Inert elsewhere, measured: `carmenta-form`'s 240 single-line clips (short +/// side 150, 4.9x before) read identically, 0.407 % CER either way; +/// `carmenta-capture` (620x200, 3.68x) is under the cap by construction, so +/// the LIVE record is untouched. +pub const DET_MAX_UP_DEFAULT: f32 = 4.0; + pub fn mobiledet_input( img: &ImageBuffer, min_side: usize, @@ -676,7 +726,7 @@ pub fn mobiledet_input( let short = w.min(h) as f32; let max_short = det_max_short().max(min_side); let ratio = if (short as usize) < min_side { - min_side as f32 / short + (min_side as f32 / short).min(det_max_up()) } else if (short as usize) > max_short { // The CEILING, and the one this policy was missing — see the section // below. Exclusive with the floor: `max_short` is clamped up to diff --git a/crates/ffai-carmenta/src/lib.rs b/crates/ffai-carmenta/src/lib.rs index 7d8bc0c..daa3654 100644 --- a/crates/ffai-carmenta/src/lib.rs +++ b/crates/ffai-carmenta/src/lib.rs @@ -45,6 +45,8 @@ pub mod rules; pub mod harvest; pub mod image; pub mod conv3x3; +pub mod deskew; +pub mod dwconv; /// The LIVE frame reader — **native only.** /// /// It is the one part of Carmenta that is architecturally threaded: a @@ -61,6 +63,7 @@ pub mod live; pub mod par; pub mod suppress; pub mod svtr; +pub mod words; pub mod mobiledet; pub mod onnx_graph; pub mod doclayout; diff --git a/crates/ffai-carmenta/src/live.rs b/crates/ffai-carmenta/src/live.rs index 1e0f6c1..81fdb62 100644 --- a/crates/ffai-carmenta/src/live.rs +++ b/crates/ffai-carmenta/src/live.rs @@ -434,7 +434,11 @@ impl LiveSession { for (k, blocks) in fresh { self.bands[k].cached = blocks; } - OcrOutput { blocks: self.bands.iter().flat_map(|b| b.cached.clone()).collect() } + OcrOutput { + blocks: self.bands.iter().flat_map(|b| b.cached.clone()).collect(), + skew_degrees: None, + turns: 0, + } } else { let full = self.engine.recognize(img, &self.opts)?; self.bands = calibrate_bands(&full, img.height as usize); diff --git a/crates/ffai-carmenta/src/manifests.rs b/crates/ffai-carmenta/src/manifests.rs index 6055db4..963dd8f 100644 --- a/crates/ffai-carmenta/src/manifests.rs +++ b/crates/ffai-carmenta/src/manifests.rs @@ -85,8 +85,11 @@ mod tests { for (name, toml) in EMBEDDED { let on_disk = std::fs::read_to_string(repo.join(format!("{name}.toml"))) .unwrap_or_else(|e| panic!("repository models/{name}.toml: {e}")); + // Line endings normalised: with `core.autocrlf` a Windows checkout + // can hold the two copies with different endings while git + // considers both clean, and that is not drift. assert!( - on_disk == *toml, + on_disk.replace("\r\n", "\n") == toml.replace("\r\n", "\n"), "crates/ffai-carmenta/models/{name}.toml has drifted from models/{name}.toml — \ copy the repository file over the crate's" ); diff --git a/crates/ffai-carmenta/src/mobiledet.rs b/crates/ffai-carmenta/src/mobiledet.rs index a160448..3587b2d 100644 --- a/crates/ffai-carmenta/src/mobiledet.rs +++ b/crates/ffai-carmenta/src/mobiledet.rs @@ -109,11 +109,176 @@ impl RepConv { } fn forward(&self, x: &Tensor) -> Result { - let x = self.conv.forward(x)?; - match self.act { - Some((s, b)) => hardswish(&x)?.affine(s, b), - None => Ok(x), + let cfg = self.conv.config(); + let w = self.conv.weight(); + let y = if crate::dwconv::applies(x, w, cfg.groups) { + // The 14 `dw_conv`s: candle runs a grouped conv as one + // convolution per channel (see `dwconv`). + let p = cfg.padding; + crate::dwconv::depthwise(x, w, (cfg.stride, cfg.stride), (p, p))? + } else if fast() && is_pointwise(w, cfg) { + pointwise(x, w)? + } else { + let y = self.conv.forward(x)?; + return match self.act { + Some((s, b)) => hardswish(&y)?.affine(s, b), + None => Ok(y), + }; + }; + if !fast() { + let y = match self.conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?)?, + None => y, + }; + return match self.act { + Some((s, b)) => hardswish(&y)?.affine(s, b), + None => Ok(y), + }; + } + let bias = match self.conv.bias() { + Some(b) => b.to_vec1::()?, + None => Vec::new(), + }; + y.contiguous()?.apply_op1_no_bwd(&RepEpilogue { bias, act: self.act }) + } +} + +/// `FFAI_MDET_FAST=0` sends the detector back through candle's own ops — 1x1 +/// convs, 3x3 convs, transposed convs and the unfused activation chain — so +/// every fast path below stays one variable away from its oracle. +fn fast() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("FFAI_MDET_FAST").as_deref() != Ok("0")) +} + +fn is_pointwise(w: &Tensor, cfg: &Conv2dConfig) -> bool { + w.dims4().is_ok_and(|(_, _, kh, kw)| (kh, kw) == (1, 1)) + && cfg.stride == 1 + && cfg.padding == 0 + && cfg.groups == 1 + && cfg.dilation == 1 +} + +/// A 1x1 conv in NCHW is `(oc, ic) x (ic, h*w)`: no im2col copy and no layout +/// permute, both of which candle's conv path builds. No bias. +fn pointwise(x: &Tensor, w: &Tensor) -> Result { + let (oc, ic, _, _) = w.dims4()?; + let (n, _, h, wd) = x.dims4()?; + w.reshape((oc, ic))? + .broadcast_matmul(&x.reshape((n, ic, h * wd))?)? + .reshape((n, oc, h, wd)) +} + +/// A conv layer through the fastest path that applies: 1x1 as a matmul, 3x3 +/// through `conv3x3`'s register-tiled kernel (which falls back to candle for +/// any shape it does not cover). +fn conv_fast(conv: &Conv2d, x: &Tensor) -> Result { + if !fast() { + return conv.forward(x); + } + if is_pointwise(conv.weight(), conv.config()) { + let y = pointwise(x, conv.weight())?; + return match conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?), + None => Ok(y), + }; + } + crate::conv3x3::apply(x, conv) +} + +/// A transposed conv whose kernel equals its stride (DBHead's two, k = s = 2) +/// tiles its output without overlap: each input pixel writes its own +/// `kh x kw` block. That is one matmul, `(oc*kh*kw, ic) x (ic, h*w)`, and an +/// interleave, where candle's conv-transpose walks every tap with a dot +/// product per output pixel. +fn conv_transpose_fast(conv: &ConvTranspose2d, x: &Tensor) -> Result { + let cfg = conv.config(); + let w = conv.weight(); + let (ic, oc, kh, kw) = w.dims4()?; + let tiles = cfg.padding == 0 + && cfg.output_padding == 0 + && cfg.dilation == 1 + && kh == cfg.stride + && kw == cfg.stride; + if !fast() || !tiles { + return conv.forward(x); + } + let (n, _, h, wd) = x.dims4()?; + let wt = w.permute((1, 2, 3, 0))?.reshape((oc * kh * kw, ic))?; + let y = wt.broadcast_matmul(&x.reshape((n, ic, h * wd))?)?; // (n, oc*kh*kw, h*w) + let y = y + .reshape((n, oc, kh, kw, h, wd))? + .permute((0, 1, 4, 2, 5, 3))? // (n, oc, h, kh, w, kw) + .reshape((n, oc, h * kh, wd * kw))?; + match conv.bias() { + Some(b) => y.broadcast_add(&b.reshape((1, oc, 1, 1))?), + None => Ok(y), + } +} + +/// Bias, then (when the layer has one) hardswish and the scalar affine, in one +/// pass where candle's chain is up to seven full passes over the activation. +/// The arithmetic is candle's, operation for operation (`affine(m, a)` is +/// `v * m + a` with both cast to f32, `clamp` is maximum-then-minimum), so the +/// result is bit-identical. +struct RepEpilogue { + bias: Vec, + act: Option<(f64, f64)>, +} + +impl candle_core::CustomOp1 for RepEpilogue { + fn name(&self) -> &'static str { + "ffai-mobiledet-epilogue" + } + + fn cpu_fwd( + &self, + st: &candle_core::CpuStorage, + l: &candle_core::Layout, + ) -> Result<(candle_core::CpuStorage, candle_core::Shape)> { + use crate::par::prelude::*; + let candle_core::CpuStorage::F32(inp) = st else { + candle_core::bail!("ffai-mobiledet-epilogue: f32 only") + }; + let Some((o0, o1)) = l.contiguous_offsets() else { + candle_core::bail!("ffai-mobiledet-epilogue: non-contiguous input") + }; + let inp = &inp[o0..o1]; + let (_, c, h, w) = l.shape().dims4()?; + if !self.bias.is_empty() && self.bias.len() != c { + candle_core::bail!("ffai-mobiledet-epilogue: bias length {} for {c} channels", self.bias.len()) } + let sixth = (1.0f64 / 6.0) as f32; + let mut out = vec![0f32; inp.len()]; + let plane = (h * w).max(1); + out.par_chunks_mut(plane).zip(inp.par_chunks(plane)).enumerate().for_each( + |(i, (o, x))| { + let bias = self.bias.get(i % c).copied(); + match (bias, self.act) { + (b, Some((s, a))) => { + let (s, a) = (s as f32, a as f32); + for (o, &x) in o.iter_mut().zip(x) { + let v = match b { + Some(b) => x + b, + None => x, + }; + let t = v * 1.0 + 3.0; + let t = if t < 0.0 { 0.0 } else { t }; + let t = if t > 6.0 { 6.0 } else { t }; + let hs = v * (t * sixth + 0.0); + *o = hs * s + a; + } + } + (Some(b), None) => { + for (o, &x) in o.iter_mut().zip(x) { + *o = x + b; + } + } + (None, None) => o.copy_from_slice(x), + } + }, + ); + Ok((candle_core::CpuStorage::F32(out), l.shape().clone())) } } @@ -138,7 +303,7 @@ impl Se { fn forward(&self, x: &Tensor) -> Result { let w = x.mean_keepdim(D::Minus1)?.mean_keepdim(D::Minus2)?; - let w = self.conv2.forward(&self.conv1.forward(&w)?.relu()?)?; + let w = conv_fast(&self.conv2, &conv_fast(&self.conv1, &w)?.relu()?)?; x.broadcast_mul(&hardsigmoid(&w, self.slope)?) } } @@ -167,7 +332,7 @@ impl RseLayer { } fn forward(&self, x: &Tensor) -> Result { - let x = self.conv.forward(x)?; + let x = conv_fast(&self.conv, x)?; &x + self.se.forward(&x)? } } @@ -249,7 +414,7 @@ impl MobileDet { } cur = blk.pw.forward(&cur)?; if let Some(t) = TAPS.iter().position(|&(n, _, _)| n == BLOCKS[i].0) { - feats.push(self.taps[t].forward(&cur)?); + feats.push(conv_fast(&self.taps[t], &cur)?); } } @@ -279,9 +444,9 @@ impl MobileDet { 1, )?; - let y = self.head1.forward(&fused)?.relu()?; - let y = self.head2.forward(&y)?.relu()?; - candle_nn::ops::sigmoid(&self.head3.forward(&y)?) + let y = conv_fast(&self.head1, &fused)?.relu()?; + let y = conv_transpose_fast(&self.head2, &y)?.relu()?; + candle_nn::ops::sigmoid(&conv_transpose_fast(&self.head3, &y)?) } } @@ -515,14 +680,115 @@ fn box_score_fast(prob: &[f32], w: usize, h: usize, quad: &[(f32, f32); 4]) -> f } } +/// The page's text skew in degrees, read from the probability map's own text +/// regions: positive when lines fall to the right (clockwise on screen, y +/// down). `None` when the map holds too little line-shaped text to say. +/// +/// Each 8-connected region above the binarisation threshold gets the same +/// minimum-area rectangle [`boxes_from_probability`] fits. A region at least +/// [`SKEW_MIN_ASPECT`] times longer than it is tall is a text line (or a long +/// word), and its long axis is the direction the text runs; the estimate is +/// the median of those directions weighted by length, so a page's long lines +/// outvote its stray marks. It is reported only when [`SKEW_AGREE`] of that +/// weight sits within a degree of the median: a page whose regions disagree +/// (a collage, a rotated table beside upright prose) is left as it is. +/// +/// This is the measurement §8.8 did not need: on its corpora deskew moved CER +/// by 0.1 pp. PdfCraft's rendered pages measured 93.3 % CER at 2 degrees and +/// 98.8 % at 5, because an axis-aligned crop of a long tilted line takes in +/// the lines above and below it. +#[must_use] +/// +/// `sx` and `sy` are the map's scale against the image per axis (the same +/// pair [`boxes_from_probability`] takes): the detector input is rounded to +/// multiples of 32, so the two can differ slightly, and an angle is measured +/// in IMAGE pixels. +pub fn skew_degrees(prob: &[f32], w: usize, h: usize, sx: f32, sy: f32) -> Option { + let bin_thr = env_f32("FFAI_DB_BIN", BIN_THRESHOLD); + let mut mask: Vec = prob.iter().map(|&p| p > bin_thr).collect(); + let mut stack = Vec::new(); + let mut pts: Vec<(f32, f32)> = Vec::new(); + // (angle in degrees, weight = long side in map pixels) + let mut votes: Vec<(f32, f32)> = Vec::new(); + for start in 0..w * h { + if !mask[start] { + continue; + } + pts.clear(); + stack.push(start); + mask[start] = false; + while let Some(i) = stack.pop() { + let (x, y) = (i % w, i / w); + pts.push((x as f32, y as f32)); + for dy in -1i32..=1 { + for dx in -1i32..=1 { + let (nx, ny) = (x as i32 + dx, y as i32 + dy); + if nx < 0 || ny < 0 || nx >= w as i32 || ny >= h as i32 { + continue; + } + let j = ny as usize * w + nx as usize; + if mask[j] { + mask[j] = false; + stack.push(j); + } + } + } + } + let Some(rect) = min_area_rect(&convex_hull(pts.clone())) else { continue }; + let (rw, rh) = rect.dims(); + let (long, short) = (rw.max(rh), rw.min(rh)); + if short < MIN_SIDE || long < SKEW_MIN_ASPECT * short { + continue; + } + // Direction of the long side, folded into (-90, 90] degrees. + let (ux, uy) = rect.u; + let (dx, dy) = if rw >= rh { (ux, uy) } else { (-uy, ux) }; + let (dx, dy) = (dx / sx, dy / sy); + let mut a = dy.atan2(dx).to_degrees(); + if a > 90.0 { + a -= 180.0; + } else if a <= -90.0 { + a += 180.0; + } + // Text running vertically is the orientation stage's business. + if a.abs() <= SKEW_MAX_DEGREES { + votes.push((a, long)); + } + } + let total: f32 = votes.iter().map(|v| v.1).sum(); + if votes.len() < 3 || total <= 0.0 { + return None; + } + votes.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut acc = 0.0; + let median = votes + .iter() + .find(|v| { + acc += v.1; + acc >= total / 2.0 + }) + .map_or(0.0, |v| v.0); + let agree: f32 = votes.iter().filter(|v| (v.0 - median).abs() <= 1.0).map(|v| v.1).sum(); + (agree >= SKEW_AGREE * total).then_some(median) +} + +/// A region this many times longer than tall votes on the skew. +const SKEW_MIN_ASPECT: f32 = 4.0; +/// Share of the vote weight that must agree with the median to within a degree. +const SKEW_AGREE: f32 = 0.6; +/// Widest skew [`skew_degrees`] looks for, either way. +pub const SKEW_MAX_DEGREES: f32 = 20.0; + /// Probability map -> word boxes in SOURCE image coordinates. /// /// This is `PaddleOCR`'s `DBPostProcess`, with one deliberate deviation recorded /// rather than hidden: the reference emits rotated quadrilaterals and we emit /// their axis-aligned bounds, because everything downstream — line grouping and -/// crop extraction — is axis-aligned. §8.8 measured deskew at 0.1 pp on these -/// corpora, so the rotation is not currently worth the cascade. The rotated -/// rect is still computed, because its area and perimeter drive the unclip. +/// crop extraction — is axis-aligned. §8.8 measured deskew at 0.1 pp on its +/// corpora; a skewed page is now straightened BEFORE this runs instead (see +/// [`skew_degrees`] and `OcrOptions::deskew`), so the boxes stay axis-aligned. +/// The rotated rect is still computed, because its area and perimeter drive +/// the unclip. /// /// The unclip is not optional. DB is trained against *shrunk* polygons, so a /// raw thresholded region is systematically tighter than the text it covers — diff --git a/crates/ffai-carmenta/src/par.rs b/crates/ffai-carmenta/src/par.rs index d54fc3e..2121a52 100644 --- a/crates/ffai-carmenta/src/par.rs +++ b/crates/ffai-carmenta/src/par.rs @@ -62,6 +62,30 @@ pub mod serial { } } + /// The `par_chunks` of `rayon::prelude`, serially. + pub trait ParallelSliceChunks { + fn par_chunks(&self, n: usize) -> core::slice::Chunks<'_, T>; + } + + impl ParallelSliceChunks for [T] { + #[inline] + fn par_chunks(&self, n: usize) -> core::slice::Chunks<'_, T> { + self.chunks(n) + } + } + + /// The `par_chunks_mut` of `rayon::prelude`, serially. + pub trait ParallelSliceMut { + fn par_chunks_mut(&mut self, n: usize) -> core::slice::ChunksMut<'_, T>; + } + + impl ParallelSliceMut for [T] { + #[inline] + fn par_chunks_mut(&mut self, n: usize) -> core::slice::ChunksMut<'_, T> { + self.chunks_mut(n) + } + } + /// `(0..n).into_par_iter()`, serially. pub trait IntoParallelIterator { type Iter; diff --git a/crates/ffai-carmenta/src/route.rs b/crates/ffai-carmenta/src/route.rs index 620d99a..29f2dc0 100644 --- a/crates/ffai-carmenta/src/route.rs +++ b/crates/ffai-carmenta/src/route.rs @@ -30,7 +30,7 @@ use crate::doclayout::{DocLayout, Region}; use crate::formula::FormulaModel; use crate::table::TableModel; use ffai_core::error::Result; -use ffai_core::types::{BoundingBox, ImageBuffer, OcrLine}; +use ffai_core::types::{BoundingBox, ImageBuffer, OcrLine, OcrWord}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; @@ -191,6 +191,47 @@ fn crop(img: &ImageBuffer, x0: usize, y0: usize, x1: usize, y1: usize) -> ImageB ImageBuffer { width: cw as u32, height: ch as u32, format: img.format, data } } +/// A line's word boxes after its `text[b0..b1]` was spliced into `new_text`. +/// Words wholly outside the splice keep their boxes; the words it touches +/// become one word holding the spliced token, boxed by their union. Empty when +/// the words cannot be found in `text` in order, or none touches the splice: +/// a box that no longer matches its text is worse than no box. +fn splice_words(words: &[OcrWord], text: &str, b0: usize, b1: usize, new_text: &str) -> Vec { + let (mut kept, mut touched) = (Vec::with_capacity(words.len()), Vec::new()); + let mut merged_at = None; + let (mut s, mut e, mut cursor) = (b0, b1, 0); + for w in words { + let Some(at) = text.get(cursor..).and_then(|t| t.find(&w.text)) else { + return Vec::new(); + }; + let (w0, w1) = (cursor + at, cursor + at + w.text.len()); + cursor = w1; + if w1 <= b0 || w0 >= b1 { + kept.push(w.clone()); + } else { + merged_at.get_or_insert(kept.len()); + (s, e) = (s.min(w0), e.max(w1)); + touched.push(w); + } + } + let Some(i) = merged_at else { + return Vec::new(); + }; + // `e` is an offset into `text`; the splice moved everything after `b1`. + let end = (e + new_text.len()).checked_sub(text.len()); + let Some(token) = end.and_then(|end| new_text.get(s..end)) else { + return Vec::new(); + }; + let bbox = touched.iter().filter_map(|w| w.bbox).reduce(|a, b| { + let (x0, y0) = (a.x.min(b.x), a.y.min(b.y)); + let (x1, y1) = ((a.x + a.width).max(b.x + b.width), (a.y + a.height).max(b.y + b.height)); + BoundingBox { x: x0, y: y0, width: x1 - x0, height: y1 - y0 } + }); + let confidence = touched.iter().filter_map(|w| w.confidence).reduce(f32::min); + kept.insert(i, OcrWord { text: token.to_string(), bbox, confidence }); + kept +} + /// Fraction of `l` that lies inside `r`. fn covered(l: &BoundingBox, r: &Region) -> f32 { let (lx1, ly1) = (l.x + l.width, l.y + l.height); @@ -450,6 +491,8 @@ pub fn apply( g.score, g.x0, g.y0, rw, rh, latex.len() ); } + // Only the words the splice touched lose their own boxes. + out[hi].words = splice_words(&out[hi].words, &out[hi].text, b0, b1, &t); out[hi].text = t; spliced.push(g as *const Region); } @@ -608,6 +651,8 @@ pub fn apply( latex.len() ); } + // Only the words the splice touched lose their own boxes. + out[hi].words = splice_words(&out[hi].words, &out[hi].text, b0, b1, &t); out[hi].text = t; } if dbg { @@ -707,3 +752,51 @@ fn bind_cells( } texts } + +#[cfg(test)] +mod tests { + use super::*; + + fn word(text: &str, x: f32) -> OcrWord { + OcrWord { + text: text.into(), + bbox: Some(BoundingBox { x, y: 10.0, width: 20.0, height: 12.0 }), + confidence: Some(0.9), + } + } + + #[test] + fn splice_keeps_the_words_it_does_not_touch() { + let text = "area x2+y2 of the disc"; + let words = [word("area", 0.0), word("x2+y2", 30.0), word("of", 60.0), word("the", 90.0), word("disc", 120.0)]; + // "2+y" (bytes 6..9) becomes LaTeX, inside the second word. + let mut t = text.to_string(); + t.replace_range(6..9, "$2+y$"); + let got = splice_words(&words, text, 6, 9, &t); + let texts: Vec<&str> = got.iter().map(|w| w.text.as_str()).collect(); + assert_eq!(texts, ["area", "x$2+y$2", "of", "the", "disc"]); + assert_eq!(got[0], words[0]); + assert_eq!(got[4], words[4]); + assert_eq!(got[1].bbox, words[1].bbox); + } + + #[test] + fn splice_over_several_words_merges_their_boxes() { + let text = "so a + b = c holds"; + let words = [word("so", 0.0), word("a", 30.0), word("+", 60.0), word("b", 90.0), word("=", 120.0), word("c", 150.0), word("holds", 180.0)]; + let mut t = text.to_string(); + t.replace_range(3..12, "$a+b=c$"); + let got = splice_words(&words, text, 3, 12, &t); + let texts: Vec<&str> = got.iter().map(|w| w.text.as_str()).collect(); + assert_eq!(texts, ["so", "$a+b=c$", "holds"]); + let b = got[1].bbox.unwrap(); + assert!((b.x - 30.0).abs() < 1e-6 && (b.width - 140.0).abs() < 1e-6, "{b:?}"); + } + + #[test] + fn splice_clears_words_that_do_not_match_the_text() { + let text = "alpha beta"; + let words = [word("alpha", 0.0), word("gamma", 30.0)]; + assert!(splice_words(&words, text, 0, 3, "$al$pha beta").is_empty()); + } +} diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index 09d8b87..59559c7 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -136,6 +136,72 @@ fn swish(x: &Tensor) -> Result { x * candle_nn::ops::sigmoid(x)? } +/// `FFAI_SVTR_FUSED=0` sends the backbone through the unfused candle chain +/// and candle's conv for 1x1s, so the fast path stays one variable from its +/// oracle. +fn fused() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("FFAI_SVTR_FUSED").as_deref() != Ok("0")) +} + +/// See [`Svtr::block_epilogue`]. Each parameter is per channel, or one value +/// broadcast to every channel. +struct Epilogue { + bias: Vec, + s1: Vec, + b1: Vec, + s2: Vec, + b2: Vec, +} + +impl candle_core::CustomOp1 for Epilogue { + fn name(&self) -> &'static str { + "ffai-svtr-epilogue" + } + + fn cpu_fwd( + &self, + s: &candle_core::CpuStorage, + l: &candle_core::Layout, + ) -> Result<(candle_core::CpuStorage, candle_core::Shape)> { + use crate::par::prelude::*; + let candle_core::CpuStorage::F32(inp) = s else { + candle_core::bail!("ffai-svtr-epilogue: f32 only") + }; + let Some((o0, o1)) = l.contiguous_offsets() else { + candle_core::bail!("ffai-svtr-epilogue: non-contiguous input") + }; + let inp = &inp[o0..o1]; + let (_, c, h, w) = l.shape().dims4()?; + let ok = |p: &[f32]| p.len() == 1 || p.len() == c; + if !(ok(&self.bias) && ok(&self.s1) && ok(&self.b1) && ok(&self.s2) && ok(&self.b2)) { + candle_core::bail!("ffai-svtr-epilogue: parameter length is neither 1 nor {c}") + } + let at = |p: &[f32], ch: usize| if p.len() == 1 { p[0] } else { p[ch] }; + // Candle's `/ 6.0` is an affine with this multiplier. + let sixth = (1.0f64 / 6.0) as f32; + let mut out = vec![0f32; inp.len()]; + let plane = (h * w).max(1); + out.par_chunks_mut(plane).zip(inp.par_chunks(plane)).enumerate().for_each( + |(i, (o, x))| { + let ch = i % c; + let (bias, s1, b1) = (at(&self.bias, ch), at(&self.s1, ch), at(&self.b1, ch)); + let (s2, b2) = (at(&self.s2, ch), at(&self.b2, ch)); + for (o, &x) in o.iter_mut().zip(x) { + let v = x + bias; + let v = v * s1 + b1; + let t = v * 1.0 + 3.0; + let t = if t < 0.0 { 0.0 } else { t }; + let t = if t > 6.0 { 6.0 } else { t }; + let v = (t * v) * sixth + 0.0; + *o = v * s2 + b2; + } + }, + ); + Ok((candle_core::CpuStorage::F32(out), l.shape().clone())) + } +} + impl Svtr { pub fn new(vb: VarBuilder<'static>, n_class: usize) -> Result { Ok(Self { vb, n_class, cache: RwLock::new(HashMap::new()) }) @@ -185,15 +251,53 @@ impl Svtr { bias: bool, ) -> Result { let w = self.fetch(&format!("{name}.w_0"))?; + let y = if crate::dwconv::applies(x, &w, groups) { + // 14 of the 30 backbone convs. Candle splits a grouped conv into + // one convolution per channel (see `dwconv`), and the kernel takes + // the asymmetric stride directly instead of computing a stride-1 + // output and keeping every other row of it. + crate::dwconv::depthwise(x, &w, stride, pad)? + } else { + self.dense(x, w, stride, pad, groups)? + }; + if bias { + let b = self.fetch(&format!("{name}.b_0"))?; + y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?) + } else { + Ok(y) + } + } + + /// Candle's conv for everything that is not depthwise: the stem and the + /// pointwise and head convs. + fn dense( + &self, + x: &Tensor, + w: Tensor, + stride: (usize, usize), + pad: (usize, usize), + groups: usize, + ) -> Result { let (oc, icg, kh, kw) = w.dims4()?; - let _ = (oc, icg, kh, kw); + if fused() && (kh, kw) == (1, 1) && stride == (1, 1) && pad == (0, 0) && groups == 1 { + // A 1x1 conv in NCHW IS a matmul — `(oc, ic) x (ic, h*w)` — with + // no im2col copy and no layout permute on either side. Candle's + // conv path builds both; 16 of the 30 backbone convs are 1x1. + let (n, _, h, wd) = x.dims4()?; + let y = w.reshape((oc, icg))?.broadcast_matmul(&x.reshape((n, icg, h * wd))?)?; + return y.reshape((n, oc, h, wd)); + } // candle's conv2d takes ONE stride and ONE padding; the recorded model // uses asymmetric values (e.g. (2,1) to collapse height faster than - // width). Emulate by padding manually and striding per axis. + // width). Emulate by padding manually and striding per axis. A + // symmetric stride (the (2,2) stem) is candle's own, so it computes a + // quarter of the outputs rather than all of them. let x = x.pad_with_zeros(2, pad.0, pad.0)?.pad_with_zeros(3, pad.1, pad.1)?; - let cfg = candle_nn::Conv2dConfig { padding: 0, stride: 1, dilation: 1, groups, + let native = if stride.0 == stride.1 { stride.0 } else { 1 }; + let cfg = candle_nn::Conv2dConfig { padding: 0, stride: native, dilation: 1, groups, ..Default::default() }; let y = candle_nn::Conv2d::new(w, None, cfg).forward(&x)?; + let stride = if native > 1 { (1, 1) } else { stride }; let y = if stride.0 > 1 { let n = y.dim(2)?.div_ceil(stride.0); y.index_select( @@ -220,12 +324,7 @@ impl Svtr { } else { y }; - if bias { - let b = self.fetch(&format!("{name}.b_0"))?; - y.broadcast_add(&b.reshape((1, b.elem_count(), 1, 1))?) - } else { - Ok(y) - } + Ok(y) } fn bn(&self, x: &Tensor, name: &str) -> Result { @@ -240,6 +339,28 @@ impl Svtr { .broadcast_add(&shift.reshape((1, c, 1, 1))?) } + /// `lab(hardswish(lab(conv + bias)))` in ONE pass over the activation. + /// + /// The unfused form is eight candle ops — bias add, two affine blocks of + /// two ops each, and hardswish's add, clamp, multiply and divide — each a + /// full read, write and allocation of the block's output. PdfCraft's page + /// profile put those passes at a third of the backbone. The arithmetic + /// below is candle's own, operation for operation (`+ 3.0` is + /// `v * 1 + 3`, `clamp` is maximum-then-minimum, `/ 6.0` is + /// `v * (1/6) + 0`), and Rust never contracts a multiply-add on its own, so + /// the result is bit-identical to the unfused chain. `FFAI_SVTR_FUSED=0` + /// restores that chain for an A/B. + fn block_epilogue(&self, x: &Tensor, conv: &str, i: &mut usize) -> Result { + let v = |t: Tensor| t.flatten_all().and_then(|t| t.to_vec1::()); + let bias = v(self.fetch(&format!("{conv}.b_0"))?)?; + let s1 = v(self.fetch(&format!("learnable_affine_block_{i}.w_0"))?)?; + let b1 = v(self.fetch(&format!("learnable_affine_block_{i}.w_1"))?)?; + let s2 = v(self.fetch(&format!("learnable_affine_block_{}.w_0", *i + 1))?)?; + let b2 = v(self.fetch(&format!("learnable_affine_block_{}.w_1", *i + 1))?)?; + *i += 2; + x.contiguous()?.apply_op1_no_bwd(&Epilogue { bias, s1, b1, s2, b2 }) + } + /// Learnable affine block: per-channel scale then shift, consumed in order. fn lab(&self, x: &Tensor, i: &mut usize) -> Result { let w = self.fetch(&format!("learnable_affine_block_{i}.w_0"))?; @@ -337,15 +458,28 @@ impl Svtr { /// (1, 3, 48, W) normalised to [-1, 1] -> (1, T, n_class) probabilities. pub fn forward(&self, x: &Tensor) -> Result { + // Three stages on the profile's `rec_*` rows, so `FFAI_PROFILE=1` + // says WHERE a recognizer forward goes rather than one opaque number. + let x = crate::profile::timed(|p| &p.rec_cnn, || self.backbone(x))?; + let (x, shortcut) = crate::profile::timed(|p| &p.rec_rnn, || self.encoder(&x))?; + crate::profile::timed(|p| &p.rec_head, || self.head(&x, &shortcut)) + } + + fn backbone(&self, x: &Tensor) -> Result { let mut x = self.bn(&self.conv(x, "conv2d_0", (2, 2), (1, 1), 1, false)?, "batch_norm2d_0")?; let mut lab = 0usize; for st in backbone_spec() { match st { Stage::Blk(Blk(name, groups, stride, pad)) => { - x = self.conv(&x, name, stride, pad, groups, true)?; - x = self.lab(&x, &mut lab)?; - x = hardswish(&x)?; - x = self.lab(&x, &mut lab)?; + if fused() { + x = self.conv(&x, name, stride, pad, groups, false)?; + x = self.block_epilogue(&x, name, &mut lab)?; + } else { + x = self.conv(&x, name, stride, pad, groups, true)?; + x = self.lab(&x, &mut lab)?; + x = hardswish(&x)?; + x = self.lab(&x, &mut lab)?; + } } Stage::Se(Se(reduce, expand)) => { let s = x.mean_keepdim(2)?.mean_keepdim(3)?; @@ -355,6 +489,12 @@ impl Svtr { } } } + Ok(x) + } + + /// Neck pool, the two SVTR blocks and the final norm. Returns the encoder + /// output and the pooled SHORTCUT the head concatenates with it. + fn encoder(&self, x: &Tensor) -> Result<(Tensor, Tensor)> { // Neck pool k=(3,2) s=(3,2): H 3->1, W 80->40 (the timesteps). let (_, _, h, w) = x.dims4()?; let x = x.avg_pool2d_with_stride((3, 2), (3, 2))?; @@ -369,11 +509,15 @@ impl Svtr { let x = self.enc_block(&x, "layer_norm_0", "linear_0", "linear_1", "layer_norm_1", "linear_2", "linear_3")?; let x = self.enc_block(&x, "layer_norm_2", "linear_4", "linear_5", "layer_norm_3", "linear_6", "linear_7")?; let x = self.ln(&x, "layer_norm_4", LN_EPS_FINAL)?; + Ok((x, shortcut)) + } + + fn head(&self, x: &Tensor, shortcut: &Tensor) -> Result { // Head: (B,N,C) -> (B,C,1,N) -> conv133 -> cat(shortcut) -> conv134 -> conv135 let (b, n, c) = x.dims3()?; let hd = x.reshape((b, 1, n, c))?.permute((0, 3, 1, 2))?.contiguous()?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_133", (1, 1), (0, 0), 1, false)?, "batch_norm2d_148")?)?; - let hd = Tensor::cat(&[&shortcut, &hd], 1)?; + let hd = Tensor::cat(&[shortcut, &hd], 1)?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_134", (1, 1), (0, 1), 1, false)?, "batch_norm2d_149")?)?; let hd = swish(&self.bn(&self.conv(&hd, "conv2d_135", (1, 1), (0, 0), 1, false)?, "batch_norm2d_150")?)?; let x = hd.squeeze(2)?.transpose(1, 2)?.contiguous()?; @@ -564,6 +708,71 @@ pub fn ctc_greedy_masked( Ok((out, crate::crnn::line_confidence(&kept))) } +/// [`ctc_greedy`] (or [`ctc_greedy_masked`] when `mask` is given) that also +/// returns every emission with the frames it spans, for word geometry. +/// +/// The per-frame choice is made exactly as in those two — the tensor argmax +/// and max unmasked, the same masked fold otherwise — and the string and +/// confidence are built by the same loop, so both are byte-identical to theirs. +/// Returns `(text, confidence, emissions, timesteps)`. +pub fn ctc_greedy_emits( + probs: &Tensor, + charset: &[String], + mask: Option<&[bool]>, +) -> Result<(String, Option, Vec, usize)> { + let (_, t, n) = probs.dims3()?; + let (best, conf): (Vec, Vec) = match mask { + None => { + let best = probs.argmax(D::Minus1)?.flatten_all()?.to_vec1::()?; + let conf = probs.max(D::Minus1)?.flatten_all()?.to_vec1::()?; + (best.into_iter().map(|k| k as usize).collect(), conf) + } + Some(mask) => { + if mask.len() != n || !mask[0] { + return Err(candle_core::Error::Msg(format!( + "class mask has {} entries; the head has {n} classes and the blank must be allowed", + mask.len() + ))); + } + let flat = probs.flatten_all()?.to_vec1::()?; + (0..t) + .map(|i| { + flat[i * n..(i + 1) * n] + .iter() + .enumerate() + .filter(|(j, _)| mask[*j]) + .fold((0usize, f32::NEG_INFINITY), |b, (j, &p)| if p > b.1 { (j, p) } else { b }) + }) + .unzip() + } + }; + let mut out = String::new(); + let mut kept = Vec::new(); + let mut emits: Vec = Vec::new(); + let mut prev = usize::MAX; + // Whether the run `prev` belongs to was pushed as an emission. + let mut open = false; + for i in 0..t { + let k = best[i]; + if k != 0 && k != prev { + open = false; + if let Some(s) = charset.get(k - 1) { + out.push_str(s); + emits.push(crate::words::Emit { text: s.clone(), t0: i, t1: i, p: conf[i] }); + open = true; + } + kept.push(conf[i]); + } else if k != 0 + && open + && let Some(e) = emits.last_mut() + { + e.t1 = i; + } + prev = k; + } + Ok((out, crate::crnn::line_confidence(&kept), emits, t)) +} + /// One emitted run of characters, with WHERE ON THE LINE it came from. #[derive(Debug, Clone)] pub struct CharSpan { @@ -643,6 +852,7 @@ use std::sync::{PoisonError, RwLock}; /// Load from a safetensors file produced by `tools/carmenta_svtr_prepare.py`. pub fn load(path: &std::path::Path, device: &Device, n_class: usize) -> Result { + #[allow(unsafe_code)] let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[path], DType::F32, device)? }; Svtr::new(vb, n_class) } diff --git a/crates/ffai-carmenta/src/words.rs b/crates/ffai-carmenta/src/words.rs new file mode 100644 index 0000000..4482252 --- /dev/null +++ b/crates/ffai-carmenta/src/words.rs @@ -0,0 +1,260 @@ +//! Word boxes for a line-level recognizer, from where its decoder emitted. +//! +//! The document engines recognise whole LINES, so `OcrLine::words` used to be +//! empty and a caller drawing a word-level text layer had to re-split the line +//! itself. PdfCraft's attempt with [`crate::image::split_ink_words`] matched +//! the word count on 38 of 472 lines: a fixed gap rule cannot tell a word space +//! from the space inside "r n" at one size and "W A" at another. +//! +//! The recognizer already knows. CTC walks the crop left to right one +//! timestep per column band, so each emitted character comes with the frames +//! that emitted it, and the decoded SPACES say which characters form a word. +//! That fixes how many words there are and roughly where each one sits; the +//! pixels then say exactly where its ink is: +//! +//! 1. between two neighbouring words, the cut is the widest ink-free column +//! run between the last frame of one and the first frame of the next (the +//! midpoint when there is none); +//! 2. inside its cuts, a word's box is snapped to its ink — the first and last +//! ink columns, and the band of ink rows carrying the most ink, which keeps +//! the neighbouring lines that a DB line box (unclipped to ~2.5x the ink +//! height) reaches into out of it. +//! +//! The words' texts are the line's text split at its spaces, so joining them +//! with single spaces gives the line back. + +use ffai_core::types::{BoundingBox, OcrWord}; + +/// One decoded character run and the CTC frames (inclusive) that emitted it. +#[derive(Debug, Clone, PartialEq)] +pub struct Emit { + pub text: String, + pub t0: usize, + pub t1: usize, + /// Probability of the emitting class at its first frame. + pub p: f32, +} + +/// Ink mask over a crop of the grayscale image: a pixel is ink when it is +/// further from the crop's median (its paper) than 40 % of the crop's 98th +/// percentile deviation, and at least 10 grey levels — so light-on-dark and +/// grey-on-grey lines work as well as black on white. +struct Ink { + x0: usize, + y0: usize, + w: usize, + h: usize, + on: Vec, +} + +impl Ink { + fn new(gray: &[f32], img_w: usize, (x0, y0, x1, y1): (usize, usize, usize, usize)) -> Option { + let (w, h) = (x1.checked_sub(x0)?, y1.checked_sub(y0)?); + if w == 0 || h == 0 { + return None; + } + let px = |x: usize, y: usize| gray[(y0 + y) * img_w + x0 + x]; + let mut sample: Vec = (0..h).flat_map(|y| (0..w).map(move |x| (x, y))).step_by(7).map(|(x, y)| px(x, y)).collect(); + sample.sort_by(f32::total_cmp); + let paper = sample[sample.len() / 2]; + let mut dev: Vec = sample.iter().map(|v| (v - paper).abs()).collect(); + dev.sort_by(f32::total_cmp); + let p98 = dev[(dev.len() * 98 / 100).min(dev.len() - 1)]; + let thr = (0.4 * p98).max(10.0); + let on = (0..h).flat_map(|y| (0..w).map(move |x| (x, y))).map(|(x, y)| (px(x, y) - paper).abs() > thr).collect(); + Some(Self { x0, y0, w, h, on }) + } + + fn at(&self, x: usize, y: usize) -> bool { + self.on[y * self.w + x] + } + + /// Whether column `x` (crop coordinates) has ink in rows `ys`. + fn col(&self, x: usize, ys: std::ops::Range) -> bool { + ys.into_iter().any(|y| self.at(x, y)) + } +} + +/// Split `emits` (one line's decode) into words and box each one on the page. +/// +/// `crop` is the rectangle the recognizer read, in image pixels, `t` its +/// number of timesteps, and `fold` the same per-string normalisation applied +/// to the line's text, so word texts agree with it. +#[must_use] +pub fn boxes( + emits: &[Emit], + t: usize, + gray: &[f32], + img_w: usize, + crop: (usize, usize, usize, usize), + fold: impl Fn(&mut String), +) -> Vec { + // Group emissions into words at the spaces. + let mut groups: Vec<&[Emit]> = Vec::new(); + let mut start = None; + for (i, e) in emits.iter().enumerate() { + let space = e.text.chars().all(char::is_whitespace); + match (space, start) { + (true, Some(s)) => { + groups.push(&emits[s..i]); + start = None; + } + (false, None) => start = Some(i), + _ => {} + } + } + if let Some(s) = start { + groups.push(&emits[s..]); + } + if groups.is_empty() || t == 0 { + return Vec::new(); + } + let (cx0, cy0, cx1, cy1) = crop; + let cw = (cx1 - cx0) as f32; + // Frame edges in crop columns. + let left = |f: usize| ((f as f32 / t as f32) * cw) as usize; + let right = |f: usize| ((((f + 1) as f32 / t as f32) * cw).ceil() as usize).min(cx1 - cx0); + let ink = Ink::new(gray, img_w, crop); + + // Cuts between neighbouring words. + let mut cuts = vec![0usize]; + for pair in groups.windows(2) { + let a = right(pair[0].last().map_or(0, |e| e.t1)); + let b = left(pair[1].first().map_or(0, |e| e.t0)).max(a); + let cut = ink + .as_ref() + .and_then(|ink| { + // Widest ink-free run of columns in [a, b). + let (mut best, mut run_start, mut best_len) = (None, None, 0usize); + for x in a..=b.min(ink.w) { + let free = x < b.min(ink.w) && !ink.col(x, 0..ink.h); + match (free, run_start) { + (true, None) => run_start = Some(x), + (false, Some(s)) => { + if x - s > best_len { + best_len = x - s; + best = Some((s + x) / 2); + } + run_start = None; + } + _ => {} + } + } + best + }) + .unwrap_or((a + b) / 2); + cuts.push(cut.max(*cuts.last().unwrap_or(&0))); + } + cuts.push(cx1 - cx0); + + groups + .iter() + .enumerate() + .map(|(i, g)| { + let mut text: String = g.iter().map(|e| e.text.as_str()).collect(); + fold(&mut text); + let conf = g.iter().map(|e| e.p).sum::() / g.len() as f32; + let (lo, hi) = (cuts[i], cuts[i + 1].max(cuts[i] + 1)); + let bbox = ink.as_ref().and_then(|ink| snap(ink, lo, hi)).unwrap_or(BoundingBox { + x: (cx0 + lo) as f32, + y: cy0 as f32, + width: (hi - lo) as f32, + height: (cy1 - cy0) as f32, + }); + OcrWord { text, bbox: Some(bbox), confidence: Some(conf) } + }) + .collect() +} + +/// The ink box of crop columns `[lo, hi)`: the band of ink rows carrying the +/// most ink (gaps up to a tenth of the crop height bridged, so an accent stays +/// with its letter), then the first and last ink columns in that band. +/// +/// Most ink, not the densest row: the DB line box reaches into the lines above +/// and below, and a neighbour's full row of descenders can out-count any one +/// row of this word, but never this word's whole band. +fn snap(ink: &Ink, lo: usize, hi: usize) -> Option { + let hi = hi.min(ink.w); + if lo >= hi { + return None; + } + let rows: Vec = (0..ink.h).map(|y| (lo..hi).filter(|&x| ink.at(x, y)).count()).collect(); + let tol = (ink.h / 10).max(1); + // (top, bottom, mass) of each gap-bridged band. + let mut bands: Vec<(usize, usize, usize)> = Vec::new(); + let mut cur: Option<(usize, usize, usize)> = None; + for (y, &n) in rows.iter().enumerate() { + if n == 0 { + continue; + } + cur = match cur { + Some((t, b, m)) if y - b <= tol + 1 => Some((t, y, m + n)), + Some(band) => { + bands.push(band); + Some((y, y, n)) + } + None => Some((y, y, n)), + }; + } + bands.extend(cur); + let &(top, bot, _) = bands.iter().max_by_key(|b| b.2)?; + let cols: Vec = (lo..hi).filter(|&x| ink.col(x, top..bot + 1)).collect(); + let (&x0, &x1) = (cols.first()?, cols.last()?); + Some(BoundingBox { + x: (ink.x0 + x0) as f32, + y: (ink.y0 + top) as f32, + width: (x1 + 1 - x0) as f32, + height: (bot + 1 - top) as f32, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn e(text: &str, t0: usize, t1: usize) -> Emit { + Emit { text: text.into(), t0, t1, p: 0.9 } + } + + /// Two ink blobs on white with a wide gap; the decoder put its space in + /// the gap. Each word's box must be exactly its blob. + #[test] + fn words_snap_to_their_ink() { + let (w, h) = (100usize, 30usize); + let mut gray = vec![255f32; w * h]; + for y in 10..20 { + for x in (10..30).chain(60..90) { + gray[y * w + x] = 0.0; + } + } + // 10 timesteps over the 100-px crop: "ab" in frames 1-2, space 4-5, + // "cd" in frames 6-8. + let emits = [e("a", 1, 1), e("b", 2, 2), e(" ", 4, 5), e("c", 6, 7), e("d", 8, 8)]; + let words = boxes(&emits, 10, &gray, w, (0, 0, w, h), |_| {}); + assert_eq!(words.iter().map(|w| w.text.as_str()).collect::>(), ["ab", "cd"]); + let b0 = words[0].bbox.unwrap(); + let b1 = words[1].bbox.unwrap(); + assert_eq!((b0.x, b0.y, b0.width, b0.height), (10.0, 10.0, 20.0, 10.0)); + assert_eq!((b1.x, b1.y, b1.width, b1.height), (60.0, 10.0, 30.0, 10.0)); + } + + /// A neighbouring line's descenders inside the crop's top rows stay out. + #[test] + fn a_neighbouring_line_is_not_in_the_box() { + let (w, h) = (60usize, 40usize); + let mut gray = vec![255f32; w * h]; + for y in 0..3 { + for x in 5..55 { + gray[y * w + x] = 0.0; // the line above + } + } + for y in 15..28 { + for x in 10..50 { + gray[y * w + x] = 0.0; + } + } + let words = boxes(&[e("word", 1, 8)], 10, &gray, w, (0, 0, w, h), |_| {}); + let b = words[0].bbox.unwrap(); + assert_eq!((b.x, b.y, b.width, b.height), (10.0, 15.0, 40.0, 13.0)); + } +} diff --git a/crates/ffai-cli/CHANGELOG.md b/crates/ffai-cli/CHANGELOG.md index ad523af..b94f87d 100644 --- a/crates/ffai-cli/CHANGELOG.md +++ b/crates/ffai-cli/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-cli-v0.8.0...ffai-cli-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-cli-v0.7.0...ffai-cli-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-cli/Cargo.toml b/crates/ffai-cli/Cargo.toml index f0ae1f6..456aed8 100644 --- a/crates/ffai-cli/Cargo.toml +++ b/crates/ffai-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "ffai-cli" description = "The ffai binary — a thin shell over the FFai library crates" # follows its dependencies. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-core/CHANGELOG.md b/crates/ffai-core/CHANGELOG.md index 76449f9..9c444a9 100644 --- a/crates/ffai-core/CHANGELOG.md +++ b/crates/ffai-core/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-core-v0.8.0...ffai-core-v0.9.0) - 2026-10-31 + +### Added + +- `OcrOptions::deskew`, and `OcrOutput::skew_degrees` reporting the angle a straightened page ran at +- `OcrOutput::turns`: the clockwise quarter turns `auto_orient` applied, which gives each box its reading direction +- `threads::available_threads` and `threads::affinity_pool`, which honour the process affinity mask on Windows + +### Changed + +- **BREAKING:** `OcrOptions` gains `deskew` and `OcrOutput` gains `skew_degrees` and `turns`, which breaks construction by struct literal and exhaustive destructuring +- `unsafe_code` is allowed at each site, so path and git dependents build without warnings + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-core-v0.7.2...ffai-core-v0.8.0) - 2026-09-25 ### Changed diff --git a/crates/ffai-core/Cargo.toml b/crates/ffai-core/Cargo.toml index 506ba3f..b381a53 100644 --- a/crates/ffai-core/Cargo.toml +++ b/crates/ffai-core/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai shared types, engine traits, and the engine registry" # published, so the crates.io copy of 0.6.1 cannot build ffai-diana — a gap # invisible locally, where path dependencies mask it, and caught only by # `cargo publish --dry-run`. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -24,6 +24,11 @@ candle-core = { workspace = true } rayon = { workspace = true } thiserror = { workspace = true } +# `threads::available_threads` reads the process affinity mask, which std does +# not on Windows. Already in the tree (0.61) through rayon/std consumers. +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_Threading"] } + [features] # GPU backends forward straight to candle. Note: today these pull C/CUDA/Metal # tooling — the pure-Rust replacement path (cool-japan/oxicuda) is tracked in diff --git a/crates/ffai-core/src/engine.rs b/crates/ffai-core/src/engine.rs index b7a69d2..1d7b944 100644 --- a/crates/ffai-core/src/engine.rs +++ b/crates/ffai-core/src/engine.rs @@ -269,10 +269,22 @@ pub struct OcrOptions { /// way. Scanners store portrait forms sideways and phones photograph in /// any orientation; read sideways, a page silently comes back as a few /// low-confidence lines. Returned boxes stay in the INPUT image's - /// coordinates. Off by default: it costs an extra detection pass. + /// coordinates. Off by default. An engine may read the page as given + /// first and decide orientation only when that read is unconvincing, so + /// on an upright page it can cost nothing beyond the read itself. /// /// Engines that do not implement it must return an error when it is set. pub auto_orient: bool, + /// Measure how far the page's text runs off horizontal and, when it is + /// skewed, straighten the page before reading it. A scan a couple of + /// degrees off square otherwise reads as noise: an axis-aligned crop of a + /// long tilted line takes in the lines above and below it. A square page + /// is read exactly as without it. Boxes stay in the INPUT image's + /// coordinates; `OcrOutput::skew_degrees` reports the angle when the page + /// was straightened. + /// + /// Engines that do not implement it must return an error when it is set. + pub deskew: bool, } /// Detection options (Diana). diff --git a/crates/ffai-core/src/fastmath.rs b/crates/ffai-core/src/fastmath.rs index c85026d..7085692 100644 --- a/crates/ffai-core/src/fastmath.rs +++ b/crates/ffai-core/src/fastmath.rs @@ -854,6 +854,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { { if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { // SAFETY: both features probed immediately above. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_avx2(row, max) }; } } @@ -867,6 +868,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { // stops LLVM cold no matter what the baseline is. // // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_neon(row, max) }; } #[cfg(target_arch = "wasm32")] @@ -879,6 +881,7 @@ pub fn exp_sub_sum_inplace(row: &mut [f32], max: f32) -> f32 { // requirement rather than an upgrade. // // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { exp_sub_sum_simd128(row, max) }; } #[allow(unreachable_code)] @@ -910,6 +913,7 @@ pub fn exp_sub_sum_scalar(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_avx2(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1001,6 +1005,7 @@ unsafe fn exp_sub_sum_avx2(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_neon(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1080,6 +1085,7 @@ unsafe fn exp_sub_sum_neon(row: &mut [f32], max: f32) -> f32 { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn exp_sub_sum_simd128(row: &mut [f32], max: f32) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1155,17 +1161,20 @@ pub fn max_f32(xs: &[f32]) -> f32 { { if is_x86_feature_detected!("avx2") { // SAFETY: probed immediately above. + #[allow(unsafe_code)] return unsafe { max_f32_avx2(xs) }; } } #[cfg(target_arch = "aarch64")] { // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { max_f32_neon(xs) }; } #[cfg(target_arch = "wasm32")] { // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] return unsafe { max_f32_simd128(xs) }; } #[allow(unreachable_code)] @@ -1186,6 +1195,7 @@ pub fn max_f32_scalar(xs: &[f32]) -> f32 { /// Caller must have verified `avx2`. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] +#[allow(unsafe_code)] unsafe fn max_f32_avx2(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1222,6 +1232,7 @@ unsafe fn max_f32_avx2(xs: &[f32]) -> f32 { /// `neon` is a compile-time guarantee on `aarch64` here. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn max_f32_neon(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1251,6 +1262,7 @@ unsafe fn max_f32_neon(xs: &[f32]) -> f32 { /// `simd128` is a compile-time guarantee on `wasm32` here. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn max_f32_simd128(xs: &[f32]) -> f32 { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1297,6 +1309,7 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { { if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { // SAFETY: probed immediately above. + #[allow(unsafe_code)] unsafe { gelu_tanh_avx2(xs) }; return; } @@ -1304,12 +1317,14 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { #[cfg(target_arch = "aarch64")] { // SAFETY: `neon` is a compile-time guarantee on this target. + #[allow(unsafe_code)] unsafe { gelu_tanh_neon(xs) }; return; } #[cfg(target_arch = "wasm32")] { // SAFETY: `simd128` is a compile-time guarantee on this target. + #[allow(unsafe_code)] unsafe { gelu_tanh_simd128(xs) }; return; } @@ -1335,6 +1350,7 @@ pub fn gelu_tanh_inplace(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,fma")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_avx2(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1412,6 +1428,7 @@ unsafe fn gelu_tanh_avx2(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_neon(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: @@ -1482,6 +1499,7 @@ unsafe fn gelu_tanh_neon(xs: &mut [f32]) { // goes stale the moment the arithmetic changes, and hides nothing. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +#[allow(unsafe_code)] unsafe fn gelu_tanh_simd128(xs: &mut [f32]) { // SAFETY: whole-body wrapper inserted by the edition-2024 // `unsafe_op_in_unsafe_fn` migration. This block adds no new obligation: diff --git a/crates/ffai-core/src/lib.rs b/crates/ffai-core/src/lib.rs index e9cd7ad..017f693 100644 --- a/crates/ffai-core/src/lib.rs +++ b/crates/ffai-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod error; pub mod fastmath; pub mod fastops; pub mod registry; +pub mod threads; pub mod types; /// The shared tensor framework (Hugging Face candle). @@ -71,6 +72,7 @@ pub const fn best_device() -> candle::Device { // Linux and break on Windows - so the lint is allowed rather than obeyed. // Caught only when CI first ran on Linux; it never fired on the author's box. #[allow(clippy::missing_const_for_fn)] +#[allow(unsafe_code)] pub fn release_load_arena() { #[cfg(windows)] { diff --git a/crates/ffai-core/src/threads.rs b/crates/ffai-core/src/threads.rs new file mode 100644 index 0000000..dc68620 --- /dev/null +++ b/crates/ffai-core/src/threads.rs @@ -0,0 +1,68 @@ +//! How many threads this process may actually run on. +//! +//! `std::thread::available_parallelism` ignores the process affinity mask on +//! Windows: under `start /affinity 4` (one core) it still reports every +//! logical CPU. rayon sizes its global pool the same way, and candle hands +//! gemm `Parallelism::Rayon(physical cores)` on every matmul, so a process +//! pinned to one core ran two dozen threads on it. +//! +//! `PdfCraft` measured what sizing the pool to the mask is worth (ABBA, output +//! identical): on 8 cores, 0.927x the time (10/10 rounds) and peak memory +//! 1 959 -> 1 240 MB; on 1 core, time neutral and memory 1 285 -> 747 MB. It +//! could not do it itself, because reading the mask is an FFI call and it +//! forbids `unsafe`. This module is that one call, behind a safe function. +//! +//! Use [`affinity_pool`] and run the engine inside it: +//! +//! ```no_run +//! let pool = ffai_core::threads::affinity_pool().expect("thread pool"); +//! pool.install(|| { +//! // engine.recognize(&image, &options) — every rayon task the engine and +//! // candle spawn now runs on this pool's threads. +//! }); +//! ``` + +/// The number of CPUs this process may run on: the affinity mask on Windows, +/// `available_parallelism` elsewhere (which already honours affinity and +/// cgroup quotas on Linux). At least 1. +#[must_use] +pub fn available_threads() -> usize { + #[cfg(windows)] + if let Some(n) = windows_affinity() { + return n; + } + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) +} + +#[cfg(windows)] +fn windows_affinity() -> Option { + use windows_sys::Win32::System::Threading::{GetCurrentProcess, GetProcessAffinityMask}; + let (mut process, mut system) = (0usize, 0usize); + // SAFETY: `GetCurrentProcess` returns a pseudo-handle that is always valid + // for the calling process and needs no closing; both out-pointers point at + // live, writable `usize`s for the duration of the call. + #[allow(unsafe_code)] + let ok = unsafe { GetProcessAffinityMask(GetCurrentProcess(), &raw mut process, &raw mut system) }; + // The mask covers the process's processor group (at most 64 CPUs). + (ok != 0 && process != 0).then(|| process.count_ones() as usize) +} + +/// A rayon pool with [`available_threads`] threads. +/// +/// # Errors +/// When rayon cannot spawn the threads. +pub fn affinity_pool() -> Result { + rayon::ThreadPoolBuilder::new().num_threads(available_threads()).build() +} + +#[cfg(test)] +mod tests { + #[test] + fn at_least_one_and_at_most_the_machine() { + let n = super::available_threads(); + assert!(n >= 1); + // On Windows the mask can only narrow the count, never widen it. + #[cfg(windows)] + assert!(n <= std::thread::available_parallelism().map_or(usize::MAX, std::num::NonZeroUsize::get)); + } +} diff --git a/crates/ffai-core/src/types.rs b/crates/ffai-core/src/types.rs index 60fbb8f..019179d 100644 --- a/crates/ffai-core/src/types.rs +++ b/crates/ffai-core/src/types.rs @@ -355,6 +355,27 @@ pub struct OcrBlock { #[derive(Debug, Clone, Default, PartialEq)] pub struct OcrOutput { pub blocks: Vec, + /// The angle the page's text ran at, in degrees clockwise on screen + /// (y down), when the engine straightened it before reading + /// (`OcrOptions::deskew`); `None` when the page was read as given. + /// + /// Boxes are always in the INPUT image's coordinates. On a straightened + /// page each line and word box is the axis-aligned bounds of its + /// rectangle turned by this angle, so the rectangle itself — what a text + /// layer drawn along the page's slope wants — is recoverable from the box + /// and the angle (`ffai_carmenta::deskew::rotated_corners`). + /// + /// On a page also turned upright (`turns` non-zero) the angle is measured + /// on the upright page, not on the input. + pub skew_degrees: Option, + /// Clockwise quarter turns (0-3) the engine applied to the input to read + /// it upright (`OcrOptions::auto_orient`); 0 when it was read as given. + /// + /// Boxes stay in the INPUT image's coordinates either way, so this is + /// what gives each box its reading direction: at 1 turn the text in a box + /// runs bottom to top, at 2 right to left upside down, at 3 top to + /// bottom. A 90-degree and a 270-degree page give the same boxes. + pub turns: u8, } impl OcrOutput { @@ -388,6 +409,8 @@ impl OcrOutput { .collect(), bbox: None, }], + skew_degrees: None, + turns: 0, } } diff --git a/crates/ffai-diana/CHANGELOG.md b/crates/ffai-diana/CHANGELOG.md index fbbb74d..2729e7f 100644 --- a/crates/ffai-diana/CHANGELOG.md +++ b/crates/ffai-diana/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-diana-v0.8.0...ffai-diana-v0.9.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.8.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-diana-v0.7.8...ffai-diana-v0.8.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-diana/Cargo.toml b/crates/ffai-diana/Cargo.toml index 3eb26d2..476d6d5 100644 --- a/crates/ffai-diana/Cargo.toml +++ b/crates/ffai-diana/Cargo.toml @@ -3,7 +3,7 @@ name = "ffai-diana" description = "Diana — FFai's detection component (YOLO26 on candle, pure Rust)" # 0.7.3 is a documentation-only republish: README restructured to the shared # Remade With Rust page layout. No code, API or output change. -version = "0.8.0" +version = "0.9.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -35,7 +35,7 @@ ffai-core = { workspace = true } # Declared directly rather than inherited: `default-features = false` cannot # override a workspace dependency's defaults, and setting it at the workspace # level would silently strip `fetch` from every other crate that expects it. -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } candle-core = { workspace = true } candle-nn = { workspace = true } serde = { workspace = true } diff --git a/crates/ffai-diana/README.md b/crates/ffai-diana/README.md index 18d1404..8676549 100644 --- a/crates/ffai-diana/README.md +++ b/crates/ffai-diana/README.md @@ -33,9 +33,9 @@ cargo add ffai-diana ffai-core ffai-media ```toml [dependencies] -ffai-diana = "0.7" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-diana = "0.9" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Detect diff --git a/crates/ffai-diana/examples/alloc_ab.rs b/crates/ffai-diana/examples/alloc_ab.rs index 4ef1c30..52785e3 100644 --- a/crates/ffai-diana/examples/alloc_ab.rs +++ b/crates/ffai-diana/examples/alloc_ab.rs @@ -76,11 +76,19 @@ fn main() -> Result<(), Box> { let reps: usize = std::env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(200); let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).parent().and_then(|p| p.parent()).unwrap(); - let img = ffai_media::load_image(&root.join("corpora/clips/diana-coco/coco-032.png"))?; let engine = ffai_diana::engine::Yolo26::build(&tier, ffai_diana::image::Geometry::Rect, root.join("models")); let opts = DetectOptions { confidence: 0.25, ..Default::default() }; + // `alloc_ab corpus`: every `coco-*.png` in the Diana corpus, + // so the allocator sees the spread of image content and detection counts + // rather than one image's allocation pattern repeated. + if std::env::args().nth(3).as_deref() == Some("corpus") { + return corpus(&engine, &opts, &tier, reps, &root.join("corpora/clips/diana-coco")); + } + + let img = ffai_media::load_image(&root.join("corpora/clips/diana-coco/coco-032.png"))?; + // Warm: first call pays lazy weight materialisation and the allocator's // own arena bring-up, neither of which is what we are comparing. let warm = engine.detect(&img, &opts)?; @@ -111,3 +119,49 @@ fn main() -> Result<(), Box> { ); Ok(()) } + +/// Corpus mode: one timed sample is one full PASS over every image, so the +/// sample carries the corpus's whole allocation mix. Images are decoded once, +/// before timing, so the decoder's own allocations are not in the number. +fn corpus( + engine: &ffai_diana::engine::Yolo26, + opts: &DetectOptions, + tier: &str, + passes: usize, + dir: &std::path::Path, +) -> Result<(), Box> { + let mut paths: Vec<_> = std::fs::read_dir(dir)? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.extension().is_some_and(|x| x == "png") + && p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("coco-")) + }) + .collect(); + paths.sort(); + let imgs = paths.iter().map(|p| ffai_media::load_image(p)).collect::, _>>()?; + + // Warm pass, which also takes the work-parity checksum over the corpus. + let (mut n_det, mut sum) = (0usize, 0f64); + for img in &imgs { + for d in &engine.detect(img, opts)?.detections { + n_det += 1; + sum += f64::from(d.x0) + f64::from(d.y0) + f64::from(d.x1) + f64::from(d.y1) + f64::from(d.confidence); + } + } + + let mut times = Vec::with_capacity(passes); + for _ in 0..passes { + let t = Instant::now(); + for img in &imgs { + engine.detect(img, opts)?; + } + times.push(t.elapsed().as_secs_f64() * 1e3); + } + times.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let (min, med) = (times[0], times[times.len() / 2]); + println!( + "ARM={ARM} tier={tier} images={} passes={passes} min={min:.3} med={med:.3} ndet={n_det} sum={sum:.6}", + imgs.len() + ); + Ok(()) +} diff --git a/crates/ffai-media/CHANGELOG.md b/crates/ffai-media/CHANGELOG.md index d2769e8..4388838 100644 --- a/crates/ffai-media/CHANGELOG.md +++ b/crates/ffai-media/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-media-v0.6.8...ffai-media-v0.7.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.6.8](https://github.com/Remade-With-Rust/FFAI/compare/ffai-media-v0.6.7...ffai-media-v0.6.8) - 2026-09-25 ### Added diff --git a/crates/ffai-media/Cargo.toml b/crates/ffai-media/Cargo.toml index bf61c57..b21956e 100644 --- a/crates/ffai-media/Cargo.toml +++ b/crates/ffai-media/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai media ingest/egress — demux, decode, resample (rff-backed) # rusty_jpeg 0.1 -> 0.2 picks up the progressive-JPEG fix. A dependency # change with no API change, so a patch bump -- but a load-bearing one: # 0.6.1 cannot decode 35 of OmniDocBench's 236 pages. -version = "0.6.8" +version = "0.7.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-media/README.md b/crates/ffai-media/README.md index 271468b..d764da9 100644 --- a/crates/ffai-media/README.md +++ b/crates/ffai-media/README.md @@ -23,7 +23,7 @@ Both dispatch on the bytes' magic, not the file name: 169 of OmniDocBench's Engl ## PDF pages as a viewer shows them (`pdf` feature) ```toml -ffai-media = { version = "0.6", features = ["pdf"] } +ffai-media = { version = "0.7", features = ["pdf"] } ``` ```rust diff --git a/crates/ffai-mercury-wasm/CHANGELOG.md b/crates/ffai-mercury-wasm/CHANGELOG.md index e683ecc..f846f1f 100644 --- a/crates/ffai-mercury-wasm/CHANGELOG.md +++ b/crates/ffai-mercury-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-wasm-v0.2.0...ffai-mercury-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-wasm-v0.1.3...ffai-mercury-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-mercury-wasm/Cargo.toml b/crates/ffai-mercury-wasm/Cargo.toml index ef334ce..84dfbe6 100644 --- a/crates/ffai-mercury-wasm/Cargo.toml +++ b/crates/ffai-mercury-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-mercury-wasm" description = "Mercury in the browser — pure-Rust Whisper speech recognition compiled to WebAssembly, no whisper.cpp, no ONNX runtime" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -28,7 +28,7 @@ ffai-core = { workspace = true } # the constructor this crate is built on, first exists in 1.0.2. A `^1.0.1` # requirement resolves happily in the workspace and then fails to compile the # moment it is packaged, which is exactly how it failed. -ffai-mercury = { path = "../ffai-mercury", version = "1.0.5", default-features = false } +ffai-mercury = { path = "../ffai-mercury", version = "1.1.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-mercury/CHANGELOG.md b/crates/ffai-mercury/CHANGELOG.md index 0e3b882..eba2f9c 100644 --- a/crates/ffai-mercury/CHANGELOG.md +++ b/crates/ffai-mercury/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-v1.0.5...ffai-mercury-v1.1.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [1.0.5](https://github.com/Remade-With-Rust/FFAI/compare/ffai-mercury-v1.0.4...ffai-mercury-v1.0.5) - 2026-09-25 ### Other diff --git a/crates/ffai-mercury/Cargo.toml b/crates/ffai-mercury/Cargo.toml index f5262cf..3183529 100644 --- a/crates/ffai-mercury/Cargo.toml +++ b/crates/ffai-mercury/Cargo.toml @@ -6,7 +6,7 @@ description = "Mercury — FFai's voice component: pure-Rust ASR (Whisper/Whispe # to 0.6.1 -- the API is purely additive, but this crate advertises byte-stable # determinism, so a changed waveform is a compatibility break for anyone who # pinned bytes. Its dependencies stay at the published 0.6.1. -version = "1.0.5" +version = "1.1.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -18,7 +18,7 @@ categories = ["multimedia::audio", "science", "accessibility"] [dependencies] ffai-core = { workspace = true } -ffai-models = { path = "../ffai-models", version = "0.6.6", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep +ffai-models = { path = "../ffai-models", version = "0.7.0", default-features = false } # not `workspace = true`: cargo forbids overriding default-features on an inherited dep candle-nn = { workspace = true } candle-transformers = { workspace = true } tokenizers = { workspace = true } diff --git a/crates/ffai-mercury/README.md b/crates/ffai-mercury/README.md index bf56798..dbf08c7 100644 --- a/crates/ffai-mercury/README.md +++ b/crates/ffai-mercury/README.md @@ -11,9 +11,9 @@ Mercury is [FFai](https://github.com/Remade-With-Rust/FFAI)'s voice component. S ```toml [dependencies] -ffai-mercury = "0.7" -ffai-core = "0.8" -ffai-media = "0.6" +ffai-mercury = "1.1" +ffai-core = "0.9" +ffai-media = "0.7" ``` ## Transcribe @@ -258,7 +258,7 @@ MIT OR Apache-2.0. Model weights carry their own licenses, surfaced at selection first use. It is also the single largest thing in the dependency graph: ```toml -ffai-mercury = { version = "0.7", default-features = false } # ship your own weights +ffai-mercury = { version = "1.1", default-features = false } # ship your own weights ``` | build | dependencies | C crypto | diff --git a/crates/ffai-models/CHANGELOG.md b/crates/ffai-models/CHANGELOG.md index 8394846..120f507 100644 --- a/crates/ffai-models/CHANGELOG.md +++ b/crates/ffai-models/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-models-v0.6.6...ffai-models-v0.7.0) - 2026-10-31 + +### Added + +- `ModelManifest::fetch_only`: resolve and download only the named files + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. +- Drops `dirs` (and with it the MPL `option-ext`) + +### Fixed + +- An engine fetching from the hub downloads only the files it opens, as offline resolution already did + ## [0.6.6](https://github.com/Remade-With-Rust/FFAI/compare/ffai-models-v0.6.5...ffai-models-v0.6.6) - 2026-09-25 ### Added diff --git a/crates/ffai-models/Cargo.toml b/crates/ffai-models/Cargo.toml index e9bb220..37676f5 100644 --- a/crates/ffai-models/Cargo.toml +++ b/crates/ffai-models/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai model weight management — manifests, cache, licensing surf # excludes the 0.7.0 every other crate in this release now needs. Left at # 0.6.2 it would put TWO ffai-core versions in every downstream tree and the # shared `ffai_core` types would stop being the same types. -version = "0.6.6" +version = "0.7.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -20,7 +20,6 @@ categories = ["caching", "filesystem"] ffai-core = { workspace = true } serde = { workspace = true } toml = { workspace = true } -dirs = { workspace = true } # Optional: the Hugging Face downloader. # # It is 226 crates — reqwest, hyper, rustls, aws-lc-sys — and a consumer that diff --git a/crates/ffai-models/src/lib.rs b/crates/ffai-models/src/lib.rs index bb13778..3b961be 100644 --- a/crates/ffai-models/src/lib.rs +++ b/crates/ffai-models/src/lib.rs @@ -154,8 +154,29 @@ impl ModelManifest { /// still network I/O the first time, which is why benchmarks warm the /// cache outside any timed region (see docs/benchmarking.md). pub fn fetch(&self) -> Result { + let all: Vec<&str> = self.files.iter().map(|f| f.name.as_str()).collect(); + self.fetch_only(&all) + } + + /// [`Self::fetch`], for just the named files — the cache-and-download twin + /// of [`Self::resolve_only_in`]. + /// + /// A manifest can list files no engine loads (`ppocrv5-mobile-det` lists + /// its training checkpoint, the source of the fused weights), and a hosted + /// repo need not carry them. Fetching every declared file made the first + /// download of the document engine fail on a file it never opens. A name + /// the manifest does not declare is an error: it has no checksum. + pub fn fetch_only(&self, needed: &[&str]) -> Result { + for name in needed { + if !self.files.iter().any(|f| f.name == *name) { + return Err(Error::Model(format!( + "model `{}` declares no file `{name}` — it has no checksum to verify against", + self.name + ))); + } + } let mut files = BTreeMap::new(); - for file in &self.files { + for file in self.files.iter().filter(|f| needed.contains(&f.name.as_str())) { if let Some(path) = self.local_path(&file.name) { verify_checksum(&path, file)?; files.insert(file.name.clone(), path); @@ -299,11 +320,32 @@ pub fn cache_dir() -> PathBuf { if let Ok(dir) = std::env::var("FFAI_CACHE") { return PathBuf::from(dir); } - dirs::cache_dir() + os_cache_dir() .unwrap_or_else(std::env::temp_dir) .join("ffai") } +/// The platform's per-user cache directory, by the rules `dirs::cache_dir` +/// applies: `%LOCALAPPDATA%` on Windows, `~/Library/Caches` on macOS, +/// `$XDG_CACHE_HOME` (when absolute) or `~/.cache` elsewhere. +/// +/// Written out rather than taken from `dirs`, whose `dirs-sys` pulls +/// `option-ext`, an MPL-2.0 crate: one call site was enough to turn a +/// permissive-only `cargo deny` policy red for every consumer (`PdfCraft`, +/// 2026-10-30). +fn os_cache_dir() -> Option { + let abs = |v: std::ffi::OsString| Some(PathBuf::from(v)).filter(|p| p.is_absolute()); + if cfg!(windows) { + std::env::var_os("LOCALAPPDATA").and_then(abs) + } else if cfg!(target_os = "macos") { + std::env::var_os("HOME").and_then(abs).map(|h| h.join("Library/Caches")) + } else { + std::env::var_os("XDG_CACHE_HOME") + .and_then(abs) + .or_else(|| std::env::var_os("HOME").and_then(abs).map(|h| h.join(".cache"))) + } +} + #[cfg(test)] mod tests { use super::*; @@ -430,4 +472,24 @@ mod tests { assert!(err.contains("declares no file `nope.bin`"), "{err}"); let _ = std::fs::remove_dir_all(&root); } + + /// The download path refuses an undeclared name before touching the + /// network or the cache, exactly as the offline path does. + #[test] + fn fetch_only_refuses_undeclared_names_before_any_lookup() { + let m = ModelManifest::from_toml( + r#" + name = "tiny-test" + task = "ocr" + license = "MIT" + hf_repo = "example/does-not-matter" + + [[files]] + name = "w.bin" + "#, + ) + .unwrap(); + let err = m.fetch_only(&["nope.bin"]).unwrap_err().to_string(); + assert!(err.contains("declares no file `nope.bin`"), "{err}"); + } } diff --git a/crates/ffai-wasm/CHANGELOG.md b/crates/ffai-wasm/CHANGELOG.md index ae702cd..abf76fb 100644 --- a/crates/ffai-wasm/CHANGELOG.md +++ b/crates/ffai-wasm/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-wasm-v0.2.0...ffai-wasm-v0.3.0) - 2026-10-31 + +### Changed + +- Requires `ffai-core` 0.9: `OcrOptions` gained `deskew` and `OcrOutput` gained `skew_degrees` and `turns`, so the core types this crate exposes changed. + ## [0.2.0](https://github.com/Remade-With-Rust/FFAI/compare/ffai-wasm-v0.1.5...ffai-wasm-v0.2.0) - 2026-10-08 ### Changed diff --git a/crates/ffai-wasm/Cargo.toml b/crates/ffai-wasm/Cargo.toml index 4124f6a..f5b8a16 100644 --- a/crates/ffai-wasm/Cargo.toml +++ b/crates/ffai-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-wasm" description = "Diana in the browser — YOLO26 object detection compiled to WebAssembly, pure Rust, no ONNX runtime" -version = "0.2.0" +version = "0.3.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/deny.toml b/deny.toml index b5296f7..bd3fc0b 100644 --- a/deny.toml +++ b/deny.toml @@ -25,6 +25,15 @@ ignore = [ # Not present in any shipped crate's normal dependency graph. # Review 2026-11-15. Replacement candidate: skrifa. "RUSTSEC-2026-0192", + # zip 8.6.0 - three extraction advisories: SUID/SGID bits preserved, symlink + # targets outside the destination, permissions applied through symlinks. Fixed + # only in zip 9, and candle-core 0.11 pins `^8.6` (npy/pickle). All three are in + # `ZipArchive::extract` / `ZipStreamReader::extract`; candle opens archives and + # reads entries by name and never extracts, and no crate here depends on zip + # directly. Added 2026-10-31. Review 2026-11-15, or when candle moves to zip 9. + "RUSTSEC-2026-0367", + "RUSTSEC-2026-0368", + "RUSTSEC-2026-0369", # RESOLVED 2026-08-19, ignores REMOVED rather than left to rot. pyo3 went # 0.23.5 -> 0.29.2, so RUSTSEC-2025-0020 (PyString::from_object buffer # overflow) and RUSTSEC-2026-0177 (missing Sync bound on diff --git a/models/ppocrv5-mobile-det.toml b/models/ppocrv5-mobile-det.toml index bba4797..fc9491c 100644 --- a/models/ppocrv5-mobile-det.toml +++ b/models/ppocrv5-mobile-det.toml @@ -7,6 +7,9 @@ name = "ppocrv5-mobile-det" task = "ocr" description = "PP-OCRv5 mobile det (DBNet/PP-LCNetV3) — staged detector lineage" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "det.safetensors" diff --git a/models/ppocrv5-mobile-rec.toml b/models/ppocrv5-mobile-rec.toml index c9b07bb..669064b 100644 --- a/models/ppocrv5-mobile-rec.toml +++ b/models/ppocrv5-mobile-rec.toml @@ -24,6 +24,9 @@ name = "ppocrv5-mobile-rec" task = "ocr" description = "PP-OCRv5 mobile rec (SVTR, 18385-class CTC) — opt-in via engine name mobiledet-svtr" license = "Apache-2.0" +# Published 2026-10-30 (first commit 637f0fc); every file is still verified +# against the sha256 below, so the host is a convenience, not a trust root. +hf_repo = "remade-with-rust/carmenta-ppocrv5-mobile" [[files]] name = "rec.safetensors" diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 65b3650..e0cbace 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -37,7 +37,7 @@ audit-as-crates-io = true [policy.candle-kernels] audit-as-crates-io = true -[policy.candle-metal-kernels] +[policy."candle-metal-kernels:0.11.0@git:fa366d84d12c5732cbc34c973fe92b8f480bfdbe"] audit-as-crates-io = true [policy.candle-ug] @@ -88,6 +88,9 @@ audit-as-crates-io = true [policy.rusty_json_turbo-accel] audit-as-crates-io = true +[policy."serde_json:1.0.151@git:77a58ed4c8e281f66e2d6b3a5b9ccd979ebec05b"] +audit-as-crates-io = false + [policy.tokenizers] audit-as-crates-io = true @@ -852,7 +855,7 @@ version = "0.11.11" criteria = "safe-to-deploy" [[exemptions.quinn-proto]] -version = "0.11.16" +version = "0.11.19" criteria = "safe-to-deploy" [[exemptions.r-efi]] @@ -1304,7 +1307,7 @@ version = "0.26.11" criteria = "safe-to-deploy" [[exemptions.webpki-roots]] -version = "1.0.9" +version = "1.0.11" criteria = "safe-to-deploy" [[exemptions.weezl]] diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 20231e9..5b152f1 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -9,6 +9,10 @@ audited_as = "0.7.4" version = "0.8.0" audited_as = "0.7.5" +[[unpublished.ffai-argus]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-argus-wasm]] version = "0.1.3" audited_as = "0.1.2" @@ -17,10 +21,18 @@ audited_as = "0.1.2" version = "0.2.0" audited_as = "0.1.3" +[[unpublished.ffai-argus-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[unpublished.ffai-bench]] version = "0.7.7" audited_as = "0.7.6" +[[unpublished.ffai-bench]] +version = "0.8.0" +audited_as = "0.7.7" + [[unpublished.ffai-carmenta]] version = "0.11.0" audited_as = "0.10.2" @@ -29,6 +41,10 @@ audited_as = "0.10.2" version = "0.12.0" audited_as = "0.11.0" +[[unpublished.ffai-carmenta]] +version = "0.13.0" +audited_as = "0.12.0" + [[unpublished.ffai-carmenta-wasm]] version = "0.2.3" audited_as = "0.2.2" @@ -37,6 +53,10 @@ audited_as = "0.2.2" version = "0.3.0" audited_as = "0.2.3" +[[unpublished.ffai-carmenta-wasm]] +version = "0.4.0" +audited_as = "0.3.0" + [[unpublished.ffai-cli]] version = "0.7.0" audited_as = "0.6.9" @@ -45,10 +65,18 @@ audited_as = "0.6.9" version = "0.8.0" audited_as = "0.7.0" +[[unpublished.ffai-cli]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-core]] version = "0.8.0" audited_as = "0.7.2" +[[unpublished.ffai-core]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-diana]] version = "0.7.8" audited_as = "0.7.7" @@ -57,14 +85,26 @@ audited_as = "0.7.7" version = "0.8.0" audited_as = "0.7.8" +[[unpublished.ffai-diana]] +version = "0.9.0" +audited_as = "0.8.0" + [[unpublished.ffai-media]] version = "0.6.8" audited_as = "0.6.7" +[[unpublished.ffai-media]] +version = "0.7.0" +audited_as = "0.6.8" + [[unpublished.ffai-mercury]] version = "1.0.5" audited_as = "1.0.4" +[[unpublished.ffai-mercury]] +version = "1.1.0" +audited_as = "1.0.5" + [[unpublished.ffai-mercury-wasm]] version = "0.1.3" audited_as = "0.1.2" @@ -73,10 +113,18 @@ audited_as = "0.1.2" version = "0.2.0" audited_as = "0.1.3" +[[unpublished.ffai-mercury-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[unpublished.ffai-models]] version = "0.6.6" audited_as = "0.6.5" +[[unpublished.ffai-models]] +version = "0.7.0" +audited_as = "0.6.6" + [[unpublished.ffai-wasm]] version = "0.1.5" audited_as = "0.1.4" @@ -85,6 +133,10 @@ audited_as = "0.1.4" version = "0.2.0" audited_as = "0.1.5" +[[unpublished.ffai-wasm]] +version = "0.3.0" +audited_as = "0.2.0" + [[publisher.aho-corasick]] version = "1.1.4" when = "2025-10-28" @@ -1526,6 +1578,17 @@ criteria = "safe-to-deploy" delta = "0.3.0 -> 0.4.0" notes = "Minor updates for a new `rand` crate version, nothing awry." +[[audits.bytecode-alliance.audits.rustls]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +delta = "0.23.45 -> 0.23.46" + +[[audits.bytecode-alliance.audits.rustls-webpki]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.103.15 -> 0.103.16" +notes = "Updated with new certificates" + [[audits.bytecode-alliance.audits.semver]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -1591,6 +1654,18 @@ who = "Pat Hickey " criteria = "safe-to-deploy" version = "0.3.0" +[[audits.bytecode-alliance.audits.webpki-root-certs]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +version = "1.0.4" +notes = "Purely a data crate." + +[[audits.bytecode-alliance.audits.webpki-root-certs]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "1.0.4 -> 1.0.11" +notes = "Standard root updates" + [[audits.bytecode-alliance.audits.xattr]] who = "Andrew Brown " criteria = "safe-to-deploy" diff --git a/tools/carmenta_stage_weights.py b/tools/carmenta_stage_weights.py new file mode 100644 index 0000000..8287829 --- /dev/null +++ b/tools/carmenta_stage_weights.py @@ -0,0 +1,119 @@ +"""Stage the mobiledet-svtr weights for a versioned public host. Uploads NOTHING. + +Why: the `ppocrv5-mobile-det` and `ppocrv5-mobile-rec` manifests carry SHA-256 +but no `hf_repo`, so `fetch` cannot download them and a crates.io user has no +way to get the document engine's weights (PdfCraft, 2026-10-30, finding #12). +`ffai-models` already downloads from a Hugging Face repo named in a manifest's +`hf_repo`, by file name, verifying each file against its pinned hash; what is +missing is the repo. Publishing weights is an owner's decision, so this stops at +a staging directory and prints the two commands that finish the job. + + python -I tools/carmenta_stage_weights.py OUT_DIR [HF_REPO] + +OUT_DIR gets exactly the files the engine loads, each verified against the +crate's embedded manifest, plus a model card and the upstream licence notice. +HF_REPO (default `remade-with-rust/carmenta-ppocrv5-mobile`) is only used in the +printed commands and manifest lines. + +The weights are PaddlePaddle's PP-OCRv5 mobile detector and recognizer +(Apache-2.0), converted by `carmenta_mobiledet_prepare.py` + +`carmenta_mobiledet_fuse.py` and `carmenta_svtr_prepare.py`. Redistribution +under Apache-2.0 requires the licence and attribution, which the card carries. +""" +import hashlib +import os +import re +import shutil +import sys +from pathlib import Path + +MANIFESTS = Path("crates/ffai-carmenta/models") +CACHE = Path(os.environ.get("LOCALAPPDATA", "")) / "ffai" / "models" +# model -> the files the engine actually loads (engine.rs `fetch` calls). +FILES = { + "ppocrv5-mobile-det": ["det-fused.safetensors"], + "ppocrv5-mobile-rec": ["rec.safetensors", "charset.txt"], +} + +CARD = """--- +license: apache-2.0 +tags: [ocr, text-detection, text-recognition, safetensors] +--- + +# Carmenta: PP-OCRv5 mobile (detection + SVTR recognition), safetensors + +Weights for the `mobiledet-svtr` document engine of +[ffai-carmenta](https://crates.io/crates/ffai-carmenta), the pure-Rust OCR +engine of [FFAI](https://github.com/Remade-With-Rust/FFAI). + +| file | model | sha256 | +|---|---|---| +{rows} + +These are **PaddlePaddle's PP-OCRv5 mobile** text detector (DBNet / PP-LCNetV3) +and text recognizer (SVTR, 18 383 characters + blank), released by PaddleOCR +under the Apache License 2.0. They were converted to safetensors (the detector +with its re-parameterised branches and BatchNorms fused into plain +convolutions) and checked against PaddlePaddle's own exported programs. No +weights were retrained. Copyright for the original models remains with the +PaddlePaddle Authors; see `LICENSE`. + +Use: `ffai_carmenta::engine::CraftCrnn::offline(RecStage::Svtr, +DetStage::MobileDet, root)` with `root//`, or let the `fetch` +feature download them by the hashes pinned in the crate's manifests. +""" + + +def sha256(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def pinned(model: str) -> dict: + """file name -> sha256 from the crate's embedded manifest.""" + text = (MANIFESTS / f"{model}.toml").read_text(encoding="utf8") + return dict(re.findall(r'name = "([^"]+)"\s*\nsha256 = "([0-9a-f]{64})"', text)) + + +def main(): + if len(sys.argv) < 2: + sys.exit(__doc__) + out = Path(sys.argv[1]) + repo = sys.argv[2] if len(sys.argv) > 2 else "remade-with-rust/carmenta-ppocrv5-mobile" + out.mkdir(parents=True, exist_ok=True) + rows = [] + for model, files in FILES.items(): + pins = pinned(model) + for name in files: + src = CACHE / model / name + want = pins.get(name) + if want is None: + sys.exit(f"{model}.toml pins no `{name}`") + got = sha256(src) + if got != want: + sys.exit(f"{src}: sha256 {got} does not match the manifest's {want}; refusing to stage") + shutil.copy2(src, out / name) + rows.append(f"| `{name}` | {model} | `{want}` |") + print(f"staged {name} ({src.stat().st_size:,} B), sha256 verified") + (out / "README.md").write_text(CARD.format(rows="\n".join(rows)), encoding="utf8") + (out / "LICENSE").write_text( + "The model weights in this repository are derived from PaddleOCR's PP-OCRv5\n" + "mobile detection and recognition models, Copyright (c) PaddlePaddle Authors,\n" + "licensed under the Apache License, Version 2.0:\n" + "https://www.apache.org/licenses/LICENSE-2.0\n" + "Source: https://github.com/PaddlePaddle/PaddleOCR\n", + encoding="utf8", + ) + print(f"\nStaged in {out}. Nothing was uploaded. To publish (the owner's decision):\n") + print(f" huggingface-cli repo create {repo.split('/', 1)[1]} --organization {repo.split('/', 1)[0]} --type model") + print(f" huggingface-cli upload {repo} {out} . --commit-message \"PP-OCRv5 mobile det+rec for ffai-carmenta\"") + print("\nthen add this line under `license = ...` in BOTH manifests") + print("(crates/ffai-carmenta/models/ppocrv5-mobile-{det,rec}.toml and the copies in models/):\n") + print(f' hf_repo = "{repo}"') + + +if __name__ == "__main__": + main()