Codex/rust pdf stage19 visual evidence

docs/rust-pdf-stage20-evidence.md

@@ -0,0 +1,37 @@
+# PDF 原生内核阶段 20：textpage 生命周期与字符几何物化
+
+## 实现边界
+
+本阶段以 Stage 19 提交 `be0b1a1` 为基线，只处理 Stage 19 报告确认的两个候选，不继续扩大 Path 迁移。私有协议从 26 升到 27；公开 `auto|python|rust` 计算选择和 `auto|legacy|session` 渲染选择保持不变。旧扩展在 `auto` 下仍显式回退，`rust` 下继续报协议不匹配。
+
+新增页面级自有文本快照入口：Python 仅核验 `FPDFText_LoadPage`、`FPDFText_ClosePage`、`FPDFText_CountChars` 与既有字符/颜色函数 ABI，并把当前同库 page 句柄交给 Rust；Rust 在宿主 PDFium 锁内短暂加载、计数、读取并关闭 textpage，产物仍只保存纯数值，不延长 PDFium 句柄生命周期。原 textpage 参数入口保留为兼容测试和参考路径；非标准 page、旧扩展、ABI 不匹配或异常数值时继续返回能力选择并走既有 Python 参考。
+
+`NativeTextSnapshot.geometry()` 的兼容 Python 字符物化保留完整字典形状、插入顺序和可变语义，同时减少重复工作：字体字典按 Rust 字体索引一次构建，字段名一次缓存，bbox 输入直接构造列表，origin/loose/tight 坐标一次转换为 Python 对象并由字符字段和对应侧表共享同一对象。公开 `PDFPageTextGeometry.chars`、tight/loose/origin 字典、行成员身份、pickle 结果和独立副本行为不变。
+
+## 验证
+
+- 新增页面级 textpage 入口与既有 textpage 入口的完整快照等价测试，覆盖真实加载/关闭路径和命中诊断；协议注册测试更新到 27。
+- 32 PDF／299 页公开输出双跑 Stage 20 后与 Stage 19 保存的完整 ModelJson、MiddleJson、素材哈希和诊断逐字段一致，报告为 `public-correctness.json`。
+- MinerU 当前 `dev@f504cff`：31 份 eligible Flash 文本完整输出一致；32 份实际 medium shared 完整输出一致。未修改 MinerU，未跟踪 `examples/` 保持原状。
+- Rust/session 完整测试：5475 passed / 1 skipped。Python/legacy 完整测试：4769 passed / 707 skipped。Cargo workspace tests、Clippy `-D warnings`、rustfmt、Ruff check 和改动文件 format check 均通过。
+- ABI3 wheel 为 `docvortex-0.5.7-cp310-abi3-macosx_11_0_arm64.whl`。在 CPython 3.14 独立依赖环境中，DocVortex 核心路径和 MinerU Flash 文本路径的 `auto|rust` 输出一致，协议 27，页面级 bridge 可调用。源码扩展与 wheel 扩展 SHA-256 均为 `77c16ff6c926daea4b8ec631f192ba0f3593791cd885391cbe13d4cdf6d862d8`。
+
+## 性能与资源
+
+正式计时均为每文档一次预热、五次热运行；公开 parse 按文档先基线后候选，MinerU 按文档交替方向执行，RSS 使用独立进程树采样。结果仅代表本机语料与当前 PDFium 环境，不外推托管环境。
+
+| 链路 | Stage 19 基线 | Stage 20 候选 | 降幅 | 首轮最大耗时比 | 首轮最大 RSS 比 |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| DocVortex 公开 parse，32 PDF | 14.966399 s | 14.923734 s | 0.29% | 1.06446 | 1.00480 |
+| MinerU Flash 文本，31 PDF | 14.827657 s | 14.759856 s | 0.46% | 1.02284 | 1.00549 |
+| MinerU medium shared，32 PDF | 14.738283 s | 14.601869 s | 0.93% | 1.07333 | 1.12712 |
+
+首轮超过 5% 耗时的样本均反序复测：公开 parse 的 `annual_report_management_roles_table.pdf` 复测比 0.99650；shared 的 `demo2.pdf`、`mixed_elements_pages_03_06.pdf`、`中文论文3.pdf` 复测比分别为 1.01313、1.00109、0.97148。shared 首轮唯一 RSS 超过 5% 的 `mixed_text_layout_sample.pdf.xor` 反序复测比为 1.02646。所有退化均未持续超过 5%，完整输出均相等。
+
+诊断层面，同一 32 PDF cProfile 中 `prepare_visual_evidence()` 从 0.703 s 降至 0.612 s；八样本焦点集的字符几何物化阶段从 Stage 19 记录的 0.257527 s 降至 0.226503 s（约 12.0%）。页面级 snapshot 在该焦点集实际执行 161 次、无回退。`_extract_owned_text_snapshot()` 本轮约为 0.529 s，与 Stage 19 的 0.525 s 接近，说明 textpage Python 包装不是原始快照的主要成本；后续应把字符 FFI 读取和规范化本身作为 Stage 21 候选。
+
+## 后续状态
+
+- 原始 2 倍性能目标仍未完成；本报告不把正确性或小幅实际收益表述为该目标完成。
+- Stage 21 不应继续优化 textpage 包装层。剩余更大热点包括 `_detect_table_candidates()`／表格物化、`_document_requires_full_geometry()` 的逐行 Python 输入打包，以及 `_plain_source_records()`；应在新的 profile 基础上选择下一批批量迁移。
+- 证据目录为 `output/pdf/native-kernel-20260929-stage20/`。本阶段分支不自动合并、发版或发布 PyPI。

rust/docvortex-core/src/lib.rs

@@ -1,6 +1,6 @@
 //! 不访问 Python 对象或 PDFium 的单线程批量计算内核。
 
-pub const PROTOCOL_VERSION: u32 = 26;
+pub const PROTOCOL_VERSION: u32 = 27;
 pub mod columns;
 pub mod note_index;
 pub mod row_geometry;

rust/docvortex-python/src/lib.rs

@@ -63,6 +63,10 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
         snapshot::read_pdfium_text_snapshot,
         module
     )?)?;
+    module.add_function(wrap_pyfunction!(
+        snapshot::read_pdfium_page_text_snapshot,
+        module
+    )?)?;
     module.add_function(wrap_pyfunction!(
         snapshot::visual_evidence_stage_stats,
         module

rust/docvortex-python/src/snapshot.rs

@@ -157,6 +157,63 @@ pub struct NativeTextSnapshot {
     raw_count: usize,
 }
 
+/// 由 Rust 短暂持有并释放 textpage，字符规范化仍复用同一个自有快照构建入口。
+#[pyfunction]
+pub fn read_pdfium_page_text_snapshot(
+    py: Python<'_>,
+    page_addresses: [usize; 3],
+    page_handle: usize,
+    addresses: Vec<usize>,
+    color_addresses: [usize; 2],
+    extended: bool,
+    frame: [f64; 4],
+    rotation: i32,
+    visibility: Option<HashMap<usize, (bool, Option<[f64; 4]>)>>,
+) -> PyResult<Option<NativeTextSnapshot>> {
+    if page_addresses.contains(&0) || page_handle == 0 {
+        return Err(PyValueError::new_err("invalid PDFium page text handle"));
+    }
+    // 安全条件：页面句柄、三个 textpage 函数和后续字符函数均来自同一运行库，并由宿主锁串行化。
+    let (text_handle, count) = unsafe {
+        let load: unsafe extern "system" fn(usize) -> usize =
+            std::mem::transmute(page_addresses[0]);
+        let close: unsafe extern "system" fn(usize) = std::mem::transmute(page_addresses[1]);
+        let count_chars: unsafe extern "system" fn(usize) -> i32 =
+            std::mem::transmute(page_addresses[2]);
+        let text_handle = load(page_handle);
+        if text_handle == 0 {
+            return Err(read_error(ReadError::Pdfium(
+                "Failed to load PDFium text page.",
+            )));
+        }
+        let count = count_chars(text_handle);
+        if count < 0 {
+            close(text_handle);
+            return Err(read_error(ReadError::Pdfium(
+                "negative PDFium character count",
+            )));
+        }
+        (text_handle, count as usize)
+    };
+    let result = read_pdfium_text_snapshot(
+        py,
+        addresses,
+        color_addresses,
+        text_handle,
+        count,
+        extended,
+        frame,
+        rotation,
+        visibility,
+    );
+    // 构建成功或失败都先关闭本次 Rust 加载的 textpage，产物只保留纯数值。
+    unsafe {
+        let close: unsafe extern "system" fn(usize) = std::mem::transmute(page_addresses[1]);
+        close(text_handle);
+    }
+    result
+}
+
 /// 同步借用 PDFium 读取原始字符；函数和句柄由已核验 ABI 的宿主 guard 保活。
 #[pyfunction]
 pub fn read_pdfium_text_snapshot(
@@ -367,63 +424,100 @@ impl NativeTextSnapshot {
             .import("docvortex.document.pdf.text._contracts")?
             .getattr("Bbox")?;
         let chars = PyList::empty(py);
+        // 一次缓存字段名，避免每个字符重复走解释器 intern 查找。
+        let key_bbox = pyo3::intern!(py, "bbox");
+        let key_char = pyo3::intern!(py, "char");
+        let key_rotation = pyo3::intern!(py, "rotation");
+        let key_font = pyo3::intern!(py, "font");
+        let key_char_idx = pyo3::intern!(py, "char_idx");
+        let key_source_indices = pyo3::intern!(py, "source_indices");
+        let key_raw_code = pyo3::intern!(py, "raw_code");
+        let key_text_object_id = pyo3::intern!(py, "text_object_id");
+        let key_text_render_mode = pyo3::intern!(py, "text_render_mode");
+        let key_writing_angle = pyo3::intern!(py, "writing_angle");
+        let key_origin = pyo3::intern!(py, "origin");
+        let key_name = pyo3::intern!(py, "name");
+        let key_flags = pyo3::intern!(py, "flags");
+        let key_size = pyo3::intern!(py, "size");
+        let key_weight = pyo3::intern!(py, "weight");
+        let key_loose_bbox = pyo3::intern!(py, "loose_bbox");
+        let key_tight_bbox = pyo3::intern!(py, "tight_bbox");
+        let key_text_is_visible = pyo3::intern!(py, "text_is_visible");
         let tight = PyDict::new(py);
         let loose = PyDict::new(py);
         let origins = PyDict::new(py);
-        let mut fonts: HashMap<usize, Bound<'py, PyDict>> = HashMap::new();
+
+        // 字体表由 Rust 索引直接寻址，避免每个字符再做一次哈希查找。
+        let mut fonts: Vec<Option<Bound<'py, PyDict>>> = vec![None; self.data.fonts.len()];
         let mut names: HashMap<usize, Bound<'py, pyo3::types::PyString>> = HashMap::new();
+        for (font_id, slot) in fonts.iter_mut().enumerate() {
+            let font = &self.data.fonts[font_id];
+            let value = PyDict::new(py);
+            let name = names
+                .entry(font.name_id)
+                .or_insert_with(|| pyo3::types::PyString::new(py, &font.name));
+            value.set_item(key_name, &*name)?;
+            value.set_item(key_flags, font.flags)?;
+            value.set_item(key_size, font.size)?;
+            value.set_item(key_weight, font.weight)?;
+            *slot = Some(value);
+        }
+
         for ch in &self.data.chars {
-            if let std::collections::hash_map::Entry::Vacant(entry) = fonts.entry(ch.font) {
-                let font = &self.data.fonts[ch.font];
-                let value = PyDict::new(py);
-                let name = names
-                    .entry(font.name_id)
-                    .or_insert_with(|| pyo3::types::PyString::new(py, &font.name));
-                value.set_item(pyo3::intern!(py, "name"), &*name)?;
-                value.set_item(pyo3::intern!(py, "flags"), font.flags)?;
-                value.set_item(pyo3::intern!(py, "size"), font.size)?;
-                value.set_item(pyo3::intern!(py, "weight"), font.weight)?;
-                entry.insert(value);
-            }
             let value = PyDict::new(py);
+            let bbox = bbox_type.call1((PyList::new(py, ch.bbox)?,))?;
+            let text = pyo3::types::PyString::new(py, &ch.text);
+            // 先转换一次不可变坐标容器；字符字段和 loose/tight/origin 侧表必须共享同一对象。
+            let origin = ch
+                .origin
+                .map(|p| (p[0], p[1]))
+                .into_pyobject(py)?
+                .into_any()
+                .unbind();
+            let loose_bbox = ch
+                .loose
+                .map(|b| (b[0], b[1], b[2], b[3]))
+                .into_pyobject(py)?
+                .into_any()
+                .unbind();
+            let tight_bbox = ch
+                .tight
+                .map(|b| (b[0], b[1], b[2], b[3]))
+                .into_pyobject(py)?
+                .into_any()
+                .unbind();
+            value.set_item(key_bbox, bbox)?;
+            value.set_item(key_char, text)?;
+            value.set_item(key_rotation, ch.rotation)?;
             value.set_item(
-                pyo3::intern!(py, "bbox"),
-                bbox_type.call1((ch.bbox.to_vec(),))?,
+                key_font,
+                fonts[ch.font].as_ref().expect("font materialized"),
             )?;
-            value.set_item(pyo3::intern!(py, "char"), &ch.text)?;
-            value.set_item(pyo3::intern!(py, "rotation"), ch.rotation)?;
-            value.set_item(pyo3::intern!(py, "font"), &fonts[&ch.font])?;
-            value.set_item(pyo3::intern!(py, "char_idx"), ch.index)?;
+            value.set_item(key_char_idx, ch.index)?;
             value.set_item(
-                pyo3::intern!(py, "source_indices"),
+                key_source_indices,
                 PyTuple::new(py, ch.sources.iter().copied())?,
             )?;
-            value.set_item(pyo3::intern!(py, "raw_code"), ch.code)?;
-            value.set_item(pyo3::intern!(py, "text_object_id"), ch.object)?;
-            value.set_item(pyo3::intern!(py, "text_render_mode"), ch.mode)?;
-            value.set_item(pyo3::intern!(py, "writing_angle"), ch.writing_angle)?;
-            value.set_item(pyo3::intern!(py, "origin"), ch.origin.map(|p| (p[0], p[1])))?;
+            value.set_item(key_raw_code, ch.code)?;
+            value.set_item(key_text_object_id, ch.object)?;
+            value.set_item(key_text_render_mode, ch.mode)?;
+            value.set_item(key_writing_angle, ch.writing_angle)?;
+            value.set_item(key_origin, &origin)?;
             if self.data.extended {
-                value.set_item(
-                    pyo3::intern!(py, "loose_bbox"),
-                    ch.loose.map(|b| (b[0], b[1], b[2], b[3])),
-                )?;
-                value.set_item(
-                    pyo3::intern!(py, "tight_bbox"),
-                    ch.tight.map(|b| (b[0], b[1], b[2], b[3])),
-                )?;
+                value.set_item(key_loose_bbox, &loose_bbox)?;
+                value.set_item(key_tight_bbox, &tight_bbox)?;
                 if ch.loose.is_some() && ch.rotation.abs() > 1e-9 {
-                    loose.set_item(ch.index, value.get_item("loose_bbox")?.unwrap())?;
+                    loose.set_item(ch.index, loose_bbox.clone_ref(py))?;
                 }
                 if ch.tight.is_some() {
-                    tight.set_item(ch.index, value.get_item("tight_bbox")?.unwrap())?;
+                    tight.set_item(ch.index, tight_bbox.clone_ref(py))?;
                 }
                 if ch.origin.is_some() {
-                    origins.set_item(ch.index, value.get_item("origin")?.unwrap())?;
+                    origins.set_item(ch.index, origin.clone_ref(py))?;
                 }
             }
             if let Some(visible) = ch.visible {
-                value.set_item(pyo3::intern!(py, "text_is_visible"), visible)?;
+                value.set_item(key_text_is_visible, visible)?;
             }
             chars.append(value)?;
         }

src/docvortex/_compute_backend.py

@@ -10,7 +10,7 @@
 from pathlib import Path
 from types import ModuleType
 
-_PROTOCOL_VERSION = 26
+_PROTOCOL_VERSION = 27
 _SELECTED_MODE = None
 _LOAD_FAILURE = None
 

src/docvortex/document/pdf/native_text_geometry.py

@@ -136,7 +136,6 @@ def _extract_page_text_geometry(
     """在调用方持有的页面和锁内读取字符，使批量提取与独立接口共用实现。"""
     textpage = None
     try:
-        textpage = page.get_textpage()
         raw_page_bbox: list[float] = list(page.get_bbox())
         page_rotation: int = 0
         try:
@@ -145,13 +144,15 @@ def _extract_page_text_geometry(
             pass
         visibility = _text_object_visibility(page, tuple(raw_page_bbox), page_rotation) if visible_only else None
         if compact:
-            from .snapshot_bridge import read_text_snapshot
+            from .snapshot_bridge import read_page_text_snapshot
 
-            snapshot = read_text_snapshot(textpage, raw_page_bbox, page_rotation, include_extended_geometry, visibility)
+            # 原生路径由 Rust 短暂加载 textpage；失败且需要参考时才创建 Python 包装。
+            snapshot = read_page_text_snapshot(page, raw_page_bbox, page_rotation, include_extended_geometry, visibility)
             if snapshot is not None:
                 return snapshot
             if compact_only:
                 return None
+        textpage = page.get_textpage()
         options = {"visibility_by_object": visibility} if visible_only else {}
         chars = get_chars(textpage, raw_page_bbox, page_rotation, include_geometry=include_extended_geometry, **options)
         raw_codes = {char["char_idx"]: char["raw_code"] for char in chars}