From b8365dca350b3eff52278686d34f89184efd3da1 Mon Sep 17 00:00:00 2001 From: myhloli Date: Tue, 29 Sep 2026 21:02:34 +0800 Subject: [PATCH 1/2] perf(pdf): own textpages and streamline geometry materialization --- rust/docvortex-core/src/lib.rs | 2 +- rust/docvortex-python/src/lib.rs | 4 + rust/docvortex-python/src/snapshot.rs | 168 +++++++++++---- src/docvortex/_compute_backend.py | 2 +- .../document/pdf/native_text_geometry.py | 7 +- src/docvortex/document/pdf/snapshot_bridge.py | 195 ++++++++++++++---- tests/test_native_parity.py | 3 +- tests/test_native_text_snapshot.py | 22 +- 8 files changed, 321 insertions(+), 82 deletions(-) diff --git a/rust/docvortex-core/src/lib.rs b/rust/docvortex-core/src/lib.rs index afad33f3..f850e557 100644 --- a/rust/docvortex-core/src/lib.rs +++ b/rust/docvortex-core/src/lib.rs @@ -1,6 +1,6 @@ //! 不访问 Python 对象或 PDFium 的单线程批量计算内核。 -pub const PROTOCOL_VERSION: u32 = 26; +pub const PROTOCOL_VERSION: u32 = 27; pub mod columns; pub mod note_index; pub mod row_geometry; diff --git a/rust/docvortex-python/src/lib.rs b/rust/docvortex-python/src/lib.rs index 94ada135..83ac342c 100644 --- a/rust/docvortex-python/src/lib.rs +++ b/rust/docvortex-python/src/lib.rs @@ -63,6 +63,10 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { snapshot::read_pdfium_text_snapshot, module )?)?; + module.add_function(wrap_pyfunction!( + snapshot::read_pdfium_page_text_snapshot, + module + )?)?; module.add_function(wrap_pyfunction!( snapshot::visual_evidence_stage_stats, module diff --git a/rust/docvortex-python/src/snapshot.rs b/rust/docvortex-python/src/snapshot.rs index c87d039a..5d0fa949 100644 --- a/rust/docvortex-python/src/snapshot.rs +++ b/rust/docvortex-python/src/snapshot.rs @@ -157,6 +157,63 @@ pub struct NativeTextSnapshot { raw_count: usize, } +/// 由 Rust 短暂持有并释放 textpage，字符规范化仍复用同一个自有快照构建入口。 +#[pyfunction] +pub fn read_pdfium_page_text_snapshot( + py: Python<'_>, + page_addresses: [usize; 3], + page_handle: usize, + addresses: Vec, + color_addresses: [usize; 2], + extended: bool, + frame: [f64; 4], + rotation: i32, + visibility: Option)>>, +) -> PyResult> { + if page_addresses.contains(&0) || page_handle == 0 { + return Err(PyValueError::new_err("invalid PDFium page text handle")); + } + // 安全条件：页面句柄、三个 textpage 函数和后续字符函数均来自同一运行库，并由宿主锁串行化。 + let (text_handle, count) = unsafe { + let load: unsafe extern "system" fn(usize) -> usize = + std::mem::transmute(page_addresses[0]); + let close: unsafe extern "system" fn(usize) = std::mem::transmute(page_addresses[1]); + let count_chars: unsafe extern "system" fn(usize) -> i32 = + std::mem::transmute(page_addresses[2]); + let text_handle = load(page_handle); + if text_handle == 0 { + return Err(read_error(ReadError::Pdfium( + "Failed to load PDFium text page.", + ))); + } + let count = count_chars(text_handle); + if count < 0 { + close(text_handle); + return Err(read_error(ReadError::Pdfium( + "negative PDFium character count", + ))); + } + (text_handle, count as usize) + }; + let result = read_pdfium_text_snapshot( + py, + addresses, + color_addresses, + text_handle, + count, + extended, + frame, + rotation, + visibility, + ); + // 构建成功或失败都先关闭本次 Rust 加载的 textpage，产物只保留纯数值。 + unsafe { + let close: unsafe extern "system" fn(usize) = std::mem::transmute(page_addresses[1]); + close(text_handle); + } + result +} + /// 同步借用 PDFium 读取原始字符；函数和句柄由已核验 ABI 的宿主 guard 保活。 #[pyfunction] pub fn read_pdfium_text_snapshot( @@ -367,63 +424,100 @@ impl NativeTextSnapshot { .import("docvortex.document.pdf.text._contracts")? .getattr("Bbox")?; let chars = PyList::empty(py); + // 一次缓存字段名，避免每个字符重复走解释器 intern 查找。 + let key_bbox = pyo3::intern!(py, "bbox"); + let key_char = pyo3::intern!(py, "char"); + let key_rotation = pyo3::intern!(py, "rotation"); + let key_font = pyo3::intern!(py, "font"); + let key_char_idx = pyo3::intern!(py, "char_idx"); + let key_source_indices = pyo3::intern!(py, "source_indices"); + let key_raw_code = pyo3::intern!(py, "raw_code"); + let key_text_object_id = pyo3::intern!(py, "text_object_id"); + let key_text_render_mode = pyo3::intern!(py, "text_render_mode"); + let key_writing_angle = pyo3::intern!(py, "writing_angle"); + let key_origin = pyo3::intern!(py, "origin"); + let key_name = pyo3::intern!(py, "name"); + let key_flags = pyo3::intern!(py, "flags"); + let key_size = pyo3::intern!(py, "size"); + let key_weight = pyo3::intern!(py, "weight"); + let key_loose_bbox = pyo3::intern!(py, "loose_bbox"); + let key_tight_bbox = pyo3::intern!(py, "tight_bbox"); + let key_text_is_visible = pyo3::intern!(py, "text_is_visible"); let tight = PyDict::new(py); let loose = PyDict::new(py); let origins = PyDict::new(py); - let mut fonts: HashMap> = HashMap::new(); + + // 字体表由 Rust 索引直接寻址，避免每个字符再做一次哈希查找。 + let mut fonts: Vec>> = vec![None; self.data.fonts.len()]; let mut names: HashMap> = HashMap::new(); + for (font_id, slot) in fonts.iter_mut().enumerate() { + let font = &self.data.fonts[font_id]; + let value = PyDict::new(py); + let name = names + .entry(font.name_id) + .or_insert_with(|| pyo3::types::PyString::new(py, &font.name)); + value.set_item(key_name, &*name)?; + value.set_item(key_flags, font.flags)?; + value.set_item(key_size, font.size)?; + value.set_item(key_weight, font.weight)?; + *slot = Some(value); + } + for ch in &self.data.chars { - if let std::collections::hash_map::Entry::Vacant(entry) = fonts.entry(ch.font) { - let font = &self.data.fonts[ch.font]; - let value = PyDict::new(py); - let name = names - .entry(font.name_id) - .or_insert_with(|| pyo3::types::PyString::new(py, &font.name)); - value.set_item(pyo3::intern!(py, "name"), &*name)?; - value.set_item(pyo3::intern!(py, "flags"), font.flags)?; - value.set_item(pyo3::intern!(py, "size"), font.size)?; - value.set_item(pyo3::intern!(py, "weight"), font.weight)?; - entry.insert(value); - } let value = PyDict::new(py); + let bbox = bbox_type.call1((PyList::new(py, ch.bbox)?,))?; + let text = pyo3::types::PyString::new(py, &ch.text); + // 先转换一次不可变坐标容器；字符字段和 loose/tight/origin 侧表必须共享同一对象。 + let origin = ch + .origin + .map(|p| (p[0], p[1])) + .into_pyobject(py)? + .into_any() + .unbind(); + let loose_bbox = ch + .loose + .map(|b| (b[0], b[1], b[2], b[3])) + .into_pyobject(py)? + .into_any() + .unbind(); + let tight_bbox = ch + .tight + .map(|b| (b[0], b[1], b[2], b[3])) + .into_pyobject(py)? + .into_any() + .unbind(); + value.set_item(key_bbox, bbox)?; + value.set_item(key_char, text)?; + value.set_item(key_rotation, ch.rotation)?; value.set_item( - pyo3::intern!(py, "bbox"), - bbox_type.call1((ch.bbox.to_vec(),))?, + key_font, + fonts[ch.font].as_ref().expect("font materialized"), )?; - value.set_item(pyo3::intern!(py, "char"), &ch.text)?; - value.set_item(pyo3::intern!(py, "rotation"), ch.rotation)?; - value.set_item(pyo3::intern!(py, "font"), &fonts[&ch.font])?; - value.set_item(pyo3::intern!(py, "char_idx"), ch.index)?; + value.set_item(key_char_idx, ch.index)?; value.set_item( - pyo3::intern!(py, "source_indices"), + key_source_indices, PyTuple::new(py, ch.sources.iter().copied())?, )?; - value.set_item(pyo3::intern!(py, "raw_code"), ch.code)?; - value.set_item(pyo3::intern!(py, "text_object_id"), ch.object)?; - value.set_item(pyo3::intern!(py, "text_render_mode"), ch.mode)?; - value.set_item(pyo3::intern!(py, "writing_angle"), ch.writing_angle)?; - value.set_item(pyo3::intern!(py, "origin"), ch.origin.map(|p| (p[0], p[1])))?; + value.set_item(key_raw_code, ch.code)?; + value.set_item(key_text_object_id, ch.object)?; + value.set_item(key_text_render_mode, ch.mode)?; + value.set_item(key_writing_angle, ch.writing_angle)?; + value.set_item(key_origin, &origin)?; if self.data.extended { - value.set_item( - pyo3::intern!(py, "loose_bbox"), - ch.loose.map(|b| (b[0], b[1], b[2], b[3])), - )?; - value.set_item( - pyo3::intern!(py, "tight_bbox"), - ch.tight.map(|b| (b[0], b[1], b[2], b[3])), - )?; + value.set_item(key_loose_bbox, &loose_bbox)?; + value.set_item(key_tight_bbox, &tight_bbox)?; if ch.loose.is_some() && ch.rotation.abs() > 1e-9 { - loose.set_item(ch.index, value.get_item("loose_bbox")?.unwrap())?; + loose.set_item(ch.index, loose_bbox.clone_ref(py))?; } if ch.tight.is_some() { - tight.set_item(ch.index, value.get_item("tight_bbox")?.unwrap())?; + tight.set_item(ch.index, tight_bbox.clone_ref(py))?; } if ch.origin.is_some() { - origins.set_item(ch.index, value.get_item("origin")?.unwrap())?; + origins.set_item(ch.index, origin.clone_ref(py))?; } } if let Some(visible) = ch.visible { - value.set_item(pyo3::intern!(py, "text_is_visible"), visible)?; + value.set_item(key_text_is_visible, visible)?; } chars.append(value)?; } diff --git a/src/docvortex/_compute_backend.py b/src/docvortex/_compute_backend.py index b8ee692a..13dc9546 100644 --- a/src/docvortex/_compute_backend.py +++ b/src/docvortex/_compute_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from types import ModuleType -_PROTOCOL_VERSION = 26 +_PROTOCOL_VERSION = 27 _SELECTED_MODE = None _LOAD_FAILURE = None diff --git a/src/docvortex/document/pdf/native_text_geometry.py b/src/docvortex/document/pdf/native_text_geometry.py index 05b0e6a2..b91f49c1 100644 --- a/src/docvortex/document/pdf/native_text_geometry.py +++ b/src/docvortex/document/pdf/native_text_geometry.py @@ -136,7 +136,6 @@ def _extract_page_text_geometry( """在调用方持有的页面和锁内读取字符，使批量提取与独立接口共用实现。""" textpage = None try: - textpage = page.get_textpage() raw_page_bbox: list[float] = list(page.get_bbox()) page_rotation: int = 0 try: @@ -145,13 +144,15 @@ def _extract_page_text_geometry( pass visibility = _text_object_visibility(page, tuple(raw_page_bbox), page_rotation) if visible_only else None if compact: - from .snapshot_bridge import read_text_snapshot + from .snapshot_bridge import read_page_text_snapshot - snapshot = read_text_snapshot(textpage, raw_page_bbox, page_rotation, include_extended_geometry, visibility) + # 原生路径由 Rust 短暂加载 textpage；失败且需要参考时才创建 Python 包装。 + snapshot = read_page_text_snapshot(page, raw_page_bbox, page_rotation, include_extended_geometry, visibility) if snapshot is not None: return snapshot if compact_only: return None + textpage = page.get_textpage() options = {"visibility_by_object": visibility} if visible_only else {} chars = get_chars(textpage, raw_page_bbox, page_rotation, include_geometry=include_extended_geometry, **options) raw_codes = {char["char_idx"]: char["raw_code"] for char in chars} diff --git a/src/docvortex/document/pdf/snapshot_bridge.py b/src/docvortex/document/pdf/snapshot_bridge.py index 8c68a774..18fa210a 100644 --- a/src/docvortex/document/pdf/snapshot_bridge.py +++ b/src/docvortex/document/pdf/snapshot_bridge.py @@ -26,35 +26,14 @@ def snapshot_bridge_info(): } -def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): - """在原锁和 textpage 生命周期内读取、规范化与去重，标准运算错误直接传播。""" - global _CALLS, _EMPTY, _REASON - native = get_native() - if native is None or not hasattr(native, "read_pdfium_text_snapshot"): - _REASON = "Python backend or native text snapshot unavailable" - return None - if ( - type(textpage) is not pdfium.PdfTextPage - or not textpage.raw - or type(extended) is not bool - or type(rotation) is not int - or rotation not in (0, 90, 180, 270) - or type(frame) is not list - or len(frame) != 4 - or any(type(value) is not float or not math.isfinite(value) for value in frame) - or not _native_visibility_supported(visibility) - or visibility is not None - and 0 in visibility - ): - _REASON = "nonstandard text page, numerical metadata, or visibility" - return None +def _standard_character_symbols(): + """核验字符读取所需 ABI，并返回函数强引用和 Rust 可调用的地址。""" if ct.sizeof(raw.FS_RECTF) != 16 or any( getattr(raw.FS_RECTF, name).offset != offset for name, offset in (("left", 0), ("top", 4), ("right", 8), ("bottom", 12)) ): - _REASON = "FS_RECTF ABI mismatch" - return None + return None, "FS_RECTF ABI mismatch" args = (raw.FPDF_TEXTPAGE, ct.c_int) - double_pointer, uint_pointer = ct.POINTER(ct.c_double), ct.POINTER(ct.c_uint) + double_pointer = ct.POINTER(ct.c_double) specs = ( ("FPDFText_GetUnicode", ct.c_uint, args), ("FPDFText_GetCharAngle", ct.c_float, args), @@ -66,8 +45,6 @@ def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): ("FPDFText_GetTextObject", raw.FPDF_PAGEOBJECT, args), ("FPDFTextObj_GetTextRenderMode", ct.c_int, (raw.FPDF_PAGEOBJECT,)), ("FPDFText_GetCharOrigin", ct.c_int, (*args, double_pointer, double_pointer)), - ("FPDFText_GetFillColor", ct.c_int, (*args, uint_pointer, uint_pointer, uint_pointer, uint_pointer)), - ("FPDFText_GetStrokeColor", ct.c_int, (*args, uint_pointer, uint_pointer, uint_pointer, uint_pointer)), ) functions, addresses = [], [] for name, result, arguments in specs: @@ -78,14 +55,116 @@ def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): or tuple(function.argtypes or ()) != arguments or getattr(function, "errcheck", None) is not None ): - _REASON = f"unsupported symbol or ABI: {name}" - return None + return None, f"unsupported symbol or ABI: {name}" functions.append(function) addresses.append(ct.cast(function, ct.c_void_p).value) - # 函数强引用和 textpage 在整个读取阶段存活，产物不保存任何 PDFium 地址。 + return (functions, addresses), None + + +def _standard_page_text_symbols(): + """核验 Rust 短暂加载、计数和关闭 textpage 的三个页面级函数。""" + specs = ( + ("FPDFText_LoadPage", raw.FPDF_TEXTPAGE, (raw.FPDF_PAGE,)), + ("FPDFText_ClosePage", None, (raw.FPDF_TEXTPAGE,)), + ("FPDFText_CountChars", ct.c_int, (raw.FPDF_TEXTPAGE,)), + ) + functions, addresses = [], [] + for name, result, arguments in specs: + function = getattr(raw, name, None) + if ( + not isinstance(function, ct._CFuncPtr) + or function.restype is not result + or tuple(function.argtypes or ()) != arguments + or getattr(function, "errcheck", None) is not None + ): + return None, f"unsupported page text symbol or ABI: {name}" + functions.append(function) + addresses.append(ct.cast(function, ct.c_void_p).value) + return (functions, addresses), None + + +def _color_symbols(): + """核验隐藏文字判定所需的填充和描边颜色函数。""" + uint_pointer = ct.POINTER(ct.c_uint) + specs = ( + ( + "FPDFText_GetFillColor", + ct.c_int, + (raw.FPDF_TEXTPAGE, ct.c_int, uint_pointer, uint_pointer, uint_pointer, uint_pointer), + ), + ( + "FPDFText_GetStrokeColor", + ct.c_int, + (raw.FPDF_TEXTPAGE, ct.c_int, uint_pointer, uint_pointer, uint_pointer, uint_pointer), + ), + ) + functions, addresses = [], [] + for name, result, arguments in specs: + function = getattr(raw, name, None) + if ( + not isinstance(function, ct._CFuncPtr) + or function.restype is not result + or tuple(function.argtypes or ()) != arguments + or getattr(function, "errcheck", None) is not None + ): + return None, f"unsupported color symbol or ABI: {name}" + functions.append(function) + addresses.append(ct.cast(function, ct.c_void_p).value) + return (functions, addresses), None + + +def _ordinary_snapshot_arguments(source, raw_handle, frame, rotation, extended, visibility): + """集中执行两个入口共用的类型、数值和可见性输入检查。""" + if ( + not raw_handle + or type(extended) is not bool + or type(rotation) is not int + or rotation not in (0, 90, 180, 270) + or type(frame) is not list + or len(frame) != 4 + or any(type(value) is not float or not math.isfinite(value) for value in frame) + or not _native_visibility_supported(visibility) + or visibility is not None + and 0 in visibility + ): + return False + del source + return True + + +def _record_snapshot_result(snapshot, count): + """按原始字符数更新命中诊断；标准算法失败仍由异常直接传播。""" + global _CALLS, _EMPTY + if count: + _CALLS += 1 + else: + _EMPTY += 1 + return snapshot + + +def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): + """在原锁和 Python textpage 生命周期内读取、规范化与去重，保留兼容测试入口。""" + global _REASON + native = get_native() + if native is None or not hasattr(native, "read_pdfium_text_snapshot"): + _REASON = "Python backend or native text snapshot unavailable" + return None + if type(textpage) is not pdfium.PdfTextPage or not _ordinary_snapshot_arguments( + textpage, textpage.raw, frame, rotation, extended, visibility + ): + _REASON = "nonstandard text page, numerical metadata, or visibility" + return None + characters, character_reason = _standard_character_symbols() + colors, color_reason = _color_symbols() + if characters is None or colors is None: + _REASON = character_reason or color_reason + return None + character_functions, character_addresses = characters + color_functions, color_addresses = colors packed_visibility = ( {0 if key is None else key: value for key, value in visibility.items()} if visibility is not None else None ) + # 函数强引用和 textpage 在整个读取阶段存活，产物不保存任何 PDFium 地址。 with pdfium_guard(): count = textpage.count_chars() if count < 0: @@ -93,8 +172,8 @@ def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): return None try: snapshot = native.read_pdfium_text_snapshot( - addresses[:10], - addresses[10:], + character_addresses, + color_addresses, ct.cast(textpage.raw, ct.c_void_p).value, count, extended, @@ -107,9 +186,51 @@ def read_text_snapshot(textpage, frame, rotation, extended, visibility=None): if snapshot is None: _REASON = "raw PDF numerical metadata outside native canonical admission" return None - if count: - _CALLS += 1 - else: - _EMPTY += 1 _REASON = None - return snapshot + return _record_snapshot_result(snapshot, count) + + +def read_page_text_snapshot(page, frame, rotation, extended, visibility=None): + """让 Rust 在宿主锁内短暂加载并关闭 textpage，避免原生路径创建 Python 包装。""" + global _REASON + native = get_native() + if native is None or not hasattr(native, "read_pdfium_page_text_snapshot"): + _REASON = "Python backend or native page text snapshot unavailable" + return None + if type(page) is not pdfium.PdfPage or not _ordinary_snapshot_arguments( + page, page.raw, frame, rotation, extended, visibility + ): + _REASON = "nonstandard PDF page, numerical metadata, or visibility" + return None + characters, character_reason = _standard_character_symbols() + colors, color_reason = _color_symbols() + page_text, page_reason = _standard_page_text_symbols() + if characters is None or colors is None or page_text is None: + _REASON = character_reason or color_reason or page_reason + return None + character_functions, character_addresses = characters + color_functions, color_addresses = colors + page_functions, page_addresses = page_text + packed_visibility = ( + {0 if key is None else key: value for key, value in visibility.items()} if visibility is not None else None + ) + # 页面由调用方保活；Rust 仅在同锁作用内创建和关闭 textpage，不延长任何句柄。 + with pdfium_guard(): + try: + snapshot = native.read_pdfium_page_text_snapshot( + page_addresses, + ct.cast(page.raw, ct.c_void_p).value, + character_addresses, + color_addresses, + extended, + frame, + rotation, + packed_visibility, + ) + except native.PdfiumReadError as exc: + raise pdfium.PdfiumError(str(exc)) from exc + if snapshot is None: + _REASON = "raw PDF numerical metadata outside native canonical admission" + return None + _REASON = None + return _record_snapshot_result(snapshot, snapshot.raw_char_count()) diff --git a/tests/test_native_parity.py b/tests/test_native_parity.py index 7e1cac43..d586853f 100644 --- a/tests/test_native_parity.py +++ b/tests/test_native_parity.py @@ -58,6 +58,7 @@ def test_native_registration_contract(native): "prepare_visual_lines": "(chars, size, page_rotation, supported_angles)", "read_pdfium_subpaths": "(addresses, handle)", "read_pdfium_text_snapshot": "(addresses, color_addresses, handle, count, extended, frame, rotation, visibility)", + "read_pdfium_page_text_snapshot": "(page_addresses, page_handle, addresses, color_addresses, extended, frame, rotation, visibility)", "read_pdfium_chars": "(addresses, handle, count, extended)", "read_pdfium_objects": "(addresses, handle, kind, max_depth)", "read_pdfium_text_visibility": "(addresses, handle, frame, rotation, max_depth)", @@ -106,7 +107,7 @@ def test_native_registration_contract(native): "TableNoteMetrics": ("builtins", "(items)"), "TableRowGeometry": ("builtins", "(boxes)"), } - constants = {"PROTOCOL_VERSION": 26, "PDFIUM_RECORD_BATCH_SIZE": 1024} + constants = {"PROTOCOL_VERSION": 27, "PDFIUM_RECORD_BATCH_SIZE": 1024} assert {name for name in dir(native) if not name.startswith("__")} == functions.keys() | classes.keys() | constants.keys() for name, signature in functions.items(): function = getattr(native, name) diff --git a/tests/test_native_text_snapshot.py b/tests/test_native_text_snapshot.py index 778568e1..a40240f7 100644 --- a/tests/test_native_text_snapshot.py +++ b/tests/test_native_text_snapshot.py @@ -16,7 +16,7 @@ from docvortex.document.pdf import PDFDocument from docvortex.document.pdf.pdfium import pdfium_guard from docvortex.document.pdf.native_text_geometry import _extract_page_text_geometry -from docvortex.document.pdf.snapshot_bridge import read_text_snapshot, snapshot_bridge_info +from docvortex.document.pdf.snapshot_bridge import read_page_text_snapshot, read_text_snapshot, snapshot_bridge_info from docvortex.document.pdf.text import _get_lines_from_chars_python from docvortex.document.pdf.text._contracts import Bbox from docvortex.analyzers.native.pdf.native_text import ( @@ -191,6 +191,24 @@ def test_snapshot_special_metadata_selects_reference_before_calculation(native): assert "nonstandard" in snapshot_bridge_info()["native_text_snapshot_unavailable_reason"] +def test_page_snapshot_bridge_loads_and_closes_textpage(native): + """页面级入口不创建 Python textpage，并与兼容入口保持完整快照等价。""" + with ( + pdfium_guard(), + pdfium.PdfDocument(_pdf()) as document, + closing(document[0]) as page, + closing(page.get_textpage()) as textpage, + ): + before = snapshot_bridge_info()["native_text_snapshot_calls"] + expected = read_text_snapshot(textpage, list(page.get_bbox()), 0, True) + actual = read_page_text_snapshot(page, list(page.get_bbox()), 0, True) + assert isinstance(actual, native.NativeTextSnapshot) + assert actual.raw_char_count() == expected.raw_char_count() + assert _plain(actual.materialize_geometry()) == _plain(expected.materialize_geometry()) + assert snapshot_bridge_info()["native_text_snapshot_calls"] == before + 2 + assert snapshot_bridge_info()["native_text_snapshot_unavailable_reason"] is None + + def test_snapshot_keeps_public_geometry_dataclass_shape(native): """公开geometry仍只有原四字段；显式修改的geometry不能被快照旁路覆盖。""" from docvortex.analyzers.pdf import prepare_text_evidence @@ -358,7 +376,7 @@ def test_owned_probe_does_not_materialize_reference_on_unsupported_metadata(nati """原生数值准入失败后只返回能力选择，不提前做一遍宿主随后还会重复的字符提取。""" with ( PDFDocument(_pdf()) as document, - patch("docvortex.document.pdf.snapshot_bridge.read_text_snapshot", return_value=None), + patch("docvortex.document.pdf.snapshot_bridge.read_page_text_snapshot", return_value=None), patch( "docvortex.document.pdf.native_text_geometry.get_chars", side_effect=AssertionError("reference materialized twice") ), From d504c5eaa090fdc6fbceefa245a2a905b9bcd700 Mon Sep 17 00:00:00 2001 From: myhloli Date: Tue, 29 Sep 2026 21:02:41 +0800 Subject: [PATCH 2/2] docs(pdf): record stage 20 geometry validation --- docs/rust-pdf-stage20-evidence.md | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/rust-pdf-stage20-evidence.md diff --git a/docs/rust-pdf-stage20-evidence.md b/docs/rust-pdf-stage20-evidence.md new file mode 100644 index 00000000..82f77152 --- /dev/null +++ b/docs/rust-pdf-stage20-evidence.md @@ -0,0 +1,37 @@ +# PDF 原生内核阶段 20：textpage 生命周期与字符几何物化 + +## 实现边界 + +本阶段以 Stage 19 提交 `be0b1a1` 为基线，只处理 Stage 19 报告确认的两个候选，不继续扩大 Path 迁移。私有协议从 26 升到 27；公开 `auto|python|rust` 计算选择和 `auto|legacy|session` 渲染选择保持不变。旧扩展在 `auto` 下仍显式回退，`rust` 下继续报协议不匹配。 + +新增页面级自有文本快照入口：Python 仅核验 `FPDFText_LoadPage`、`FPDFText_ClosePage`、`FPDFText_CountChars` 与既有字符/颜色函数 ABI，并把当前同库 page 句柄交给 Rust；Rust 在宿主 PDFium 锁内短暂加载、计数、读取并关闭 textpage，产物仍只保存纯数值，不延长 PDFium 句柄生命周期。原 textpage 参数入口保留为兼容测试和参考路径；非标准 page、旧扩展、ABI 不匹配或异常数值时继续返回能力选择并走既有 Python 参考。 + +`NativeTextSnapshot.geometry()` 的兼容 Python 字符物化保留完整字典形状、插入顺序和可变语义，同时减少重复工作：字体字典按 Rust 字体索引一次构建，字段名一次缓存，bbox 输入直接构造列表，origin/loose/tight 坐标一次转换为 Python 对象并由字符字段和对应侧表共享同一对象。公开 `PDFPageTextGeometry.chars`、tight/loose/origin 字典、行成员身份、pickle 结果和独立副本行为不变。 + +## 验证 + +- 新增页面级 textpage 入口与既有 textpage 入口的完整快照等价测试，覆盖真实加载/关闭路径和命中诊断；协议注册测试更新到 27。 +- 32 PDF／299 页公开输出双跑 Stage 20 后与 Stage 19 保存的完整 ModelJson、MiddleJson、素材哈希和诊断逐字段一致，报告为 `public-correctness.json`。 +- MinerU 当前 `dev@f504cff`：31 份 eligible Flash 文本完整输出一致；32 份实际 medium shared 完整输出一致。未修改 MinerU，未跟踪 `examples/` 保持原状。 +- Rust/session 完整测试：5475 passed / 1 skipped。Python/legacy 完整测试：4769 passed / 707 skipped。Cargo workspace tests、Clippy `-D warnings`、rustfmt、Ruff check 和改动文件 format check 均通过。 +- ABI3 wheel 为 `docvortex-0.5.7-cp310-abi3-macosx_11_0_arm64.whl`。在 CPython 3.14 独立依赖环境中，DocVortex 核心路径和 MinerU Flash 文本路径的 `auto|rust` 输出一致，协议 27，页面级 bridge 可调用。源码扩展与 wheel 扩展 SHA-256 均为 `77c16ff6c926daea4b8ec631f192ba0f3593791cd885391cbe13d4cdf6d862d8`。 + +## 性能与资源 + +正式计时均为每文档一次预热、五次热运行；公开 parse 按文档先基线后候选，MinerU 按文档交替方向执行，RSS 使用独立进程树采样。结果仅代表本机语料与当前 PDFium 环境，不外推托管环境。 + +| 链路 | Stage 19 基线 | Stage 20 候选 | 降幅 | 首轮最大耗时比 | 首轮最大 RSS 比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| DocVortex 公开 parse，32 PDF | 14.966399 s | 14.923734 s | 0.29% | 1.06446 | 1.00480 | +| MinerU Flash 文本，31 PDF | 14.827657 s | 14.759856 s | 0.46% | 1.02284 | 1.00549 | +| MinerU medium shared，32 PDF | 14.738283 s | 14.601869 s | 0.93% | 1.07333 | 1.12712 | + +首轮超过 5% 耗时的样本均反序复测：公开 parse 的 `annual_report_management_roles_table.pdf` 复测比 0.99650；shared 的 `demo2.pdf`、`mixed_elements_pages_03_06.pdf`、`中文论文3.pdf` 复测比分别为 1.01313、1.00109、0.97148。shared 首轮唯一 RSS 超过 5% 的 `mixed_text_layout_sample.pdf.xor` 反序复测比为 1.02646。所有退化均未持续超过 5%，完整输出均相等。 + +诊断层面，同一 32 PDF cProfile 中 `prepare_visual_evidence()` 从 0.703 s 降至 0.612 s；八样本焦点集的字符几何物化阶段从 Stage 19 记录的 0.257527 s 降至 0.226503 s（约 12.0%）。页面级 snapshot 在该焦点集实际执行 161 次、无回退。`_extract_owned_text_snapshot()` 本轮约为 0.529 s，与 Stage 19 的 0.525 s 接近，说明 textpage Python 包装不是原始快照的主要成本；后续应把字符 FFI 读取和规范化本身作为 Stage 21 候选。 + +## 后续状态 + +- 原始 2 倍性能目标仍未完成；本报告不把正确性或小幅实际收益表述为该目标完成。 +- Stage 21 不应继续优化 textpage 包装层。剩余更大热点包括 `_detect_table_candidates()`／表格物化、`_document_requires_full_geometry()` 的逐行 Python 输入打包，以及 `_plain_source_records()`；应在新的 profile 基础上选择下一批批量迁移。 +- 证据目录为 `output/pdf/native-kernel-20260929-stage20/`。本阶段分支不自动合并、发版或发布 PyPI。