From f36a44459cc6b2ba091c1b5f1ec1b6e767b67870 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 23 Oct 2026 10:27:43 -0700 Subject: [PATCH] fix: publish body-only previews for page chunks --- .../test_canonical_demo_bundle_contract.py | 32 +++++ .../test_shared_demo_corpus_contract.py | 51 ++++++++ .../app/services/document_agent/pdf_text.py | 16 +++ .../document_parser/profiling/doc_profiler.py | 5 +- .../profiling/profile_model.py | 2 + .../services/page_memory/body_text_preview.py | 7 + .../services/page_memory/memory_service.py | 10 +- .../services/page_memory/node_assembler.py | 51 ++++++++ .../test_page_body_text_preview_contract.py | 121 ++++++++++++++++++ ...est_page_memory_node_assembler_contract.py | 4 +- docs/page-body-text-preview.md | 65 ++++++++++ .../chunks/dataframe_chunk_converter.py | 1 + .../services/storage/zip_chunk_schema.py | 2 + 13 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 apps/worker/app/services/page_memory/body_text_preview.py create mode 100644 apps/worker/tests/contract/test_page_body_text_preview_contract.py create mode 100644 docs/page-body-text-preview.md diff --git a/apps/api/tests/contract/test_canonical_demo_bundle_contract.py b/apps/api/tests/contract/test_canonical_demo_bundle_contract.py index 89fea590e..c10ffc4c4 100644 --- a/apps/api/tests/contract/test_canonical_demo_bundle_contract.py +++ b/apps/api/tests/contract/test_canonical_demo_bundle_contract.py @@ -7,6 +7,9 @@ from contextlib import AbstractAsyncContextManager from pathlib import Path from threading import Event +import json +from io import BytesIO +from zipfile import ZipFile from typing import Literal import pytest @@ -20,6 +23,35 @@ def _write_source(directory: Path, content: str) -> None: (directory / "source.md").write_text(content, encoding="utf-8") +@pytest.mark.asyncio +async def test_canonical_bundle_keeps_page_body_preview_metadata( + api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], + tmp_path: Path, +) -> None: + async with api_client_factory(): + from app.services.demo.canonical_bundle import CanonicalDemoBundleStore + from shared.services.redis import RedisServiceFactory + from shared.services.storage.job_file_storage import JobFileStorage + from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder + + directory = tmp_path / "preview-source" + _write_source(directory, "original content") + chunks = ZipChunkSchemaBuilder().format_chunks([ + {"chunk_id": f"page-{index}", "type": "page", "path": f"doc.pdf/Section {index}", "content": "header\nbody\nfooter", "metadata": {"body_text_preview": preview}} + for index, preview in enumerate(["body", ""]) + ], {}, {}) + (directory / "chunks.json").write_text(json.dumps({"chunks": chunks})) + bundle = await CanonicalDemoBundleStore(redis_service=RedisServiceFactory.get_service()).ensure_bundle( + source_id="contract-body-preview", source_directory=directory, + ) + storage = JobFileStorage() + payload = storage.storage_adapter.download_fileobj(bundle.zip_key, storage.results_bucket) + with ZipFile(BytesIO(payload)) as archive: + archived = json.loads(archive.read("chunks.json"))["chunks"] + assert [chunk["metadata"]["body_text_preview"] for chunk in archived] == ["body", ""] + assert all(chunk["content"] == "header\nbody\nfooter" for chunk in archived) + + @pytest.mark.asyncio @pytest.mark.parametrize("stage", ("upload", "marker")) async def test_cancellation_retains_lease_until_storage_work_finishes( diff --git a/apps/api/tests/contract/test_shared_demo_corpus_contract.py b/apps/api/tests/contract/test_shared_demo_corpus_contract.py index 6205ca76c..150489da4 100644 --- a/apps/api/tests/contract/test_shared_demo_corpus_contract.py +++ b/apps/api/tests/contract/test_shared_demo_corpus_contract.py @@ -211,6 +211,57 @@ def reject_artifact_reads(*args: Any, **kwargs: Any) -> None: assert invalid_revision.status_code == 404 +@pytest.mark.asyncio +@pytest.mark.parametrize("is_demo", [False, True]) +async def test_page_body_preview_survives_private_and_demo_publication( + developer_api_client_factory: ClientFactory, is_demo: bool, +) -> None: + async with developer_api_client_factory() as client: + from shared.core.database import get_db_context + from shared.models.database.job import Job + from shared.models.database.job_result import JobResult + from shared.services.retrieval.publication_service import RetrievalPublicationService + + if is_demo: + job = await create_demo(client, file_name="preview.pdf") + else: + response = await client.post("/api/v2/jobs", json={ + "namespace": "preview-contract", "source_type": "file", "file_name": "preview.pdf", + }) + assert response.status_code == 200, response.text + job = response.json() + revision = str(uuid4()) + raw_content = "iv\nTable of Contents\nBody text\nfooter" + + def publish(db: Session) -> None: + record = db.execute(select(Job).where(Job.job_id == job["job_id"])).scalar_one() + db.add(JobResult(id=revision, job_id=record.job_id, delivery_mode="url", document_metadata={"parse_track": "page_memory"})) + db.flush() + chunks = [{ + "chunk_id": f"body-preview-{index}", "type": "page", "content": raw_content, + "path": f"preview.pdf/Section {index}", + "metadata": {"page_nums": [index + 1], "body_text_preview": preview}, + } for index, preview in enumerate(["Body text", ""])] + service = RetrievalPublicationService() + service.publish_document_state(db, job_id=record.job_id, job_result_id=revision, chunks=chunks) + service.publish_document_graph(db, job_id=record.job_id, job_result_id=revision) + record.status = "done" + + async with get_db_context() as db: + await db.run_sync(publish) + await db.commit() + response = await client.get( + f"/api/v1/documents/{job['document_id']}/chunks", + params={"job_result_id": revision, "page_size": 200}, + ) + assert response.status_code == 200, response.text + assert response.json()["job_result_id"] == revision + chunks = {chunk["chunk_id"]: chunk for chunk in response.json()["chunks"]} + assert chunks["body-preview-0"]["metadata"]["body_text_preview"] == "Body text" + assert chunks["body-preview-1"]["metadata"]["body_text_preview"] == "" + assert all(chunk["content"] == raw_content for chunk in chunks.values()) + + @pytest.mark.asyncio async def test_preparing_readiness_and_removed_materialization( developer_api_client_factory: ClientFactory, diff --git a/apps/worker/app/services/document_agent/pdf_text.py b/apps/worker/app/services/document_agent/pdf_text.py index 1e388a7a5..bde23c28d 100644 --- a/apps/worker/app/services/document_agent/pdf_text.py +++ b/apps/worker/app/services/document_agent/pdf_text.py @@ -173,6 +173,22 @@ def page_bands_map(raw: Any) -> dict[int, PageTextBands]: return {int(page): PageTextBands.from_any(value) for page, value in raw.items()} +def page_body_content_map(raw: Any) -> dict[int, str]: + """Keep only explicitly classified body lines; plain legacy text is unknown.""" + if not isinstance(raw, dict): + return {} + result: dict[int, str] = {} + for page, value in raw.items(): + has_regions = isinstance(value, PageTextBands) or ( + isinstance(value, dict) and isinstance(value.get("lines"), list) + ) + result[int(page)] = ( + "\n".join(line.text for line in PageTextBands.from_any(value).lines if line.region == "body") + if has_regions else "" + ) + return result + + def strip_margin_text( content: str, margin: str, diff --git a/apps/worker/app/services/document_parser/profiling/doc_profiler.py b/apps/worker/app/services/document_parser/profiling/doc_profiler.py index 8534793ce..f591940ea 100644 --- a/apps/worker/app/services/document_parser/profiling/doc_profiler.py +++ b/apps/worker/app/services/document_parser/profiling/doc_profiler.py @@ -8,7 +8,7 @@ from loguru import logger from app.services.document_agent.coordinator import ProfileCoordinator -from app.services.document_agent.pdf_text import page_content_map +from app.services.document_agent.pdf_text import page_body_content_map, page_content_map from app.services.document_agent.visual import purge_debug_visual_dirs, visual_debug_enabled from app.services.document_parser.orchestration.oversized_pdf_policy import ( build_oversized_pdf_profile_failed_exception, @@ -164,6 +164,9 @@ def _profile_pdf_with_db( page_full_text_cache=page_content_map( coordinator.blackboard.page_full_text_cache ), + page_body_text_cache=page_body_content_map( + coordinator.blackboard.page_full_text_cache + ), ) if profile.page_count > settings.MAX_PDF_PAGE_LIMIT: if oversized_policy != "page_memory": diff --git a/apps/worker/app/services/document_parser/profiling/profile_model.py b/apps/worker/app/services/document_parser/profiling/profile_model.py index b47810bb1..7627fcd65 100644 --- a/apps/worker/app/services/document_parser/profiling/profile_model.py +++ b/apps/worker/app/services/document_parser/profiling/profile_model.py @@ -44,6 +44,7 @@ class ParserDocumentProfile: anatomy: Any | None = None metrics: dict[str, Any] = field(default_factory=dict) page_full_text_cache: dict[int, str] = field(default_factory=dict) + page_body_text_cache: dict[int, str] = field(default_factory=dict) @property def is_pdf(self) -> bool: @@ -60,6 +61,7 @@ def has_structural_anatomy(self) -> bool: def to_dict(self) -> dict[str, Any]: data = asdict(self) data.pop("page_full_text_cache", None) + data.pop("page_body_text_cache", None) data["routing_category"] = self.routing_category.value if self.anatomy is not None and hasattr(self.anatomy, "to_dict"): data["anatomy"] = self.anatomy.to_dict() diff --git a/apps/worker/app/services/page_memory/body_text_preview.py b/apps/worker/app/services/page_memory/body_text_preview.py new file mode 100644 index 000000000..19ed29d0d --- /dev/null +++ b/apps/worker/app/services/page_memory/body_text_preview.py @@ -0,0 +1,7 @@ +"""Bound the display preview without changing stored PAGE content.""" + +from collections.abc import Iterable + + +def build_body_text_preview(texts: Iterable[str]) -> str: + return " ".join(" ".join(texts).split())[:180] diff --git a/apps/worker/app/services/page_memory/memory_service.py b/apps/worker/app/services/page_memory/memory_service.py index dcccb89cc..fc3c3ad5b 100644 --- a/apps/worker/app/services/page_memory/memory_service.py +++ b/apps/worker/app/services/page_memory/memory_service.py @@ -37,6 +37,7 @@ from shared.core.exceptions.domain_exceptions import ValidationException from shared.models.schemas.page_memory_config import PageMemoryConfig, resolve_asset_model +from app.services.page_memory.body_text_preview import build_body_text_preview @dataclass(frozen=True) @@ -122,6 +123,7 @@ def run(request: PageMemoryInput) -> tuple[str, pd.DataFrame]: page_texts=coerce_page_text_cache( getattr(profile, "page_full_text_cache", None) ), + body_text_by_page=getattr(profile, "page_body_text_cache", None), ) else: parsed_df = _build_page_dataframe( @@ -454,6 +456,7 @@ def _build_page_dataframe( page_assets_by_page=page_assets_by_page, node_summary_max_pages=page_memory_config.node_summary_max_pages, node_assembly_concurrency=page_memory_config.node_assembly_concurrency, + body_text_by_page=getattr(profile, "page_body_text_cache", None), ) logger.info( "[page_memory] C7 assembled {} node rows (verdict={})", @@ -834,6 +837,7 @@ def _build_whole_doc_dataframe( verdict: str, trace_recorder: Any | None = None, page_texts: dict[int, str] | None = None, + body_text_by_page: dict[int, str] | None = None, ) -> pd.DataFrame: pages = list(range(1, page_count + 1)) if page_count > 0 else [1] texts = coerce_page_text_cache(page_texts) @@ -853,7 +857,11 @@ def _build_whole_doc_dataframe( "connectto": "", "addtime": get_str_time(), "page_nums": ",".join(str(page) for page in pages), - "extra_metadata": {}, + "extra_metadata": { + "body_text_preview": build_body_text_preview( + (body_text_by_page or {}).get(page, "") for page in pages + ), + }, } _record_trace_stage( trace_recorder, diff --git a/apps/worker/app/services/page_memory/node_assembler.py b/apps/worker/app/services/page_memory/node_assembler.py index 1a2da30a8..f6eeaff1a 100644 --- a/apps/worker/app/services/page_memory/node_assembler.py +++ b/apps/worker/app/services/page_memory/node_assembler.py @@ -37,6 +37,8 @@ build_asset_rows, ) from app.services.page_memory.page_tagger import PageTagResult +from app.services.page_memory.body_text_preview import build_body_text_preview +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.page_memory.skeleton_extractor import SectionSkeleton from shared.services.ai.summary.engine import summarize @@ -211,6 +213,50 @@ def build_node_content( return "\n\n".join(segment for segment in segments if segment).strip() +def _find_body_heading(lines: list[str], title: str) -> int | None: + target = normalize_match_text(title) + if not target: + return None + for start in range(len(lines)): + for end in range(start + 1, min(start + 3, len(lines)) + 1): + if normalize_match_text(" ".join(lines[start:end])) == target: + return start + return None + + +def _preview_owned_body( + *, + view: NodePageView, + page_to_leaves: dict[int, list[LeafNode]], + body_text_by_page: dict[int, str], +) -> str: + """Respect physical ownership and conservative same-page title boundaries.""" + texts: list[str] = [] + for page in view.owned_pages: + lines = body_text_by_page.get(page, "").splitlines() + leaves = page_to_leaves.get(page, []) + current_index = next(index for index, leaf in enumerate(leaves) if leaf.section_path == view.leaf.section_path) + following = next((leaf for leaf in leaves[current_index + 1:] if leaf.start_page == page), None) + start = 0 + if page == view.leaf.start_page and view.leaf.title != "Root": + located = _find_body_heading(lines, view.leaf.title) + if located is None: + if following is not None: + break + continue + start = located + end = len(lines) + if following is not None: + located = _find_body_heading(lines[start:], following.title) + if located is None: + break + end = start + located + texts.append("\n".join(lines[start:end])) + if following is not None: + break + return build_body_text_preview(texts) + + def next_title_on_page( leaf: LeafNode, *, @@ -391,6 +437,7 @@ def build_node_rows( page_assets_by_page: dict[int, list[PageAsset]] | None = None, node_summary_max_pages: int = _NODE_SUMMARY_MAX_PAGES_DEFAULT, node_assembly_concurrency: int = 3, + body_text_by_page: dict[int, str] | None = None, ) -> list[dict[str, Any]]: """Assemble one row per leaf section node (node-granularity chunks).""" available_pages = set(raw_text_by_page.keys()) @@ -473,6 +520,10 @@ def _summarize_one( output_dir=output_dir, ), } + row["extra_metadata"]["body_text_preview"] = _preview_owned_body( + view=view, page_to_leaves=page_to_leaves, + body_text_by_page=body_text_by_page or {}, + ) rows.append(row) rows_by_path[leaf.section_path] = row diff --git a/apps/worker/tests/contract/test_page_body_text_preview_contract.py b/apps/worker/tests/contract/test_page_body_text_preview_contract.py new file mode 100644 index 000000000..5f090d27e --- /dev/null +++ b/apps/worker/tests/contract/test_page_body_text_preview_contract.py @@ -0,0 +1,121 @@ +"""Body-only PAGE previews survive parser ownership and result serialization.""" +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +from PIL import Image + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + + +def test_profile_preserves_classified_body_without_changing_full_text(monkeypatch) -> None: + from app.services.document_agent.pdf_text import LineDraft, build_page_text_bands + from app.services.document_parser.profiling import doc_profiler + from app.services.document_agent.manifest import DocumentProfile + + bands = build_page_text_bands([ + LineDraft("iv", region="header"), + LineDraft("Table of Contents", region="header"), + LineDraft("GLOSSARY OF TERMS", region="body"), + LineDraft("Actual body text", region="body"), + LineDraft("20", region="footer"), + ]) + + class Coordinator: + def __init__(self, **kwargs): + self.blackboard = SimpleNamespace( + page_count=3, doc_stats={}, global_signals={}, + page_full_text_cache={ + 1: bands, 2: "Legacy unclassified text", + 3: build_page_text_bands([LineDraft("header", region="header")]), + }, toc_result=None, + ) + + def run_coarse(self): + return DocumentProfile(False, ["04000000"], "Report", "generic") + + def run_lightweight_anatomy(self, **kwargs): + return None + + monkeypatch.setattr(doc_profiler, "ProfileCoordinator", Coordinator) + profile = doc_profiler._profile_pdf_with_db( + file_path="unused.pdf", filename="unused.pdf", job_id=None, + output_dir=None, db=None, + ) + assert profile.page_full_text_cache[1] == bands.content + assert profile.page_body_text_cache == {1: "GLOSSARY OF TERMS\nActual body text", 2: "", 3: ""} + assert "page_body_text_cache" not in profile.to_dict() + + +@pytest.mark.parametrize("has_body_cache", [True, False]) +def test_page_preview_respects_owned_pages_and_same_page_sections( + tmp_path: Path, monkeypatch, has_body_cache: bool, +) -> None: + from app.services.page_memory import memory_service, page_renderer, page_tagger, skeleton_extractor + from app.services.page_memory.page_tagger import PageTagResult + from app.services.page_memory.skeleton_extractor import SectionSkeleton + from shared.models.schemas.page_memory_config import PageMemoryConfig + from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks + from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder + + skeletons = [ + SectionSkeleton("doc.pdf/First", 1, 1, 1, "First", "doc.pdf"), + SectionSkeleton("doc.pdf/SharedOnly", 1, 1, 1, "SharedOnly", "doc.pdf"), + SectionSkeleton("doc.pdf/Second", 1, 1, 2, "Second", "doc.pdf"), + SectionSkeleton("doc.pdf/Empty", 1, 3, 3, "Empty", "doc.pdf"), + ] + raw = {1: "iv\nTable of Contents\nFirst\nSharedOnly\nSHARED BODY\nSecond\nSECOND BODY\nfooter", 2: "header\nSecond continuation\nfooter", 3: "Only header and footer"} + profile = SimpleNamespace(page_count=3, anatomy=SimpleNamespace(page_features=[], page_labels=[], toc_result=None), page_full_text_cache=raw) + if has_body_cache: + profile.page_body_text_cache = {1: "First\nSharedOnly\nSHARED BODY\nSecond\nSECOND BODY", 2: "Second continuation", 3: ""} + + def render(ctx, pages, **kwargs): + directory = tmp_path / kwargs["folder_name"] + directory.mkdir(parents=True, exist_ok=True) + output = [] + for page in pages: + path = directory / f"page-{page}.png" + Image.new("RGB", (2, 3), color="white").save(path) + output.append({"page": page, "png_path": str(path)}) + return output + + monkeypatch.delenv("IMAGE_MODEL", raising=False) + monkeypatch.setattr(skeleton_extractor, "extract_section_skeletons", lambda **kwargs: skeletons) + monkeypatch.setattr(page_renderer, "render_pages", render) + monkeypatch.setattr(page_tagger, "tag_pages", lambda **kwargs: [PageTagResult(page_index=page.page_index, summary="summary") for page in kwargs["pages"]]) + dataframe = memory_service._build_page_dataframe( + pdf_path="unused.pdf", filename="doc.pdf", output_dir=str(tmp_path), + profile=profile, verdict="page", page_memory_config=PageMemoryConfig(fine_min_pages=100, scope_concurrency=1, asset_extraction_enabled=False), + ) + chunks = dataframe_to_chunks(dataframe) + zipped = ZipChunkSchemaBuilder().format_chunks(chunks, {}, {}) + for output in [chunks, zipped]: + by_path = {chunk["path"]: chunk for chunk in output} + assert by_path["doc.pdf/First"]["content"] == raw[1] + assert "SAME-AS doc.pdf/First p1" in by_path["doc.pdf/Second"]["content"] + assert by_path["doc.pdf/First"]["metadata"]["body_text_preview"] == ("First" if has_body_cache else "") + assert by_path["doc.pdf/SharedOnly"]["metadata"]["body_text_preview"] == "" + assert by_path["doc.pdf/Second"]["metadata"]["body_text_preview"] == ("Second continuation" if has_body_cache else "") + assert by_path["doc.pdf/Empty"]["metadata"]["body_text_preview"] == "" + + +def test_whole_document_preview_is_body_only_and_bounded() -> None: + from app.services.page_memory.memory_service import _build_whole_doc_dataframe + from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks + + dataframe = _build_whole_doc_dataframe( + filename="whole.pdf", page_count=2, verdict="whole_doc", + page_texts={1: "header\n" + "body " * 100, 2: "footer"}, + body_text_by_page={1: "body " * 100, 2: ""}, + ) + chunk = dataframe_to_chunks(dataframe)[0] + assert chunk["metadata"]["body_text_preview"] == ("body " * 100).strip()[:180] + assert "header" in chunk["content"] and "footer" in chunk["content"] diff --git a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py index bedac5e1d..bcd0f4fbd 100644 --- a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py +++ b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py @@ -133,13 +133,13 @@ def test_build_node_rows_reuses_tags_without_vlm() -> None: leaf_a = by_path["demo.pdf/3 基本规定/3.1 职责"] assert leaf_a["page_nums"] == "231" assert leaf_a["content"] == "text-231" - assert leaf_a["extra_metadata"] == {} + assert leaf_a["extra_metadata"] == {"body_text_preview": ""} leaf_b = by_path["demo.pdf/3 基本规定/3.2 管理规定"] assert leaf_b["page_nums"] == "231,232" assert node_assembler.SAME_AS_PREFIX in leaf_b["content"] assert "text-232" in leaf_b["content"] - assert leaf_b["extra_metadata"] == {} + assert leaf_b["extra_metadata"] == {"body_text_preview": ""} def test_build_node_rows_preserves_order_under_summary_concurrency( diff --git a/docs/page-body-text-preview.md b/docs/page-body-text-preview.md new file mode 100644 index 000000000..ed1056140 --- /dev/null +++ b/docs/page-body-text-preview.md @@ -0,0 +1,65 @@ +# PAGE body text previews + +New PAGE chunks include `metadata.body_text_preview`, a string of at most 180 +Unicode characters with whitespace collapsed. It comes only from PROFILE lines +explicitly classified as `region=body`. Headers, footers, summaries and full +chunk content are not fallback sources. A known empty body or unavailable +classification produces an explicit empty string. + +The full text cache and PAGE `content`, chunk identifiers, page assets and +references remain unchanged. A separate in-memory body cache survives the +PROFILE-to-Page-Memory handoff and is excluded from serialized profile metadata. +Node assembly uses only pages physically owned by that chunk; shared `SAME-AS` +pages do not supply another node's preview. On section start pages, body text is +bounded by the actual current and following section headings. Ambiguous heading +boundaries produce no preview rather than borrowing adjacent section content. +Whole-document PAGE chunks follow the same body-only and length rules. + +The field survives parser DataFrame conversion, normal and shared-demo +publication, HTTP chunk metadata, result ZIP projection and canonical demo +bundle storage. Existing results without the field remain valid. Readers must +use a page-number label when the preview is missing or blank; they must not +guess a preview from legacy full text. + +## Verification + +Validation ran in an isolated Linux ARM64 environment: + +- All 409 Worker contracts pass, including PROFILE region preservation, + unclassified legacy text, shared-page ownership, same-page section limits, + explicit empty previews and whole-document length bounds. +- Three HTTP/PostgreSQL/storage contracts confirm normal-user and shared-demo + metadata persistence, empty-string preservation and canonical ZIP contents. +- Ruff and Pyright pass for the changed implementation modules. + +A replay of the actual SpaceX PDF physical page 20, using its saved profile +boundaries and the real node assembler, produced: + +> GLOSSARY OF TERMS The terms and abbreviations defined in this section are used throughout this prospectus: • “AI” or “artificial intelligence” refers to advanced computational tech + +This is a 180-character body preview for +`spacex-s1.pdf/GLOSSARY OF TERMS`. The original content remains intact, including +the page's existing margin text. This replay does not modify a published result. + +## Historical results and optional demo backfill + +This change affects new parses. It does not automatically repair old normal-user +results, cached Blob snapshots or immutable ZIPs. Those readers continue to use +page-number labels until trusted preview metadata is available. + +A separate demo-only metadata backfill can avoid models and full document +parsing when the exact revision's classified line cache is retained. Otherwise, +the saved `header_y`/`footer_y` profile boundaries and the same normalized +`source.pdf` can reproduce the deterministic line classification. For DOCX, +use the retained normalized PDF; converting the original again may change page +layout. Missing profile, source or ownership evidence must be reported as a +skip, without inventing a preview. + +Prepare a dry-run plan keyed by document revision and chunk identifier. Include +the content hash, previous field presence/value and proposed preview. Applying +a reviewed plan must recheck the current published revision and content hash, +update only `body_text_preview` in demo chunk metadata, and retain a reversible +journal. Repeated application must be idempotent; rollback must compare the +applied value before restoring the old field. Original content, references, +page assets and ZIP objects remain unchanged. Such a DB-only backfill benefits +HTTP demo reads and does not claim to update historical ZIP or Blob artifacts. diff --git a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py index 8eb6fb760..4968b213b 100644 --- a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py +++ b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py @@ -44,6 +44,7 @@ def iterrows(self) -> Iterable[tuple[object, _ParserRow]]: ... class ChunkMetadata(TypedDict, total=False): + body_text_preview: str keywords: list[str] entities: list[dict[str, str]] asset_title: str diff --git a/packages/shared-python/shared/services/storage/zip_chunk_schema.py b/packages/shared-python/shared/services/storage/zip_chunk_schema.py index d9043cb9c..3478cb358 100644 --- a/packages/shared-python/shared/services/storage/zip_chunk_schema.py +++ b/packages/shared-python/shared/services/storage/zip_chunk_schema.py @@ -127,6 +127,8 @@ def format_chunks( page_assets = _normalize_page_assets(existing_metadata.get("page_assets")) if page_assets: metadata["page_assets"] = page_assets + if isinstance(existing_metadata.get("body_text_preview"), str): + metadata["body_text_preview"] = existing_metadata["body_text_preview"] formatted_chunk = { "chunk_id": chunk_id,