diff --git a/CONTEXT.md b/CONTEXT.md index faaae583a..df1f3799e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -59,6 +59,18 @@ The hierarchical navigation node derived from parsed headings and section paths. The retrieval-visible text, image, or table row attached to a Document Section. +### Document Root + +The existing document-level container above its chapters. It can own opening +content without introducing another chapter. + +### Document Prefix + +All physical pages before the first recognized chapter, including covers, +opening text, and printed table-of-contents pages. These pages belong directly +to the existing document Root; they are not a separate chapter or part of the +first chapter. + ### Document Ingestion The workflow that creates a Job, accepts a file or URL source, confirms upload diff --git a/apps/worker/app/services/page_memory/_serialization.py b/apps/worker/app/services/page_memory/_serialization.py index 87caea104..835800208 100644 --- a/apps/worker/app/services/page_memory/_serialization.py +++ b/apps/worker/app/services/page_memory/_serialization.py @@ -8,6 +8,8 @@ from app.services.page_memory._utils import ( collapse_page_ranges, + is_document_prefix, + is_fallback_root, page_scope_info, ) from shared.services.chunks.path_segments import split_escaped_document_path @@ -39,11 +41,7 @@ def derive_hierarchy_page_scope( if not skeletons: return list(range(1, page_count + 1)) - has_only_root_fallback = all( - getattr(item, "title", "") == "Root" - or (getattr(item, "evidence", {}) or {}).get("source") == "fallback_root" - for item in skeletons - ) + has_only_root_fallback = all(is_fallback_root(item) for item in skeletons) if has_only_root_fallback: return list(range(1, page_count + 1)) @@ -86,6 +84,7 @@ def serialize_skeletons(skeletons: list[Any]) -> list[dict[str, Any]]: "start_page": item.start_page, "end_page": item.end_page, "parent_path": item.parent_path, + "evidence": dict(item.evidence), } for item in skeletons ] @@ -100,6 +99,9 @@ def build_hierarchy_tree(skeletons: list[Any]) -> dict[str, Any]: """ hierarchy: dict[str, Any] = {} for skel in skeletons: + # Root-owned body is not a chapter nested beneath its own filename. + if is_document_prefix(skel): + continue parts = split_escaped_document_path( getattr(skel, "section_path", "") or "" ) diff --git a/apps/worker/app/services/page_memory/_utils.py b/apps/worker/app/services/page_memory/_utils.py index 23d89f7f9..2635f2c00 100644 --- a/apps/worker/app/services/page_memory/_utils.py +++ b/apps/worker/app/services/page_memory/_utils.py @@ -6,6 +6,36 @@ from typing import Any +def is_document_prefix(skeleton: Any) -> bool: + """Identify body owned by the existing document root before its chapters.""" + return (getattr(skeleton, "evidence", {}) or {}).get("source") == "document_prefix" + + +def is_fallback_root(skeleton: Any) -> bool: + """Keep prefix ownership separate from the full-document Root fallback.""" + if is_document_prefix(skeleton): + return False + return ( + getattr(skeleton, "title", "") == "Root" + or (getattr(skeleton, "evidence", {}) or {}).get("source") == "fallback_root" + ) + + +def derive_excluded_toc_pages( + *, + skeletons: list[Any], + toc_pages: list[int] | None, +) -> list[int]: + """Exclude chapter TOCs while retaining every physical prefix page.""" + prefix_pages: set[int] = { + page + for skeleton in skeletons + if is_document_prefix(skeleton) + for page in range(skeleton.start_page, skeleton.end_page + 1) + } + return sorted({int(page) for page in (toc_pages or [])} - prefix_pages) + + def sort_skeletons(skeletons: list[Any]) -> list[Any]: """Order skeletons by start page only. @@ -73,20 +103,17 @@ def build_hierarchy_scopes( ) -> list[CoarseScope]: """Split skeleton list into per-leaf scopes for independent processing. - Each TOC leaf skeleton becomes its own scope. ``skeletons`` is leaf-level - (produced by ``extract_section_skeletons`` → ``resolve_hierarchy_page_ranges``, - which only emits leaf nodes). When multiple leaves share the same page range + Each TOC leaf skeleton and any document-root prefix owner gets its own + scope. Chapter ranges come from ``resolve_hierarchy_page_ranges``; the + root owner preserves physical pages before those chapters without adding + a section. When multiple leaves share the same page range (e.g. unlocated leaves that fell back to the same parent scope), they are merged into a single scope to keep ``scope_id`` unique. """ if not skeletons: return [] - root_fallback = all( - getattr(item, "title", "") == "Root" - or (getattr(item, "evidence", {}) or {}).get("source") == "fallback_root" - for item in skeletons - ) + root_fallback = all(is_fallback_root(item) for item in skeletons) if root_fallback: ordered = sort_skeletons(skeletons) return [ diff --git a/apps/worker/app/services/page_memory/fine_hierarchy.py b/apps/worker/app/services/page_memory/fine_hierarchy.py index f34b9d8fb..52e7365c9 100644 --- a/apps/worker/app/services/page_memory/fine_hierarchy.py +++ b/apps/worker/app/services/page_memory/fine_hierarchy.py @@ -24,7 +24,11 @@ from app.services.page_memory.page_tagger import PageTagResult from app.services.page_memory.hierarchy_level_resolver import PageHierarchyLevelResolver from app.services.page_memory.skeleton_extractor import SectionSkeleton -from app.services.page_memory._utils import page_scope_info, sort_skeletons +from app.services.page_memory._utils import ( + is_document_prefix, + page_scope_info, + sort_skeletons, +) from shared.services.ai.llm_overrides import get_text_client from shared.services.chunks.path_segments import append_document_path @@ -57,6 +61,9 @@ def refine_fat_leaf_skeletons( refined: list[SectionSkeleton] = [] for index, skeleton in enumerate(coarse_skeletons): + if is_document_prefix(skeleton): + refined.append(skeleton) + continue exclusive_end = _exclusive_end(coarse_skeletons, index) skel_pages = set(range(skeleton.start_page, exclusive_end + 1)) overlap = skel_pages & fat_leaf_pages @@ -126,6 +133,8 @@ def compute_fat_leaf_pages( excluded = {int(page) for page in (toc_pages or [])} fat_pages: set[int] = set() for idx, skel in enumerate(skeletons): + if is_document_prefix(skel): + continue exclusive_end = _exclusive_end(skeletons, idx) body_pages = [ page diff --git a/apps/worker/app/services/page_memory/memory_service.py b/apps/worker/app/services/page_memory/memory_service.py index bd97ddaa0..dcccb89cc 100644 --- a/apps/worker/app/services/page_memory/memory_service.py +++ b/apps/worker/app/services/page_memory/memory_service.py @@ -22,6 +22,7 @@ CoarseScope, build_hierarchy_scopes, collapse_page_ranges, + derive_excluded_toc_pages, page_scope_info, sort_skeletons, ) @@ -274,7 +275,10 @@ def _build_page_dataframe( page_count=page_count, ) toc_result = getattr(anatomy, "toc_result", None) if anatomy is not None else None - toc_pages = list(getattr(toc_result, "toc_pages", None) or []) + toc_pages = derive_excluded_toc_pages( + skeletons=skeletons, + toc_pages=list(getattr(toc_result, "toc_pages", None) or []), + ) coarse_pages_scope = pages_excluding_toc( _derive_hierarchy_page_scope( skeletons=skeletons, @@ -571,6 +575,9 @@ def _run_hierarchy_scope( ) scope_skeletons = sort_skeletons(scope.skeletons) + toc_pages = derive_excluded_toc_pages( + skeletons=scope_skeletons, toc_pages=toc_pages, + ) scope_manifest = _scope_manifest( scope_id=scope.scope_id, skeletons=scope_skeletons, diff --git a/apps/worker/app/services/page_memory/skeleton_extractor.py b/apps/worker/app/services/page_memory/skeleton_extractor.py index 07b366277..8154ad1bb 100644 --- a/apps/worker/app/services/page_memory/skeleton_extractor.py +++ b/apps/worker/app/services/page_memory/skeleton_extractor.py @@ -28,6 +28,7 @@ select_global_toc_hierarchies, ) from loguru import logger +from app.services.page_memory._utils import is_document_prefix from shared.services.chunks.path_segments import ( append_document_path, join_document_path, @@ -205,6 +206,21 @@ def extract_section_skeletons( ) skeletons.extend(secondary_skeletons) + first_chapter_page: int = min(skeleton.start_page for skeleton in skeletons) + if first_chapter_page > 1: + skeletons.insert( + 0, + SectionSkeleton( + section_path=join_document_path([filename]), + level=0, + start_page=1, + end_page=first_chapter_page - 1, + title="Root", + parent_path=None, + evidence={"source": "document_prefix"}, + ), + ) + _log_unlocated_title_warnings(filename=filename, skeletons=skeletons) return skeletons @@ -452,7 +468,7 @@ def _collapse_node(path: str) -> None: # Re-read children after recursive collapse may have mutated by_path child_paths = children_of.get(path, []) - if len(child_paths) == 1: + if len(child_paths) == 1 and not is_document_prefix(node): only_child_path = child_paths[0] only_child = by_path[only_child_path] grandchild_paths = children_of.get(only_child_path, []) diff --git a/apps/worker/scripts/page_memory/_debug_pm_shared.py b/apps/worker/scripts/page_memory/_debug_pm_shared.py index 900d5d122..144dd2837 100644 --- a/apps/worker/scripts/page_memory/_debug_pm_shared.py +++ b/apps/worker/scripts/page_memory/_debug_pm_shared.py @@ -37,6 +37,7 @@ from loguru import logger from app.services.page_memory._utils import ( + derive_excluded_toc_pages, page_scope_info, scope_id_for_pages, sort_skeletons, @@ -1342,7 +1343,7 @@ def load_hierarchy_artifact(path: Path) -> tuple[dict[str, Any], list[Any]]: SectionSkeleton( section_path=section_path, title=title, - level=int(node.get("level") or 1), + level=int(node.get("level", 1)), start_page=int(node.get("start_page") or 1), end_page=int(node.get("end_page") or node.get("start_page") or 1), parent_path=node.get("parent_path"), @@ -1531,7 +1532,10 @@ def build_debug_coarse_scopes( from app.services.page_memory._utils import build_hierarchy_scopes toc_result = getattr(anatomy, "toc_result", None) if anatomy is not None else None - toc_pages = list(getattr(toc_result, "toc_pages", None) or []) + toc_pages = derive_excluded_toc_pages( + skeletons=skeletons, + toc_pages=list(getattr(toc_result, "toc_pages", None) or []), + ) toc_page_set = set() for raw in toc_pages: try: diff --git a/apps/worker/scripts/page_memory/debug_pm_stage3_scope_fine_hierarchy.py b/apps/worker/scripts/page_memory/debug_pm_stage3_scope_fine_hierarchy.py index bc885c352..ce482cd60 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage3_scope_fine_hierarchy.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage3_scope_fine_hierarchy.py @@ -67,7 +67,9 @@ def _resolve_scope_processing_pages( toc_pages: list[int], ) -> tuple[list[int], list[int]]: from app.services.document_agent.structure.toc_anchoring import pages_excluding_toc + from app.services.page_memory._utils import derive_excluded_toc_pages + toc_pages = derive_excluded_toc_pages(skeletons=skeletons, toc_pages=toc_pages) processing_pages = [ int(page) for page in (scope_meta.get("processing_pages") or []) ] @@ -132,6 +134,8 @@ def _run_fine_hierarchy_for_scope( require_file(skel_path, hint=f"Stage 3 should have created {skel_path}") scope_meta, active_skeletons = load_scope_skeletons_artifact(skel_path) active_skeletons = sort_skeletons(active_skeletons) + from app.services.page_memory._utils import derive_excluded_toc_pages + toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages) strategy = str(scope_meta.get("strategy") or "coarse_scope") processing_pages, excluded_toc_pages = _resolve_scope_processing_pages( scope_meta=scope_meta, @@ -358,6 +362,8 @@ def main() -> int: toc_pages = list(getattr(toc_result, "toc_pages", None) or []) page_memory_config = PageMemoryConfig.default() skeletons = load_pipeline_skeletons(state_path) + from app.services.page_memory._utils import derive_excluded_toc_pages + toc_pages = derive_excluded_toc_pages(skeletons=skeletons, toc_pages=toc_pages) logger.info("█" * 70) logger.info(f" STAGE 3: SCOPE + FINE HIERARCHY — {filename}") diff --git a/apps/worker/scripts/page_memory/debug_pm_stage4_assets.py b/apps/worker/scripts/page_memory/debug_pm_stage4_assets.py index 72bb05c89..c08f0c0d9 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage4_assets.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage4_assets.py @@ -74,6 +74,8 @@ def _load_scope_asset_context( hint=f"Run Stage 3 first to produce {fine_hierarchy_path}", ) prior_scope, active_skeletons = load_hierarchy_artifact(fine_hierarchy_path) + from app.services.page_memory._utils import derive_excluded_toc_pages + toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages) if not active_skeletons: logger.warning( " [scope {}] no skeletons in fine_hierarchy.json — skipping", @@ -184,12 +186,7 @@ def main() -> int: if context is not None: scope_contexts.append(context) - from app.services.document_agent.structure.toc_anchoring import pages_excluding_toc - - union_pages = pages_excluding_toc( - sorted({page for context in scope_contexts for page in context.pages}), - toc_pages, - ) + union_pages = sorted({page for context in scope_contexts for page in context.pages}) logger.info( "🔬 document C5: {} unique pages across {} scopes", len(union_pages), diff --git a/apps/worker/scripts/page_memory/debug_pm_stage5_tagging_finalize.py b/apps/worker/scripts/page_memory/debug_pm_stage5_tagging_finalize.py index 09113e4aa..2dc5c5063 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage5_tagging_finalize.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage5_tagging_finalize.py @@ -83,6 +83,8 @@ def _run_tagging_for_scope( fine_hierarchy_path = scope_dir / "fine_hierarchy.json" require_file(fine_hierarchy_path, hint=f"Run Stage 3 to produce {fine_hierarchy_path}") prior_scope, active_skeletons = load_hierarchy_artifact(fine_hierarchy_path) + from app.services.page_memory._utils import derive_excluded_toc_pages + toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages) if not active_skeletons: logger.warning(" [scope {}] no skeletons — skipping", scope_id) return ScopeResult( diff --git a/apps/worker/tests/contract/test_page_memory_root_prefix_contract.py b/apps/worker/tests/contract/test_page_memory_root_prefix_contract.py new file mode 100644 index 000000000..1c63b01d3 --- /dev/null +++ b/apps/worker/tests/contract/test_page_memory_root_prefix_contract.py @@ -0,0 +1,273 @@ +"""Opening physical pages survive skeleton resolution through ZIP publication.""" + +from __future__ import annotations + +import os +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from PIL import Image + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + + +@pytest.mark.parametrize( + ("page_count", "chapter_pages", "toc_pages"), + [ + (9, [5, 6, 7, 8, 9], []), + (407, [20], [17, 18]), + (9, [5], [2, 7]), + (9, [1, 5], []), + (9, [], []), + ], + ids=[ + "meta-outline-prefix", + "spacex-prefix-includes-toc", + "chapter-toc-policy-unchanged", + "chapter-on-first-page", + "no-toc-fallback", + ], +) +def test_opening_pages_keep_root_ownership_and_page_citation_assets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + page_count: int, + chapter_pages: list[int], + toc_pages: list[int], +) -> None: + # Database contracts evict application modules after collection. Resolve + # live modules so provider patches reach the same renderer as the use case. + from app.services.document_agent.structure.anchoring_primitives import ( + SkeletonAnchor, + serialize_skeleton_anchor, + serialize_title_node, + ) + from app.services.document_agent.structure.hierarchy_locator import ( + TitleMatch, + TitleNode, + ) + from app.services.page_memory import memory_service, page_renderer, page_tagger + from app.services.page_memory.page_tagger import PageTagResult + from shared.models.schemas.page_memory_config import PageMemoryConfig + from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks + from shared.services.retrieval.search.lexical_text import section_path_from_chunk_path + from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder + from shared.services.storage.zip_doc_navigation import ZipDocNavigationBuilder + + filename: str = "document.pdf" + titles: list[str] = [f"Chapter {page}" for page in chapter_pages] + anatomy: SimpleNamespace = SimpleNamespace( + page_count=page_count, + page_features=[], + page_labels=[], + toc_result=SimpleNamespace(toc_pages=toc_pages), + toc_hierarchies=[ + { + "toc_range": [17, 18] if toc_pages else [1, 1], + "toc_range_unit": "page", + "toc_with_level": [ + {"heading": title, "level": 1, "page_number": page} + for title, page in zip(titles, chapter_pages) + ], + } + ] + if chapter_pages + else [], + skeleton_nodes=[ + serialize_title_node(TitleNode(title=title, level=1, printed_page=page)) + for title, page in zip(titles, chapter_pages) + ], + skeleton_anchor=serialize_skeleton_anchor( + SkeletonAnchor( + offset=0, + offset_status="ok", + match_overrides={ + (title,): TitleMatch( + page=page, + source="anchored", + matched_line=title, + candidates=[page], + evidence={}, + ) + for title, page in zip(titles, chapter_pages) + }, + null_page_report=[], + bulk_count=len(titles), + pruned_count=0, + locate_method="offset_guided_bulk", + ) + ), + ) + page_texts: dict[int, str] = { + page: f"Body on physical page {page}." for page in range(1, page_count + 1) + } + profile: SimpleNamespace = SimpleNamespace( + page_count=page_count, + anatomy=anatomy, + page_full_text_cache=page_texts, + ) + + def render_pages( + ctx: object, pages: list[int], **kwargs: object + ) -> list[dict[str, object]]: + image_directory: Path = tmp_path / str(kwargs["folder_name"]) + image_directory.mkdir(parents=True, exist_ok=True) + rendered: list[dict[str, object]] = [] + for page in pages: + image_path: Path = image_directory / f"page-{page}.png" + Image.new("RGB", (2, 3), color="white").save(image_path) + rendered.append({"page": page, "png_path": str(image_path)}) + return rendered + + def tag_pages(**kwargs: object) -> list[PageTagResult]: + pages = kwargs["pages"] + assert isinstance(pages, list) + return [ + PageTagResult(page_index=page.page_index, summary=f"Page {page.page_index}") + for page in pages + ] + + def retain_chapter_titles(**kwargs: object) -> list[PageTagResult]: + pages = kwargs["fat_leaf_pages"] + assert isinstance(pages, set) + assert not (pages & set(range(1, chapter_pages[0]))) + tags = kwargs["tag_results"] + assert isinstance(tags, list) + return tags + + monkeypatch.delenv("IMAGE_MODEL", raising=False) + monkeypatch.setattr(page_renderer, "render_pages", render_pages) + monkeypatch.setattr(page_tagger, "tag_pages", tag_pages) + monkeypatch.setattr(page_tagger, "tag_page_titles", retain_chapter_titles) + + dataframe = memory_service._build_page_dataframe( + pdf_path=str(tmp_path / filename), + filename=filename, + output_dir=str(tmp_path), + profile=profile, + verdict="page", + page_memory_config=PageMemoryConfig( + scope_concurrency=3 if page_count == 407 else 1, + fine_min_pages=4 if chapter_pages and chapter_pages[0] > 1 else page_count, + asset_extraction_enabled=False, + ), + ) + chunks = dataframe_to_chunks(dataframe) + published: list[dict[str, object]] = ZipChunkSchemaBuilder().format_chunks( + chunks, + image_files_map={}, + table_files_map={}, + ) + expected_prefix: list[int] = ( + list(range(1, chapter_pages[0])) + if chapter_pages + else list(range(1, page_count + 1)) + ) + root_path: str = filename if chapter_pages else f"{filename}/Root" + root_chunks = [chunk for chunk in published if chunk["path"] == root_path] + assert len(root_chunks) == bool(expected_prefix) + if expected_prefix: + root_metadata = root_chunks[0]["metadata"] + assert root_metadata["page_nums"] == expected_prefix + assert [ + asset["page_num"] for asset in root_metadata["page_assets"] + ] == expected_prefix + assert all( + page_texts[page] in root_chunks[0]["content"] for page in expected_prefix + ) + assert ( + section_path_from_chunk_path(root_path, source_file_name=filename) == "Root" + ) + navigation = ZipDocNavigationBuilder().build_doc_nav(published, filename) + root_sections = [ + section + for section in navigation["sections"] + if section["path"] == root_path + ] + assert len(root_sections) == 1 + assert root_sections[0]["chunk_count"] == len(expected_prefix) + if chapter_pages: + assert not any(chunk["path"] == f"{filename}/Root" for chunk in published) + + assert [chunk["path"] for chunk in published if chunk["path"] != root_path] == [ + f"{filename}/{title}" for title in titles + ] + covered_pages: set[int] = { + page for chunk in published for page in chunk["metadata"]["page_nums"] + } + assert covered_pages == set(range(1, page_count + 1)) - ( + set(toc_pages) - set(expected_prefix) + ) + for chunk in published: + for asset in chunk["metadata"]["page_assets"]: + assert (tmp_path / asset["artifact_ref"]).is_file() + + +def test_staged_prefix_scope_retains_root_identity_and_printed_toc( + tmp_path: Path, +) -> None: + from app.services.page_memory._serialization import serialize_hierarchy_artifact + from app.services.page_memory.skeleton_extractor import SectionSkeleton + + prefix: SectionSkeleton = SectionSkeleton( + section_path="spacex.pdf", + level=0, + start_page=1, + end_page=19, + title="Root", + parent_path=None, + evidence={"source": "document_prefix"}, + ) + artifact_path: Path = tmp_path / "fine_hierarchy.json" + artifact_path.write_text(json.dumps(serialize_hierarchy_artifact([prefix]))) + repository: Path = Path(__file__).resolve().parents[4] + # Staged helpers monkey-patch gevent, so exercise their public file boundary + # in a separate process rather than modifying the contract runner's runtime. + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import sys +from pathlib import Path +sys.path.insert(0, 'apps/worker/scripts/page_memory') +from _debug_pm_shared import load_hierarchy_artifact, build_debug_coarse_scopes +from debug_pm_stage3_scope_fine_hierarchy import _resolve_scope_processing_pages +from debug_pm_stage4_assets import _load_scope_asset_context +from app.services.page_memory._serialization import derive_hierarchy_page_scope, build_hierarchy_tree +from app.services.page_memory.fine_hierarchy import compute_fat_leaf_pages +from types import SimpleNamespace +path = Path(sys.argv[1]) +_, skeletons = load_hierarchy_artifact(path) +assert skeletons[0].level == 0 +assert skeletons[0].evidence['source'] == 'document_prefix' +assert derive_hierarchy_page_scope(skeletons=skeletons, page_count=407) == list(range(1, 20)) +assert build_hierarchy_tree(skeletons) == {} +assert compute_fat_leaf_pages(skeletons, min_pages=4, toc_pages=[17, 18]) == set() +anatomy = SimpleNamespace(toc_result=SimpleNamespace(toc_pages=[17, 18])) +scopes = build_debug_coarse_scopes(skeletons=skeletons, filename='spacex.pdf', page_count=407, anatomy=anatomy) +assert len(scopes) == 1 +assert scopes[0]['processing_pages'] == list(range(1, 20)) +assert scopes[0]['excluded_toc_pages'] == [] +pages, excluded = _resolve_scope_processing_pages(scope_meta={}, skeletons=skeletons, page_count=407, toc_pages=[17, 18]) +assert pages == list(range(1, 20)) +assert excluded == [] +context = _load_scope_asset_context(scope_id='p1-19', scope_dir=path.parent, page_count=407, toc_pages=[17, 18]) +assert context.pages == list(range(1, 20)) +""", + str(artifact_path), + ], + cwd=repository, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py index 19bc37727..c7c57c3d7 100644 --- a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py +++ b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py @@ -183,8 +183,11 @@ def _boom(*_args, **_kwargs): ) assert skeletons - assert skeletons[0].title == "Ch1" - assert skeletons[0].start_page == 2 + assert skeletons[0].section_path == "doc.pdf" + assert skeletons[0].start_page == skeletons[0].end_page == 1 + assert skeletons[1].title == "Ch1" + assert skeletons[1].section_path == "doc.pdf/Ch1" + assert skeletons[1].start_page == 2 def test_shard_plan_reads_offset_and_does_not_calibrate() -> None: diff --git a/docs/spacex-hierarchy-fix.md b/docs/spacex-hierarchy-fix.md index 62eca2ad0..3ba737dbe 100644 --- a/docs/spacex-hierarchy-fix.md +++ b/docs/spacex-hierarchy-fix.md @@ -50,3 +50,86 @@ reproduced the loop before this change and finishes in four turns afterward. Both real reader launch-comparison queries finished with precise page ranges. See [the validation report](shared-demo-validation-20261003.md) for timings and the exact verification boundary. + +## Preserve the document prefix before anchored chapters + +A separate production result omitted the pages before the first anchored +chapter: Meta's nine-page release contained only pages 5–9, and SpaceX's +407-page filing contained only pages 20–407. Resolving a TOC correctly did not +guarantee complete document coverage: only the resolved section ranges were +sent to rendering, tagging, and node assembly. + +The document prefix is every physical page before the first recognized chapter, +including printed TOC pages. It belongs directly to the existing document root. +Its chunk path is the filename itself; no `filename/Root` chapter is inserted, +and existing chapter paths and ranges stay unchanged. Publication already maps +a filename-only chunk to its canonical `Root` section. Navigation retains its +existing root-body projection at the filename path. + +The prefix has explicit `document_prefix` evidence so an isolated prefix scope +does not expand into the full-document Root fallback. It receives ordinary +rendering, page tagging, and full-page citation assets, but does not undergo +chapter-title refinement or single-child collapse. Prefix TOCs are retained +through production and staged replay; TOC policy inside chapters is unchanged. +The source evidence and level zero survive hierarchy artifact round trips. + +This repair covers opening pages only. It does not fill arbitrary middle or +trailing gaps, regenerate existing production results, or change no-TOC +full-document fallback behavior. The historical full-provider validation above +is separate from the prefix-repair validation. + +### Prefix-repair verification on 2026-10-22 + +Validation ran in an isolated Linux ARM64 environment, based on `main` commit +`2ed1889626f63240bd0894ec0c24b7a7c5217dfb`. + +The new output contract failed for both missing-prefix examples before the +repair; the first-page chapter and no-TOC fallback controls already passed. +A separate staged-artifact contract reproduced the loss of level zero on +reload before its repair. The final focused run passed **102 contracts**: + +```sh +.venv/bin/pytest apps/worker/tests/contract/test_page_memory* \ + apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py -q +``` + +The full Worker contract suite also passed **405 contracts**, including the +database contracts that reload application modules between environments: + +```sh +.venv/bin/pytest apps/worker/tests/contract -q +``` + +This includes direct-root text and full-page citation assets, TOC retention in +the prefix, unchanged chapter TOC policy, first-page chapters, no-TOC fallback, +filename-only publication/navigation projection, and staged artifact reload. +Ruff passed on the page-memory modules, changed stage scripts, and contracts; +Pyright passed on the five changed application modules with zero errors and +warnings. + +Captured production anatomy was also replayed against the actual originals: + +| Document | Real root-body output | Citation assets | Coverage evidence | +| --- | --- | --- | --- | +| Meta Q1 2026 Exhibit 99.1 | Filename path, physical pages 1–4, 11,265 text characters | Four real full-page PNGs | All nine pages assembled; five existing chapter paths retained | +| SpaceX S-1 | Filename path, physical pages 1–19, 24,086 text characters; printed TOC page 17 retained | Nineteen real full-page PNGs | New prefix plus 683 unchanged original body chunks covers all 407 pages | + +The replay used real PDF rasterization, node assembly, and ZIP chunk-schema +conversion. It reused captured chapter anatomy and substituted page-tag +responses; it did not call model providers or publish a document revision. +SpaceX rendered only the missing prefix and reused the original body results, +rather than rerunning all 407 pages. Thus this is captured-profile seam +verification, not a fresh full-provider parse or production deployment. + +### Existing demo revisions after deployment + +Deploying the Worker repair does not regenerate existing parsed revisions. +After confirming the deployed Worker includes this change, reparse the known +affected Meta release and SpaceX filing through the maintainer demo ingestion +flow. Audit the remaining READY demos for missing opening pages before +selecting additional sources. The restoration helper supports source selection +and `--reparse-ready`; keep the previous revision available while an update runs. + +Validate physical-page coverage, prefix TOC retention, filename-only root-body +paths, and page-citation assets on each new revision. Confirm that downstream +readers consume the new revision after publication.