Preserve opening pages at the document root

CONTEXT.md

@@ -59,6 +59,18 @@ The hierarchical navigation node derived from parsed headings and section paths.
 
 The retrieval-visible text, image, or table row attached to a Document Section.
 
+### Document Root
+
+The existing document-level container above its chapters. It can own opening
+content without introducing another chapter.
+
+### Document Prefix
+
+All physical pages before the first recognized chapter, including covers,
+opening text, and printed table-of-contents pages. These pages belong directly
+to the existing document Root; they are not a separate chapter or part of the
+first chapter.
+
 ### Document Ingestion
 
 The workflow that creates a Job, accepts a file or URL source, confirms upload

apps/worker/app/services/page_memory/_serialization.py

@@ -8,6 +8,8 @@
 
 from app.services.page_memory._utils import (
     collapse_page_ranges,
+    is_document_prefix,
+    is_fallback_root,
     page_scope_info,
 )
 from shared.services.chunks.path_segments import split_escaped_document_path
@@ -39,11 +41,7 @@ def derive_hierarchy_page_scope(
     if not skeletons:
         return list(range(1, page_count + 1))
 
-    has_only_root_fallback = all(
-        getattr(item, "title", "") == "Root"
-        or (getattr(item, "evidence", {}) or {}).get("source") == "fallback_root"
-        for item in skeletons
-    )
+    has_only_root_fallback = all(is_fallback_root(item) for item in skeletons)
     if has_only_root_fallback:
         return list(range(1, page_count + 1))
 
@@ -86,6 +84,7 @@ def serialize_skeletons(skeletons: list[Any]) -> list[dict[str, Any]]:
             "start_page": item.start_page,
             "end_page": item.end_page,
             "parent_path": item.parent_path,
+            "evidence": dict(item.evidence),
         }
         for item in skeletons
     ]
@@ -100,6 +99,9 @@ def build_hierarchy_tree(skeletons: list[Any]) -> dict[str, Any]:
     """
     hierarchy: dict[str, Any] = {}
     for skel in skeletons:
+        # Root-owned body is not a chapter nested beneath its own filename.
+        if is_document_prefix(skel):
+            continue
         parts = split_escaped_document_path(
             getattr(skel, "section_path", "") or ""
         )

apps/worker/app/services/page_memory/_utils.py

@@ -6,6 +6,36 @@
 from typing import Any
 
 
+def is_document_prefix(skeleton: Any) -> bool:
+    """Identify body owned by the existing document root before its chapters."""
+    return (getattr(skeleton, "evidence", {}) or {}).get("source") == "document_prefix"
+
+
+def is_fallback_root(skeleton: Any) -> bool:
+    """Keep prefix ownership separate from the full-document Root fallback."""
+    if is_document_prefix(skeleton):
+        return False
+    return (
+        getattr(skeleton, "title", "") == "Root"
+        or (getattr(skeleton, "evidence", {}) or {}).get("source") == "fallback_root"
+    )
+
+
+def derive_excluded_toc_pages(
+    *,
+    skeletons: list[Any],
+    toc_pages: list[int] | None,
+) -> list[int]:
+    """Exclude chapter TOCs while retaining every physical prefix page."""
+    prefix_pages: set[int] = {
+        page
+        for skeleton in skeletons
+        if is_document_prefix(skeleton)
+        for page in range(skeleton.start_page, skeleton.end_page + 1)
+    }
+    return sorted({int(page) for page in (toc_pages or [])} - prefix_pages)
+
+
 def sort_skeletons(skeletons: list[Any]) -> list[Any]:
     """Order skeletons by start page only.
 
@@ -73,20 +103,17 @@ def build_hierarchy_scopes(
 ) -> list[CoarseScope]:
     """Split skeleton list into per-leaf scopes for independent processing.
 
-    Each TOC leaf skeleton becomes its own scope. ``skeletons`` is leaf-level
-    (produced by ``extract_section_skeletons`` → ``resolve_hierarchy_page_ranges``,
-    which only emits leaf nodes). When multiple leaves share the same page range
+    Each TOC leaf skeleton and any document-root prefix owner gets its own
+    scope. Chapter ranges come from ``resolve_hierarchy_page_ranges``; the
+    root owner preserves physical pages before those chapters without adding
+    a section. When multiple leaves share the same page range
     (e.g. unlocated leaves that fell back to the same parent scope), they are
     merged into a single scope to keep ``scope_id`` unique.
     """
     if not skeletons:
         return []
 
-    root_fallback = all(
-        getattr(item, "title", "") == "Root"
-        or (getattr(item, "evidence", {}) or {}).get("source") == "fallback_root"
-        for item in skeletons
-    )
+    root_fallback = all(is_fallback_root(item) for item in skeletons)
     if root_fallback:
         ordered = sort_skeletons(skeletons)
         return [

apps/worker/app/services/page_memory/fine_hierarchy.py

@@ -24,7 +24,11 @@
 from app.services.page_memory.page_tagger import PageTagResult
 from app.services.page_memory.hierarchy_level_resolver import PageHierarchyLevelResolver
 from app.services.page_memory.skeleton_extractor import SectionSkeleton
-from app.services.page_memory._utils import page_scope_info, sort_skeletons
+from app.services.page_memory._utils import (
+    is_document_prefix,
+    page_scope_info,
+    sort_skeletons,
+)
 from shared.services.ai.llm_overrides import get_text_client
 from shared.services.chunks.path_segments import append_document_path
 
@@ -57,6 +61,9 @@ def refine_fat_leaf_skeletons(
     refined: list[SectionSkeleton] = []
 
     for index, skeleton in enumerate(coarse_skeletons):
+        if is_document_prefix(skeleton):
+            refined.append(skeleton)
+            continue
         exclusive_end = _exclusive_end(coarse_skeletons, index)
         skel_pages = set(range(skeleton.start_page, exclusive_end + 1))
         overlap = skel_pages & fat_leaf_pages
@@ -126,6 +133,8 @@ def compute_fat_leaf_pages(
     excluded = {int(page) for page in (toc_pages or [])}
     fat_pages: set[int] = set()
     for idx, skel in enumerate(skeletons):
+        if is_document_prefix(skel):
+            continue
         exclusive_end = _exclusive_end(skeletons, idx)
         body_pages = [
             page

apps/worker/app/services/page_memory/memory_service.py

@@ -22,6 +22,7 @@
     CoarseScope,
     build_hierarchy_scopes,
     collapse_page_ranges,
+    derive_excluded_toc_pages,
     page_scope_info,
     sort_skeletons,
 )
@@ -274,7 +275,10 @@ def _build_page_dataframe(
         page_count=page_count,
     )
     toc_result = getattr(anatomy, "toc_result", None) if anatomy is not None else None
-    toc_pages = list(getattr(toc_result, "toc_pages", None) or [])
+    toc_pages = derive_excluded_toc_pages(
+        skeletons=skeletons,
+        toc_pages=list(getattr(toc_result, "toc_pages", None) or []),
+    )
     coarse_pages_scope = pages_excluding_toc(
         _derive_hierarchy_page_scope(
             skeletons=skeletons,
@@ -571,6 +575,9 @@ def _run_hierarchy_scope(
     )
 
     scope_skeletons = sort_skeletons(scope.skeletons)
+    toc_pages = derive_excluded_toc_pages(
+        skeletons=scope_skeletons, toc_pages=toc_pages,
+    )
     scope_manifest = _scope_manifest(
         scope_id=scope.scope_id,
         skeletons=scope_skeletons,

apps/worker/app/services/page_memory/skeleton_extractor.py

@@ -28,6 +28,7 @@
     select_global_toc_hierarchies,
 )
 from loguru import logger
+from app.services.page_memory._utils import is_document_prefix
 from shared.services.chunks.path_segments import (
     append_document_path,
     join_document_path,
@@ -205,6 +206,21 @@ def extract_section_skeletons(
         )
         skeletons.extend(secondary_skeletons)
 
+    first_chapter_page: int = min(skeleton.start_page for skeleton in skeletons)
+    if first_chapter_page > 1:
+        skeletons.insert(
+            0,
+            SectionSkeleton(
+                section_path=join_document_path([filename]),
+                level=0,
+                start_page=1,
+                end_page=first_chapter_page - 1,
+                title="Root",
+                parent_path=None,
+                evidence={"source": "document_prefix"},
+            ),
+        )
+
     _log_unlocated_title_warnings(filename=filename, skeletons=skeletons)
     return skeletons
 
@@ -452,7 +468,7 @@ def _collapse_node(path: str) -> None:
         # Re-read children after recursive collapse may have mutated by_path
         child_paths = children_of.get(path, [])
 
-        if len(child_paths) == 1:
+        if len(child_paths) == 1 and not is_document_prefix(node):
             only_child_path = child_paths[0]
             only_child = by_path[only_child_path]
             grandchild_paths = children_of.get(only_child_path, [])

apps/worker/scripts/page_memory/_debug_pm_shared.py

@@ -37,6 +37,7 @@
 from loguru import logger
 
 from app.services.page_memory._utils import (
+    derive_excluded_toc_pages,
     page_scope_info,
     scope_id_for_pages,
     sort_skeletons,
@@ -1342,7 +1343,7 @@ def load_hierarchy_artifact(path: Path) -> tuple[dict[str, Any], list[Any]]:
             SectionSkeleton(
                 section_path=section_path,
                 title=title,
-                level=int(node.get("level") or 1),
+                level=int(node.get("level", 1)),
                 start_page=int(node.get("start_page") or 1),
                 end_page=int(node.get("end_page") or node.get("start_page") or 1),
                 parent_path=node.get("parent_path"),
@@ -1531,7 +1532,10 @@ def build_debug_coarse_scopes(
     from app.services.page_memory._utils import build_hierarchy_scopes
 
     toc_result = getattr(anatomy, "toc_result", None) if anatomy is not None else None
-    toc_pages = list(getattr(toc_result, "toc_pages", None) or [])
+    toc_pages = derive_excluded_toc_pages(
+        skeletons=skeletons,
+        toc_pages=list(getattr(toc_result, "toc_pages", None) or []),
+    )
     toc_page_set = set()
     for raw in toc_pages:
         try:

apps/worker/scripts/page_memory/debug_pm_stage3_scope_fine_hierarchy.py

@@ -67,7 +67,9 @@ def _resolve_scope_processing_pages(
     toc_pages: list[int],
 ) -> tuple[list[int], list[int]]:
     from app.services.document_agent.structure.toc_anchoring import pages_excluding_toc
+    from app.services.page_memory._utils import derive_excluded_toc_pages
 
+    toc_pages = derive_excluded_toc_pages(skeletons=skeletons, toc_pages=toc_pages)
     processing_pages = [
         int(page) for page in (scope_meta.get("processing_pages") or [])
     ]
@@ -132,6 +134,8 @@ def _run_fine_hierarchy_for_scope(
     require_file(skel_path, hint=f"Stage 3 should have created {skel_path}")
     scope_meta, active_skeletons = load_scope_skeletons_artifact(skel_path)
     active_skeletons = sort_skeletons(active_skeletons)
+    from app.services.page_memory._utils import derive_excluded_toc_pages
+    toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages)
     strategy = str(scope_meta.get("strategy") or "coarse_scope")
     processing_pages, excluded_toc_pages = _resolve_scope_processing_pages(
         scope_meta=scope_meta,
@@ -358,6 +362,8 @@ def main() -> int:
     toc_pages = list(getattr(toc_result, "toc_pages", None) or [])
     page_memory_config = PageMemoryConfig.default()
     skeletons = load_pipeline_skeletons(state_path)
+    from app.services.page_memory._utils import derive_excluded_toc_pages
+    toc_pages = derive_excluded_toc_pages(skeletons=skeletons, toc_pages=toc_pages)
 
     logger.info("█" * 70)
     logger.info(f"  STAGE 3: SCOPE + FINE HIERARCHY — {filename}")

apps/worker/scripts/page_memory/debug_pm_stage4_assets.py

@@ -74,6 +74,8 @@ def _load_scope_asset_context(
         hint=f"Run Stage 3 first to produce {fine_hierarchy_path}",
     )
     prior_scope, active_skeletons = load_hierarchy_artifact(fine_hierarchy_path)
+    from app.services.page_memory._utils import derive_excluded_toc_pages
+    toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages)
     if not active_skeletons:
         logger.warning(
             "   [scope {}] no skeletons in fine_hierarchy.json — skipping",
@@ -184,12 +186,7 @@ def main() -> int:
         if context is not None:
             scope_contexts.append(context)
 
-    from app.services.document_agent.structure.toc_anchoring import pages_excluding_toc
-
-    union_pages = pages_excluding_toc(
-        sorted({page for context in scope_contexts for page in context.pages}),
-        toc_pages,
-    )
+    union_pages = sorted({page for context in scope_contexts for page in context.pages})
     logger.info(
         "🔬 document C5: {} unique pages across {} scopes",
         len(union_pages),

apps/worker/scripts/page_memory/debug_pm_stage5_tagging_finalize.py

@@ -83,6 +83,8 @@ def _run_tagging_for_scope(
     fine_hierarchy_path = scope_dir / "fine_hierarchy.json"
     require_file(fine_hierarchy_path, hint=f"Run Stage 3 to produce {fine_hierarchy_path}")
     prior_scope, active_skeletons = load_hierarchy_artifact(fine_hierarchy_path)
+    from app.services.page_memory._utils import derive_excluded_toc_pages
+    toc_pages = derive_excluded_toc_pages(skeletons=active_skeletons, toc_pages=toc_pages)
     if not active_skeletons:
         logger.warning("   [scope {}] no skeletons — skipping", scope_id)
         return ScopeResult(