ci: run the Episode test suite on a standard runner and gate new failures

.github/workflows/episode-tests.yml

@@ -0,0 +1,46 @@
+name: Episode tests
+
+# The OpenChia Episode suite on a standard GitHub runner. The inherited Hermes
+# unit job (tests.yml) needs a 96-core org runner that this repository does not
+# have, so it never runs. This job runs the suite listed in tests/episode_suite.txt;
+# known failures (tests/episode_suite_known_failures.txt) are deselected so the
+# job blocks new failures while the list is worked down.
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: episode-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  episode-tests:
+    name: Episode tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up locked Python and test dependencies
+        uses: ./.github/actions/setup-pm
+        with:
+          extras: '[]'
+          test-environment: 'true'
+
+      - name: Pull the default Run container image
+        # The container executor inspects python:3.14-slim and does not pull it itself.
+        run: docker pull python:3.14-slim
+
+      - name: Run the Episode suite
+        run: python3 scripts/run_episode_suite.py --python "$HERMES_PYTHON" --workers 4 --verbose
+
+      - name: Report known failures that pass again
+        if: always()
+        run: python3 scripts/run_episode_suite.py --python "$HERMES_PYTHON" --workers 4 --check-known

scripts/run_episode_suite.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Run the OpenChia Episode test suite: one pytest process per file, in parallel.
+
+The suite is listed in ``tests/episode_suite.txt`` (directories or files).
+Tests listed in ``tests/episode_suite_known_failures.txt`` are deselected so
+the gate is green and blocks *new* failures; ``--check-known`` runs only those
+and reports which now pass, so the list shrinks as they are fixed.
+
+Exit status: 0 when every selected test passed or was skipped, 1 otherwise.
+Standard library only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import os
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SUITE = ROOT / "tests" / "episode_suite.txt"
+KNOWN = ROOT / "tests" / "episode_suite_known_failures.txt"
+# pytest exit codes: 0 passed, 5 nothing collected (everything deselected/skipped).
+OK_CODES = {0, 5}
+
+
+def entries(path: Path) -> list[str]:
+    """Non-comment lines; anything after `#` on a line is a note."""
+    lines = []
+    for raw in path.read_text(encoding="utf-8-sig").splitlines():
+        line = raw.split("#", 1)[0].strip()
+        if line:
+            lines.append(line)
+    return lines
+
+
+def suite_files() -> list[str]:
+    files: set[str] = set()
+    for entry in entries(SUITE):
+        target = ROOT / entry
+        if target.is_dir():
+            files.update(str(p.relative_to(ROOT)) for p in target.rglob("test_*.py"))
+        elif target.is_file():
+            files.add(entry)
+        else:
+            raise SystemExit(f"episode suite entry does not exist: {entry}")
+    return sorted(files)
+
+
+def known_failures() -> tuple[set[str], dict[str, list[str]]]:
+    """Whole files (collection errors) and per-file node IDs."""
+    whole, nodes = set(), {}
+    for entry in entries(KNOWN) if KNOWN.exists() else []:
+        if "::" in entry:
+            nodes.setdefault(entry.split("::", 1)[0], []).append(entry)
+        else:
+            whole.add(entry)
+    return whole, nodes
+
+
+def run_file(python: str, path: str, extra: list[str], timeout: int) -> tuple[str, int, list[str], str, float]:
+    started = time.monotonic()
+    command = [python, "-m", "pytest", "-q", "-p", "no:cacheprovider", "-rfE", path, *extra]
+    try:
+        done = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout)
+        output, code = done.stdout + done.stderr, done.returncode
+    except subprocess.TimeoutExpired as exc:
+        output, code = f"{exc.stdout or ''}{exc.stderr or ''}\nTIMEOUT after {timeout}s", 124
+    failed = sorted({re.sub(r" - .*", "", line).split(" ", 1)[1]
+                     for line in output.splitlines() if line.startswith(("FAILED ", "ERROR "))})
+    summary = (output.strip().splitlines() or [""])[-1]
+    return path, code, failed, output, time.monotonic() - started
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--python", default=os.environ.get("HERMES_PYTHON") or sys.executable)
+    parser.add_argument("--workers", type=int, default=max(2, (os.cpu_count() or 2)))
+    parser.add_argument("--timeout", type=int, default=600, help="seconds per test file")
+    parser.add_argument("--check-known", action="store_true",
+                        help="run only the known failures and report which now pass")
+    parser.add_argument("--verbose", action="store_true", help="print full output of failing files")
+    args = parser.parse_args()
+
+    whole, nodes = known_failures()
+    if args.check_known:
+        jobs = [(path, []) for path in sorted(whole)] + [(path, ids) for path, ids in sorted(nodes.items())]
+        jobs = [(path, ids) for path, ids in jobs]
+    else:
+        jobs = []
+        for path in suite_files():
+            if path in whole:
+                continue
+            jobs.append((path, [arg for node in nodes.get(path, []) for arg in ("--deselect", node)]))
+
+    with concurrent.futures.ThreadPoolExecutor(args.workers) as pool:
+        futures = [pool.submit(run_file, args.python, path, extra, args.timeout) for path, extra in jobs]
+        results = [future.result() for future in futures]
+
+    if args.check_known:
+        fixed = [path for path, code, failed, _, _ in results if code in OK_CODES]
+        still = sum(len(failed) or (code not in OK_CODES) for _, code, failed, _, _ in results)
+        for path in fixed:
+            print(f"now passing, remove from known failures: {path}")
+        print(f"known-failure entries checked: {len(results)} files; still failing tests: {still}")
+        return 0
+
+    bad = [(path, code, failed, output) for path, code, failed, output, _ in results if code not in OK_CODES]
+    for path, code, failed, output in bad:
+        print(f"\n✗ {path} (exit {code})")
+        for node in failed:
+            print(f"    {node}")
+        if args.verbose or not failed:
+            print("\n".join("      " + line for line in output.strip().splitlines()[-40:]))
+    slowest = sorted(results, key=lambda item: item[4], reverse=True)[:5]
+    print(f"\nEpisode suite: {len(results)} files, {len(bad)} failing; "
+          f"{len(whole)} files and {sum(map(len, nodes.values()))} tests deselected as known failures.")
+    print("slowest: " + ", ".join(f"{path} {seconds:.0f}s" for path, _, _, _, seconds in slowest))
+    return 1 if bad else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/episode_runtime/testing_harness/refinement_fixture.py

@@ -404,7 +404,6 @@ async def prepared_refiner(
     )
     from iterative_episode_refiner.runtime import RefinementSession
     from iterative_episode_refiner.measure_preparation import build_measure_policy
-    from iterative_episode_refiner.measure_groups import group_definition
     from iterative_episode_refiner.records import RefinementRecord
     from iterative_episode_refiner.service import IterativeEpisodeRefiner
     from iterative_episode_refiner.workspace import RefinementWorkspace
@@ -493,6 +492,9 @@ def data(kind, value):
                 "refiner_approval": refiner.workflow_approval.as_record(),
             },
         )
+        if grouped_measures:
+            # Measure groups were removed in def61cccee; only the grouped variant needs them.
+            from iterative_episode_refiner.measure_groups import group_definition
         local = data("local_measure", group_definition("local") if grouped_measures else {"purpose": "local"})
         acceptance = data("acceptance_measure", group_definition("acceptance", "composition") if grouped_measures else {"purpose": "independent acceptance"})
         workflow_ref = data("target_workflow", target.frozen_workflow.as_record())

tests/episode_suite.txt

@@ -0,0 +1,42 @@
+# OpenChia Episode test suite, run by scripts/run_episode_suite.py and the
+# "Episode tests" workflow. One path per line (directory or file); `#` starts a note.
+# Add new OpenChia test files here, or put them under one of the directories below.
+
+tests/method_loop
+tests/numeric_control_library
+tests/function_library
+tests/episode_builder
+tests/episode_runtime
+tests/iterative_episode_refiner
+tests/openchia_cli/inspector
+
+# Duet, build/run host, launch, model-call supervision and coding transports
+tests/agent/test_build_refinement_job.py
+tests/agent/test_build_result_publication.py
+tests/agent/test_coding_container_shell.py
+tests/agent/test_duet_store_transactions.py
+tests/agent/test_egress_allowlist_contract.py
+tests/agent/test_episode_advisories.py
+tests/agent/test_episode_launch_receipts.py
+tests/agent/test_episode_model_failure_reporting.py
+tests/agent/test_episode_testing_launch.py
+tests/agent/test_inline_episode_architecture_cas.py
+tests/agent/test_launch_transport_streams.py
+tests/agent/test_model_call_failed_attempt_replacement.py
+tests/agent/test_model_call_liveness.py
+tests/agent/test_provider_notices.py
+tests/agent/test_refinement_chat_coding.py
+tests/agent/test_refiner_role_routes.py
+tests/agent/test_repeatable_episode_calls.py
+tests/agent/test_run_environment_status.py
+
+# OpenChia CLI commands and views
+tests/openchia_cli/test_background_duet_recovery.py
+tests/openchia_cli/test_doctor_episode_runtime.py
+tests/openchia_cli/test_episode_architecture_structure.py
+tests/openchia_cli/test_episode_launch_setup.py
+tests/openchia_cli/test_episode_test_continuation.py
+
+# Target environment preparation and the live-model bridge script
+tests/pm/test_prepared_project.py
+tests/scripts/test_openchia_live_model_bridge.py

tests/episode_suite_known_failures.txt

@@ -0,0 +1,100 @@
+# Known failures in the Episode suite, deselected by scripts/run_episode_suite.py
+# so the gate blocks *new* failures. Each group names the change that broke it.
+# Remove entries as they are fixed (`run_episode_suite.py --check-known` lists the
+# ones that pass again). A path without `::` skips the whole file (collection error).
+# Tracking issue: chian/OpenChia#87
+
+# A. EpisodeRepeatableCallSpec requires `authority_attenuation` (3e158ebb8a, 2026-10-02);
+#    the tests construct call specs without that function selection.
+tests/agent/test_repeatable_episode_calls.py::test_call_bindings_do_not_relax_tree_or_template_validation
+tests/agent/test_repeatable_episode_calls.py::test_legacy_encoding_and_exact_approval_include_repeatable_call_authority
+tests/episode_builder/test_repeatable_call_materialization.py::test_approved_self_call_is_materialized_without_changing_the_concrete_tree
+tests/episode_builder/test_repeatable_call_materialization.py::test_unresolved_repeatable_policy_blocks_build_without_emitting_a_partial_authority
+tests/episode_runtime/test_repeatable_invocations.py::test_child_construction_requires_current_parent_exact_address_and_all_policies
+tests/episode_runtime/testing_harness/test_experiment_planning.py::test_preview_binds_real_build_and_never_turns_resolution_into_acceptance
+tests/episode_runtime/testing_harness/test_shared_execution.py::test_broker_resolves_only_approved_repeated_paths
+
+# B. `SuppliedSession.initial_state_recorded` no longer exists (c0c57e1b59, 2026-10-04).
+tests/episode_runtime/testing_harness/test_reconstruction.py
+tests/episode_runtime/testing_harness/test_host_exchange_recordings.py::test_all_channel_recording_keeps_host_evidence_out_of_external_playback
+
+# C. Refinement fixtures build assignments whose measure has no authorized checks in
+#    its inherited catalog (scoped specialists, def61cccee, 2026-10-08).
+tests/iterative_episode_refiner/test_campaign_state.py::test_candidate_cas_and_audited_retry_preserve_exact_baseline
+tests/iterative_episode_refiner/test_campaign_state.py::test_specialists_cannot_create_designers_through_an_assignment[implementer]
+tests/iterative_episode_refiner/test_campaign_state.py::test_unmeasured_edit_and_revisit_do_not_mint_credit
+tests/iterative_episode_refiner/test_candidate_environment.py::test_broken_recipe_is_durable_repairable_and_invalidates_source_only_evidence
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[checker-control]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[checker-evaluation]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[continued_checker-control]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[continued_checker-evaluation]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[plan-control]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[plan-evaluation]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[target-control]
+tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[target-evaluation]
+tests/iterative_episode_refiner/test_coding.py::test_cancel_joins_writer_and_resume_preserves_unadmitted_edits
+tests/iterative_episode_refiner/test_coding.py::test_coding_diff_uses_ordinary_admission_and_unmeasured_credit
+tests/iterative_episode_refiner/test_control_integrity.py
+tests/openchia_cli/inspector/test_observation_cache.py
+tests/openchia_cli/inspector/test_recorded_history.py::test_navigation_holds_selection_and_coding_agent_reads_bound_profile
+tests/openchia_cli/inspector/test_recorded_history.py::test_nested_history_keeps_versions_and_evidence_distinct
+tests/openchia_cli/inspector/test_recorded_history.py::test_observer_does_not_write_or_block_publication
+tests/openchia_cli/inspector/test_run_history.py::test_run_cursor_never_substitutes_newer_campaign_or_terminal_state
+tests/openchia_cli/inspector/test_terminal_and_cli.py::test_keyboard_navigation_history_and_reference_context
+tests/openchia_cli/inspector/test_terminal_and_cli.py::test_navigation_coalesces_pending_reads_and_installs_only_latest_detail
+tests/openchia_cli/inspector/test_terminal_and_cli.py::test_shell_entry_and_slash_dispatch_share_projection_without_host_creation
+
+# D. Measure records no longer carry `measure_grounding`; measure groups were removed
+#    (def61cccee).
+tests/episode_runtime/testing_harness/test_registered_measure_preparation.py
+tests/iterative_episode_refiner/test_reviewed_check_design.py
+
+# E. Tests import functions removed in def61cccee
+#    (`runtime_proposals._inherited_draft`, `measure_controls.validated_control_facts`).
+tests/episode_runtime/testing_harness/test_campaign_source_experiments.py
+tests/episode_runtime/testing_harness/test_measure_control_experiments.py
+
+# F. Builder blocks emission: "materialization omits structural roles
+#    ['component.report_group']" (plan findings `function_roles_missing`).
+tests/episode_runtime/testing_harness/test_component_execution.py
+tests/episode_runtime/testing_harness/test_scoped_execution.py::test_saved_invocation_runs_only_selected_code_and_preserves_nesting
+tests/episode_runtime/testing_harness/test_unit_execution.py::test_unit_selection_observes_only_one_unit_without_claiming_episode_completion
+
+# G. Record and receipt schema drift in hand-built refinement records
+#    (unit_receipt fields, `check_refs`, `route`, `test_history`, plan requirement mapping).
+tests/iterative_episode_refiner/test_model_history.py::test_completed_history_retains_old_units_and_excludes_other_invocations
+tests/iterative_episode_refiner/test_model_history.py::test_proposal_history_retains_outputs_and_rejections_across_physical_runs
+tests/iterative_episode_refiner/test_reports.py::test_history_shares_parent_conflicts_but_filters_child_details
+tests/iterative_episode_refiner/test_reports.py::test_missing_or_conflicting_current_evidence_does_not_rewrite_history
+tests/iterative_episode_refiner/test_reports.py::test_parent_and_child_share_exact_results_without_widening_scope
+tests/iterative_episode_refiner/test_plan_feedback.py
+tests/episode_runtime/records/test_refinement_regression.py
+tests/episode_runtime/testing_harness/test_campaign_measurements.py
+tests/episode_runtime/testing_harness/test_refinement_execution.py
+tests/episode_runtime/testing_harness/test_refinement_experiments.py
+
+# H. Behaviour changes not yet classified (expected results no longer produced).
+tests/agent/test_build_refinement_job.py::test_one_build_enters_refinement_and_cancel_stops_same_job[materialized-build-linked-refiner]
+tests/agent/test_episode_testing_launch.py::test_ordinary_run_preserves_approval_and_launches_nested_experiments
+tests/episode_builder/test_api_failure_continuation.py::test_api_failure_stops_builder[emission]
+tests/episode_runtime/testing_harness/test_later_unit.py::test_later_unit_reuses_verified_learning_without_reexecution_or_new_credit
+tests/episode_runtime/testing_harness/test_recorded_execution.py::test_shared_recorded_dispatch_uses_original_evidence_without_live_fallback
+tests/episode_runtime/testing_harness/test_shared_execution.py::test_shared_dispatch_preserves_exact_intent_and_does_not_relaunch
+
+# I. Linux-only native execution on the GitHub ubuntu runner (first CI run, 2026-10-10):
+#    user systemd transient services exit early (status 218), the repository home-I/O
+#    guard rejects the setup-pm `uv` path, and confined continuation depends on both.
+#    These pass or skip on macOS. Fix the runner provisioning or mark them native-only.
+tests/agent/test_build_refinement_job.py::test_one_build_enters_refinement_and_cancel_stops_same_job[failed-build-native-refiner]
+tests/episode_runtime/test_reasoning_workflow.py::test_live_confined_reasoning_worker
+tests/episode_runtime/test_systemd_executor.py::test_native_service_inspection_and_host_stop_include_descendants[None]
+tests/episode_runtime/test_systemd_executor.py::test_native_service_inspection_and_host_stop_include_descendants[resources1]
+tests/episode_runtime/test_target_environment_execution.py::test_preparation_uses_native_backend_with_only_explicit_environment
+tests/episode_runtime/test_target_environment_preparation.py::test_native_recipe_preparation_fresh_rebuild_and_mutated_diagnostic_cache
+tests/episode_runtime/testing_harness/test_api_failure_continuation.py::test_api_stop_then_explicit_resume_changes_future_model_only
+tests/episode_runtime/testing_harness/test_confined_continuation.py::test_confined_continuation_preserves_credit_and_rejects_divergent_worker_requests[divergent]
+tests/episode_runtime/testing_harness/test_confined_continuation.py::test_confined_continuation_preserves_credit_and_rejects_divergent_worker_requests[matching]
+tests/iterative_episode_refiner/test_nested_continuation.py::test_completed_implementer_returns_once_after_exact_nested_reconstruction
+tests/openchia_cli/test_background_duet_recovery.py::test_background_agent_reopens_its_own_route_after_foreground_changes
+tests/pm/test_prepared_project.py::test_preparation_builds_and_reuses_exact_lock_without_ambient_state
+tests/pm/test_prepared_project.py::test_wheel_only_preparation_rejects_builds_and_import_hooks