diff --git a/.github/workflows/episode-tests.yml b/.github/workflows/episode-tests.yml new file mode 100644 index 0000000000..7546de2456 --- /dev/null +++ b/.github/workflows/episode-tests.yml @@ -0,0 +1,46 @@ +name: Episode tests + +# The OpenChia Episode suite on a standard GitHub runner. The inherited Hermes +# unit job (tests.yml) needs a 96-core org runner that this repository does not +# have, so it never runs. This job runs the suite listed in tests/episode_suite.txt; +# known failures (tests/episode_suite_known_failures.txt) are deselected so the +# job blocks new failures while the list is worked down. + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: episode-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + episode-tests: + name: Episode tests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm + with: + extras: '[]' + test-environment: 'true' + + - name: Pull the default Run container image + # The container executor inspects python:3.14-slim and does not pull it itself. + run: docker pull python:3.14-slim + + - name: Run the Episode suite + run: python3 scripts/run_episode_suite.py --python "$HERMES_PYTHON" --workers 4 --verbose + + - name: Report known failures that pass again + if: always() + run: python3 scripts/run_episode_suite.py --python "$HERMES_PYTHON" --workers 4 --check-known diff --git a/scripts/run_episode_suite.py b/scripts/run_episode_suite.py new file mode 100644 index 0000000000..f17646e92c --- /dev/null +++ b/scripts/run_episode_suite.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Run the OpenChia Episode test suite: one pytest process per file, in parallel. + +The suite is listed in ``tests/episode_suite.txt`` (directories or files). +Tests listed in ``tests/episode_suite_known_failures.txt`` are deselected so +the gate is green and blocks *new* failures; ``--check-known`` runs only those +and reports which now pass, so the list shrinks as they are fixed. + +Exit status: 0 when every selected test passed or was skipped, 1 otherwise. +Standard library only. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SUITE = ROOT / "tests" / "episode_suite.txt" +KNOWN = ROOT / "tests" / "episode_suite_known_failures.txt" +# pytest exit codes: 0 passed, 5 nothing collected (everything deselected/skipped). +OK_CODES = {0, 5} + + +def entries(path: Path) -> list[str]: + """Non-comment lines; anything after `#` on a line is a note.""" + lines = [] + for raw in path.read_text(encoding="utf-8-sig").splitlines(): + line = raw.split("#", 1)[0].strip() + if line: + lines.append(line) + return lines + + +def suite_files() -> list[str]: + files: set[str] = set() + for entry in entries(SUITE): + target = ROOT / entry + if target.is_dir(): + files.update(str(p.relative_to(ROOT)) for p in target.rglob("test_*.py")) + elif target.is_file(): + files.add(entry) + else: + raise SystemExit(f"episode suite entry does not exist: {entry}") + return sorted(files) + + +def known_failures() -> tuple[set[str], dict[str, list[str]]]: + """Whole files (collection errors) and per-file node IDs.""" + whole, nodes = set(), {} + for entry in entries(KNOWN) if KNOWN.exists() else []: + if "::" in entry: + nodes.setdefault(entry.split("::", 1)[0], []).append(entry) + else: + whole.add(entry) + return whole, nodes + + +def run_file(python: str, path: str, extra: list[str], timeout: int) -> tuple[str, int, list[str], str, float]: + started = time.monotonic() + command = [python, "-m", "pytest", "-q", "-p", "no:cacheprovider", "-rfE", path, *extra] + try: + done = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout) + output, code = done.stdout + done.stderr, done.returncode + except subprocess.TimeoutExpired as exc: + output, code = f"{exc.stdout or ''}{exc.stderr or ''}\nTIMEOUT after {timeout}s", 124 + failed = sorted({re.sub(r" - .*", "", line).split(" ", 1)[1] + for line in output.splitlines() if line.startswith(("FAILED ", "ERROR "))}) + summary = (output.strip().splitlines() or [""])[-1] + return path, code, failed, output, time.monotonic() - started + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--python", default=os.environ.get("HERMES_PYTHON") or sys.executable) + parser.add_argument("--workers", type=int, default=max(2, (os.cpu_count() or 2))) + parser.add_argument("--timeout", type=int, default=600, help="seconds per test file") + parser.add_argument("--check-known", action="store_true", + help="run only the known failures and report which now pass") + parser.add_argument("--verbose", action="store_true", help="print full output of failing files") + args = parser.parse_args() + + whole, nodes = known_failures() + if args.check_known: + jobs = [(path, []) for path in sorted(whole)] + [(path, ids) for path, ids in sorted(nodes.items())] + jobs = [(path, ids) for path, ids in jobs] + else: + jobs = [] + for path in suite_files(): + if path in whole: + continue + jobs.append((path, [arg for node in nodes.get(path, []) for arg in ("--deselect", node)])) + + with concurrent.futures.ThreadPoolExecutor(args.workers) as pool: + futures = [pool.submit(run_file, args.python, path, extra, args.timeout) for path, extra in jobs] + results = [future.result() for future in futures] + + if args.check_known: + fixed = [path for path, code, failed, _, _ in results if code in OK_CODES] + still = sum(len(failed) or (code not in OK_CODES) for _, code, failed, _, _ in results) + for path in fixed: + print(f"now passing, remove from known failures: {path}") + print(f"known-failure entries checked: {len(results)} files; still failing tests: {still}") + return 0 + + bad = [(path, code, failed, output) for path, code, failed, output, _ in results if code not in OK_CODES] + for path, code, failed, output in bad: + print(f"\n✗ {path} (exit {code})") + for node in failed: + print(f" {node}") + if args.verbose or not failed: + print("\n".join(" " + line for line in output.strip().splitlines()[-40:])) + slowest = sorted(results, key=lambda item: item[4], reverse=True)[:5] + print(f"\nEpisode suite: {len(results)} files, {len(bad)} failing; " + f"{len(whole)} files and {sum(map(len, nodes.values()))} tests deselected as known failures.") + print("slowest: " + ", ".join(f"{path} {seconds:.0f}s" for path, _, _, _, seconds in slowest)) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/episode_runtime/testing_harness/refinement_fixture.py b/tests/episode_runtime/testing_harness/refinement_fixture.py index 290f643a30..a337c7fe9f 100644 --- a/tests/episode_runtime/testing_harness/refinement_fixture.py +++ b/tests/episode_runtime/testing_harness/refinement_fixture.py @@ -404,7 +404,6 @@ async def prepared_refiner( ) from iterative_episode_refiner.runtime import RefinementSession from iterative_episode_refiner.measure_preparation import build_measure_policy - from iterative_episode_refiner.measure_groups import group_definition from iterative_episode_refiner.records import RefinementRecord from iterative_episode_refiner.service import IterativeEpisodeRefiner from iterative_episode_refiner.workspace import RefinementWorkspace @@ -493,6 +492,9 @@ def data(kind, value): "refiner_approval": refiner.workflow_approval.as_record(), }, ) + if grouped_measures: + # Measure groups were removed in def61cccee; only the grouped variant needs them. + from iterative_episode_refiner.measure_groups import group_definition local = data("local_measure", group_definition("local") if grouped_measures else {"purpose": "local"}) acceptance = data("acceptance_measure", group_definition("acceptance", "composition") if grouped_measures else {"purpose": "independent acceptance"}) workflow_ref = data("target_workflow", target.frozen_workflow.as_record()) diff --git a/tests/episode_suite.txt b/tests/episode_suite.txt new file mode 100644 index 0000000000..e2f16463c7 --- /dev/null +++ b/tests/episode_suite.txt @@ -0,0 +1,42 @@ +# OpenChia Episode test suite, run by scripts/run_episode_suite.py and the +# "Episode tests" workflow. One path per line (directory or file); `#` starts a note. +# Add new OpenChia test files here, or put them under one of the directories below. + +tests/method_loop +tests/numeric_control_library +tests/function_library +tests/episode_builder +tests/episode_runtime +tests/iterative_episode_refiner +tests/openchia_cli/inspector + +# Duet, build/run host, launch, model-call supervision and coding transports +tests/agent/test_build_refinement_job.py +tests/agent/test_build_result_publication.py +tests/agent/test_coding_container_shell.py +tests/agent/test_duet_store_transactions.py +tests/agent/test_egress_allowlist_contract.py +tests/agent/test_episode_advisories.py +tests/agent/test_episode_launch_receipts.py +tests/agent/test_episode_model_failure_reporting.py +tests/agent/test_episode_testing_launch.py +tests/agent/test_inline_episode_architecture_cas.py +tests/agent/test_launch_transport_streams.py +tests/agent/test_model_call_failed_attempt_replacement.py +tests/agent/test_model_call_liveness.py +tests/agent/test_provider_notices.py +tests/agent/test_refinement_chat_coding.py +tests/agent/test_refiner_role_routes.py +tests/agent/test_repeatable_episode_calls.py +tests/agent/test_run_environment_status.py + +# OpenChia CLI commands and views +tests/openchia_cli/test_background_duet_recovery.py +tests/openchia_cli/test_doctor_episode_runtime.py +tests/openchia_cli/test_episode_architecture_structure.py +tests/openchia_cli/test_episode_launch_setup.py +tests/openchia_cli/test_episode_test_continuation.py + +# Target environment preparation and the live-model bridge script +tests/pm/test_prepared_project.py +tests/scripts/test_openchia_live_model_bridge.py diff --git a/tests/episode_suite_known_failures.txt b/tests/episode_suite_known_failures.txt new file mode 100644 index 0000000000..992fb49f35 --- /dev/null +++ b/tests/episode_suite_known_failures.txt @@ -0,0 +1,100 @@ +# Known failures in the Episode suite, deselected by scripts/run_episode_suite.py +# so the gate blocks *new* failures. Each group names the change that broke it. +# Remove entries as they are fixed (`run_episode_suite.py --check-known` lists the +# ones that pass again). A path without `::` skips the whole file (collection error). +# Tracking issue: chian/OpenChia#87 + +# A. EpisodeRepeatableCallSpec requires `authority_attenuation` (3e158ebb8a, 2026-10-02); +# the tests construct call specs without that function selection. +tests/agent/test_repeatable_episode_calls.py::test_call_bindings_do_not_relax_tree_or_template_validation +tests/agent/test_repeatable_episode_calls.py::test_legacy_encoding_and_exact_approval_include_repeatable_call_authority +tests/episode_builder/test_repeatable_call_materialization.py::test_approved_self_call_is_materialized_without_changing_the_concrete_tree +tests/episode_builder/test_repeatable_call_materialization.py::test_unresolved_repeatable_policy_blocks_build_without_emitting_a_partial_authority +tests/episode_runtime/test_repeatable_invocations.py::test_child_construction_requires_current_parent_exact_address_and_all_policies +tests/episode_runtime/testing_harness/test_experiment_planning.py::test_preview_binds_real_build_and_never_turns_resolution_into_acceptance +tests/episode_runtime/testing_harness/test_shared_execution.py::test_broker_resolves_only_approved_repeated_paths + +# B. `SuppliedSession.initial_state_recorded` no longer exists (c0c57e1b59, 2026-10-04). +tests/episode_runtime/testing_harness/test_reconstruction.py +tests/episode_runtime/testing_harness/test_host_exchange_recordings.py::test_all_channel_recording_keeps_host_evidence_out_of_external_playback + +# C. Refinement fixtures build assignments whose measure has no authorized checks in +# its inherited catalog (scoped specialists, def61cccee, 2026-10-08). +tests/iterative_episode_refiner/test_campaign_state.py::test_candidate_cas_and_audited_retry_preserve_exact_baseline +tests/iterative_episode_refiner/test_campaign_state.py::test_specialists_cannot_create_designers_through_an_assignment[implementer] +tests/iterative_episode_refiner/test_campaign_state.py::test_unmeasured_edit_and_revisit_do_not_mint_credit +tests/iterative_episode_refiner/test_candidate_environment.py::test_broken_recipe_is_durable_repairable_and_invalidates_source_only_evidence +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[checker-control] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[checker-evaluation] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[continued_checker-control] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[continued_checker-evaluation] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[plan-control] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[plan-evaluation] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[target-control] +tests/iterative_episode_refiner/test_candidate_environment.py::test_experiment_preparation_diagnostics_reach_next_input_and_requested_report[target-evaluation] +tests/iterative_episode_refiner/test_coding.py::test_cancel_joins_writer_and_resume_preserves_unadmitted_edits +tests/iterative_episode_refiner/test_coding.py::test_coding_diff_uses_ordinary_admission_and_unmeasured_credit +tests/iterative_episode_refiner/test_control_integrity.py +tests/openchia_cli/inspector/test_observation_cache.py +tests/openchia_cli/inspector/test_recorded_history.py::test_navigation_holds_selection_and_coding_agent_reads_bound_profile +tests/openchia_cli/inspector/test_recorded_history.py::test_nested_history_keeps_versions_and_evidence_distinct +tests/openchia_cli/inspector/test_recorded_history.py::test_observer_does_not_write_or_block_publication +tests/openchia_cli/inspector/test_run_history.py::test_run_cursor_never_substitutes_newer_campaign_or_terminal_state +tests/openchia_cli/inspector/test_terminal_and_cli.py::test_keyboard_navigation_history_and_reference_context +tests/openchia_cli/inspector/test_terminal_and_cli.py::test_navigation_coalesces_pending_reads_and_installs_only_latest_detail +tests/openchia_cli/inspector/test_terminal_and_cli.py::test_shell_entry_and_slash_dispatch_share_projection_without_host_creation + +# D. Measure records no longer carry `measure_grounding`; measure groups were removed +# (def61cccee). +tests/episode_runtime/testing_harness/test_registered_measure_preparation.py +tests/iterative_episode_refiner/test_reviewed_check_design.py + +# E. Tests import functions removed in def61cccee +# (`runtime_proposals._inherited_draft`, `measure_controls.validated_control_facts`). +tests/episode_runtime/testing_harness/test_campaign_source_experiments.py +tests/episode_runtime/testing_harness/test_measure_control_experiments.py + +# F. Builder blocks emission: "materialization omits structural roles +# ['component.report_group']" (plan findings `function_roles_missing`). +tests/episode_runtime/testing_harness/test_component_execution.py +tests/episode_runtime/testing_harness/test_scoped_execution.py::test_saved_invocation_runs_only_selected_code_and_preserves_nesting +tests/episode_runtime/testing_harness/test_unit_execution.py::test_unit_selection_observes_only_one_unit_without_claiming_episode_completion + +# G. Record and receipt schema drift in hand-built refinement records +# (unit_receipt fields, `check_refs`, `route`, `test_history`, plan requirement mapping). +tests/iterative_episode_refiner/test_model_history.py::test_completed_history_retains_old_units_and_excludes_other_invocations +tests/iterative_episode_refiner/test_model_history.py::test_proposal_history_retains_outputs_and_rejections_across_physical_runs +tests/iterative_episode_refiner/test_reports.py::test_history_shares_parent_conflicts_but_filters_child_details +tests/iterative_episode_refiner/test_reports.py::test_missing_or_conflicting_current_evidence_does_not_rewrite_history +tests/iterative_episode_refiner/test_reports.py::test_parent_and_child_share_exact_results_without_widening_scope +tests/iterative_episode_refiner/test_plan_feedback.py +tests/episode_runtime/records/test_refinement_regression.py +tests/episode_runtime/testing_harness/test_campaign_measurements.py +tests/episode_runtime/testing_harness/test_refinement_execution.py +tests/episode_runtime/testing_harness/test_refinement_experiments.py + +# H. Behaviour changes not yet classified (expected results no longer produced). +tests/agent/test_build_refinement_job.py::test_one_build_enters_refinement_and_cancel_stops_same_job[materialized-build-linked-refiner] +tests/agent/test_episode_testing_launch.py::test_ordinary_run_preserves_approval_and_launches_nested_experiments +tests/episode_builder/test_api_failure_continuation.py::test_api_failure_stops_builder[emission] +tests/episode_runtime/testing_harness/test_later_unit.py::test_later_unit_reuses_verified_learning_without_reexecution_or_new_credit +tests/episode_runtime/testing_harness/test_recorded_execution.py::test_shared_recorded_dispatch_uses_original_evidence_without_live_fallback +tests/episode_runtime/testing_harness/test_shared_execution.py::test_shared_dispatch_preserves_exact_intent_and_does_not_relaunch + +# I. Linux-only native execution on the GitHub ubuntu runner (first CI run, 2026-10-10): +# user systemd transient services exit early (status 218), the repository home-I/O +# guard rejects the setup-pm `uv` path, and confined continuation depends on both. +# These pass or skip on macOS. Fix the runner provisioning or mark them native-only. +tests/agent/test_build_refinement_job.py::test_one_build_enters_refinement_and_cancel_stops_same_job[failed-build-native-refiner] +tests/episode_runtime/test_reasoning_workflow.py::test_live_confined_reasoning_worker +tests/episode_runtime/test_systemd_executor.py::test_native_service_inspection_and_host_stop_include_descendants[None] +tests/episode_runtime/test_systemd_executor.py::test_native_service_inspection_and_host_stop_include_descendants[resources1] +tests/episode_runtime/test_target_environment_execution.py::test_preparation_uses_native_backend_with_only_explicit_environment +tests/episode_runtime/test_target_environment_preparation.py::test_native_recipe_preparation_fresh_rebuild_and_mutated_diagnostic_cache +tests/episode_runtime/testing_harness/test_api_failure_continuation.py::test_api_stop_then_explicit_resume_changes_future_model_only +tests/episode_runtime/testing_harness/test_confined_continuation.py::test_confined_continuation_preserves_credit_and_rejects_divergent_worker_requests[divergent] +tests/episode_runtime/testing_harness/test_confined_continuation.py::test_confined_continuation_preserves_credit_and_rejects_divergent_worker_requests[matching] +tests/iterative_episode_refiner/test_nested_continuation.py::test_completed_implementer_returns_once_after_exact_nested_reconstruction +tests/openchia_cli/test_background_duet_recovery.py::test_background_agent_reopens_its_own_route_after_foreground_changes +tests/pm/test_prepared_project.py::test_preparation_builds_and_reuses_exact_lock_without_ambient_state +tests/pm/test_prepared_project.py::test_wheel_only_preparation_rejects_builds_and_import_hooks diff --git a/tests/iterative_episode_refiner/test_emitter_transcription.py b/tests/iterative_episode_refiner/test_emitter_transcription.py index b0b7a32791..a3cf55abfc 100644 --- a/tests/iterative_episode_refiner/test_emitter_transcription.py +++ b/tests/iterative_episode_refiner/test_emitter_transcription.py @@ -65,10 +65,13 @@ def test_emitter_failure_is_a_turn_error_and_writes_nothing(tmp_path) -> None: def fail(): raise RuntimeError("emission rejected") - session = TranscribedTurnSession(workspace=tmp_path, produce=fail) + # The repository conftest puts a HERMES_HOME under tmp_path; use a workspace of our own. + workspace = tmp_path / "workspace" + workspace.mkdir() + session = TranscribedTurnSession(workspace=workspace, produce=fail) turn = session.run_turn("work") assert turn.error and "emission rejected" in turn.error - assert not any(tmp_path.iterdir()) + assert not any(workspace.iterdir()) session.close() diff --git a/tests/iterative_episode_refiner/test_reports.py b/tests/iterative_episode_refiner/test_reports.py index 798afaf5a7..64f80fc34c 100644 --- a/tests/iterative_episode_refiner/test_reports.py +++ b/tests/iterative_episode_refiner/test_reports.py @@ -117,6 +117,7 @@ def assignment(role, invocation_id, scope, parent=None): "control_bundle_ref": source.as_record(), "supersedes_assignment_refs": [], "writable_paths": [], + "materialization_targets": [], "protected_paths": [], "judgment_lineage": content_id("lineage", role).value, }, diff --git a/tests/method_loop/test_episode_tree_recursion.py b/tests/method_loop/test_episode_tree_recursion.py index 1056a113d4..88db314cd7 100644 --- a/tests/method_loop/test_episode_tree_recursion.py +++ b/tests/method_loop/test_episode_tree_recursion.py @@ -16,9 +16,23 @@ EpisodeTree, Grain, Leaf, + ReportContract, ) +# Since d75f85aa5b every child invocation needs a parent-owned report contract +# and a synthesis callback; the parent receives only the requested fields. +_REPORT = ReportContract( + decision="whether the parent needs another child unit", + measurements={}, + information={"value": "the child's result value"}, +) + + +def _synthesize(result, completion, request): + return {"value": result.value} + + @dataclass(frozen=True) class _Message(ClosedRecord): value: int @@ -97,7 +111,7 @@ def _nested_run(*, run_id: str): label="leaf", ) ), - request=EpisodeRequest(goal=child_goal, message=_Message(1)), + request=EpisodeRequest(goal=child_goal, message=_Message(1), report_contract=_REPORT), build_result=lambda record: _Message(record.units_consumed), ) root = Episode( @@ -105,6 +119,7 @@ def _nested_run(*, run_id: str): key="root", source=_SequenceSource( ChildEpisodeUnit( + synthesize_report=_synthesize, child=child, receive_result=lambda result, completion, request: _Message( result.value @@ -162,6 +177,7 @@ def test_child_goal_must_refine_the_containing_goal(): objective={"task": "child"}, ), message=_Message(1), + report_contract=_REPORT, ), build_result=lambda record: _Message(record.units_consumed), ) @@ -170,6 +186,7 @@ def test_child_goal_must_refine_the_containing_goal(): key="root", source=_SequenceSource( ChildEpisodeUnit( + synthesize_report=_synthesize, child=wrong_child, receive_result=lambda result, completion, request: result, )