From fe63374ed3d10dbc799d37e407fa809e89a33570 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 23 Oct 2026 16:14:10 +0800 Subject: [PATCH] feat(cli): sieval run --dry-run reports what the run would do The dry-run resolved the managed batch but returned only checks and counts, so a platform could not learn the plans, the launch commands or whether the run needs inference before allocating a GPU. `sieval run --dry-run [--resume [DIR]] -o json` now adds, beside the unchanged checks: - models: per served model, the plan as infer_plans.yaml would record it, each process its launch starts (argv, env, cwd), and needs_inference, whether this run starts it; gpu, what the plans were resolved for; - tasks: per task, start (fresh, resume or reuse), resume_plan, the counts runs[].resume will record (null for a fresh start), new, to_infer and to_grade; - needs_inference: whether the run starts any engine. A task inferring through an api_base model counts in its to_infer but starts none. The plans come from the same call sieval run makes to decide what to launch, so the two cannot disagree. A resume that would be refused, or planning that raises, fails the new resume_plan check. The four keys are always present; when the dry-run stops before planning, tasks and needs_inference are null, meaning unknown rather than false. No engine is launched and no result directory is written; --resume directories are read in place, and with SIEVAL_TARGET_GPU it runs on a node without GPUs. Planning sets the session up as the run does: it loads the datasets and starts each vendored service a task declares, which the --dry-run help, the guide and the CHANGELOG now say. The text output gains a short summary. Closes #214. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + docs/guide/configuration.md | 61 ++++++++ sieval/cli/output.py | 29 ++++ sieval/cli/run.py | 169 +++++++++++++++++++--- tests/integration/test_lazy_launch.py | 197 ++++++++++++++++++++++++++ tests/unit/cli/test_output.py | 78 ++++++++++ tests/unit/cli/test_run.py | 119 +++++++++++++++- tests/unit/cli/test_validation.py | 5 + 8 files changed, 641 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f81a12..5c2618aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Planned as 0.12.0. - **`SIEVAL_TARGET_GPU` declares the GPUs a serving plan is resolved for**, as `::` with the name and memory `nvidia-smi` reports, e.g. `NVIDIA H100 80GB HBM3:8:81559`. Recipe resolution reads it instead of `nvidia-smi`, on any node, so a CPU node resolves the plan, and the inference identity, a node with those GPUs would. Unset, nothing changes. Declared, an engine launches only on a node with exactly those GPUs. `infer_plans.yaml` gains `gpu`, the GPUs its plans were resolved for, outside every identity; a `sieval run --resume` declaring the same GPUs reuses the recorded plan of each served model whose `path`, `infer`, capability layer and `deterministic` are unchanged, unless the plan held an engine secret, which `infer_plans.yaml` records redacted (#215, closes #212). - `sieval.core.runners.task_resume_plan` / `plan_resume`: the per-stage resume plan, without running anything. - `TaskRunner.plan_run` / `MultiTaskRunner.plan_run` (`RunPlan`): the runner's per-sample plan of a run, `to_infer` included, read-only; the one `sieval run` launches by. +- **`sieval run --dry-run` reports what the run would do** (closes #214). `-o json` adds `gpu`, `models` (each served model's plan as `infer_plans.yaml` would record it, each process its launch starts, and `needs_inference`, whether this run starts it), `tasks` (each task's `start`, `resume_plan` with the counts `runs[].resume` will record, `null` for a fresh start, `new`, `to_infer` and `to_grade`) and `needs_inference`, whether any engine starts. The plans come from the call `sieval run` launches by. A resume that would be refused, or planning that raises, fails the new `resume_plan` check; when the dry-run stops before planning, `tasks` and `needs_inference` are `null`. It launches no engine, writes no result directory, reads `--resume` directories in place, and runs on a node without GPUs given `SIEVAL_TARGET_GPU`. Planning loads the datasets and starts each vendored service a task declares, as the run does, so a dry-run that passes its checks costs a session setup. The existing checks are unchanged. ## [0.11.0] - 2026-10-23 diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 87f73f01..0fb76fa7 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -321,6 +321,67 @@ A directory recorded before stage identities is compared through the session's `effective_config.yaml`. Without one, `--resume DIR` is refused and an in-place `--resume` trusts it. +### What a run would do + +`sieval run --dry-run [--resume [DIR]] -o json` reports what the run would do, +without launching an engine or writing a result directory. It reads the +`--resume` directories in place. On a node without GPUs, declare the target ones +in `SIEVAL_TARGET_GPU` ([Resolving for declared GPUs](infer.md#resolving-for-declared-gpus)). + +```json +{ + "checks": [{"name": "resume_plan", "ok": true, "detail": "1 task(s): 1 sample(s) to infer, 1 to grade"}], + "n_errors": 0, + "n_warnings": 0, + "gpu": "NVIDIA H100 80GB HBM3:8:81559", + "models": { + "subject": { + "plan": {"checkpoint": "/models/Qwen3-4B", "backend": "sglang", "assignments": []}, + "launch": [{"role": "full", "argv": ["sglang", "serve", "--model-path", "/models/Qwen3-4B"]}], + "needs_inference": true + } + }, + "tasks": { + "echo": { + "model": "subject", + "start": "resume", + "recorded": "outputs/run/echo", + "samples": 3, + "resume_plan": {"kept": 2, "rereported": 0, "regraded": 0, "repostprocessed": 0, "restarted": 0, "dropped": 0}, + "new": 1, + "to_infer": 1, + "to_grade": 1 + } + }, + "needs_inference": true +} +``` + +- `checks` are the config checks, then `resume_plan`, which fails when a task's + resume would be refused; that task's entry then holds `error` instead. +- `models` has one entry per model `sieval run` serves: the plan as + `infer_plans.yaml` would record it, each process the launch starts (`argv`, + and `env` and `cwd` when set), and `needs_inference`, whether this run starts + its engine. `gpu` is what the plans were resolved for. +- `tasks..start` is `fresh`, `resume` (stage by stage) or `reuse` (the + recorded `report.json` stands, recomputed when only `report` changed). + `resume_plan` counts the recorded samples as `runs[].resume` will, and is + `null` for a fresh start; `new` counts samples with no record. `to_infer` + counts the samples sent to the model under test, `to_grade` those run through + a grading stage. +- `needs_inference` is whether the run starts any engine. A task that infers + through an `api_base` model counts in its `to_infer` but starts none. + +`gpu`, `models`, `tasks` and `needs_inference` are always present. When the +config checks fail, or planning raises (the `resume_plan` check then names +why), `tasks` and every `needs_inference` are `null`: unknown, not false. + +Planning sets the session up as the run does. It loads every dataset, +downloading one not yet cached, and starts each vendored service a task +declares; neither writes under the result directory. A sample that `feedback` +sends to another iteration after a rerun from `feedback` is not counted, since +the plan cannot know it. + Deprecated, removed next release: `--resume-from DIR`, `runner_config.auto_resume` and `runner_config.resume_from` (and the same-named Python fields and arguments) map to `resume` and warn once; one that disagrees with `resume` is diff --git a/sieval/cli/output.py b/sieval/cli/output.py index 6b686b0b..dab41a19 100644 --- a/sieval/cli/output.py +++ b/sieval/cli/output.py @@ -348,6 +348,33 @@ def _render_text_task_reports(result: CommandResult) -> None: log_user("{}", task_data.get("report", "")) +def _render_dry_run_plan(data: dict) -> None: + """What ``sieval run --dry-run`` found the run would do, when it got that far.""" + tasks = data.get("tasks") + if not isinstance(tasks, dict) or not tasks: + return + log_user("") + for name, entry in (data.get("models") or {}).items(): + starts = "starts" if entry.get("needs_inference") else "not started" + log_user("model {}: engine {}", name, starts) + for name, entry in tasks.items(): + if "error" in entry: + log_user("task {}: refused", name) + continue + outcomes = ", ".join( + f"{n} {outcome}" for outcome, n in (entry["resume_plan"] or {}).items() if n + ) + log_user( + "task {}: {}{}, {} to infer, {} to grade", + name, + entry["start"], + f" ({outcomes})" if outcomes else "", + entry["to_infer"], + entry["to_grade"], + ) + log_user("needs inference: {}", "yes" if data.get("needs_inference") else "no") + + def _render_text_dry_run(result: CommandResult) -> None: if not result.ok: if not isinstance(result.data, dict): @@ -363,6 +390,7 @@ def _render_text_dry_run(result: CommandResult) -> None: log_user("✗ {}", check.get("detail", check["name"])) for w in check.get("warnings", []): log_user("⚠ {}", w) + _render_dry_run_plan(result.data) n_err = result.data.get("n_errors", 0) n_warn = result.data.get("n_warnings", 0) log_user("\nDry-run failed: {} error(s), {} warning(s).", n_err, n_warn) @@ -382,6 +410,7 @@ def _render_text_dry_run(result: CommandResult) -> None: log_user("✗ {}", check.get("detail", check["name"])) for w in check.get("warnings", []): log_user("⚠ {}", w) + _render_dry_run_plan(result.data) n_warn = result.data.get("n_warnings", 0) if n_warn: log_user("\nDry-run passed with {} warning(s).", n_warn) diff --git a/sieval/cli/run.py b/sieval/cli/run.py index 95b9260b..560cebcb 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -8,7 +8,7 @@ import shlex import sys import time -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from pathlib import Path from typing import Annotated, Any, cast @@ -341,6 +341,25 @@ async def _plan_tasks( ) +def _models_to_launch( + served: Iterable[str], + task_models: Mapping[str, str], + plans: Mapping[str, RunPlan | Exception], +) -> set[str]: + """The *served* models some task still has samples to infer with. + + A task whose plan raises infers nothing: its run raises the same before + its first sample. + """ + return { + task_models[task] + for task, plan in plans.items() + if not isinstance(plan, Exception) + and plan["to_infer"] + and task_models.get(task) in served + } + + async def _models_to_infer( config_path: Path, config: Mapping[str, Any], @@ -348,12 +367,8 @@ async def _models_to_infer( plan_dicts: dict[str, dict[str, Any]], **session: Any, ) -> set[str]: - """The served models some task still has samples to infer with. - - A run where nothing resumes infers everything, so it is not planned. A - task whose plan raises infers nothing: its run raises the same before its - first sample. - """ + """The served models to launch: all of them for a run where nothing + resumes, which is not planned; else :func:`_models_to_launch`.""" if not prepared_launches: return set() if not _resumes(session["resume"], config): @@ -361,12 +376,44 @@ async def _models_to_infer( task_models, plans = await _plan_tasks( config_path, prepared_launches, plan_dicts, **session ) + return _models_to_launch(prepared_launches, task_models, plans) + + +def _launch_json(command: BackendCommand) -> dict[str, object]: + """One process a launch starts: its role, argv and the env it adds.""" + out: dict[str, object] = {"role": command.role, "argv": list(command.cli_args)} + if command.env: + out["env"] = dict(command.env) + if command.working_dir: + out["cwd"] = command.working_dir + return out + + +def _plan_summary(plans: Mapping[str, RunPlan | Exception]) -> str: + """One line for the ``resume_plan`` check: what the tasks would do.""" + ok = [p for p in plans.values() if not isinstance(p, Exception)] + to_infer = sum(p["to_infer"] for p in ok) + to_grade = sum(p["to_grade"] for p in ok) + return f"{len(ok)} task(s): {to_infer} sample(s) to infer, {to_grade} to grade" + + +def _task_json(model: str | None, plan: RunPlan | Exception) -> dict[str, object]: + """A task's entry in the dry-run report. + + ``resume_plan`` is the plan's ``outcomes``, the counts ``runs[].resume`` + records; ``None`` for a fresh start, which has no recorded samples. + """ + if isinstance(plan, Exception): + return {"model": model, "error": cli_error_message(plan)} return { - task_models[task] - for task, plan in plans.items() - if not isinstance(plan, Exception) - and plan["to_infer"] - and task_models.get(task) in prepared_launches + "model": model, + "start": plan["start"], + "recorded": plan["recorded"], + "samples": plan["samples"], + "resume_plan": plan["outcomes"], + "new": plan["new"], + "to_infer": plan["to_infer"], + "to_grade": plan["to_grade"], } @@ -379,7 +426,13 @@ async def _run_dry_run( deterministic: bool | None = None, cli_model: CliModel | None = None, ) -> dict[str, object]: - """Resolve the same managed batch and return the pure prelaunch plan.""" + """Report what the run would do, launching no engine. + + The checks, then each served model's plan and launch argv, and each task's + plan; see ``docs/guide/configuration.md``. Reads ``--resume`` directories in + place and writes no result directory. Planning loads the datasets and + starts each vendored service a task declares, as the run does. + """ from sieval.cli.validation import run_dry_run @@ -404,8 +457,14 @@ def checks( ) ) + def unplanned(result: dict[str, object]) -> dict[str, object]: + # Every report key is present; ``None`` where the dry-run stopped first. + for key in ("gpu", "models", "tasks", "needs_inference"): + result.setdefault(key, None) + return result + if not config_path.exists(): - return checks() + return unplanned(checks()) with open(config_path) as stream: loaded = yaml.safe_load(stream) @@ -414,7 +473,7 @@ def checks( try: apply_cli_model(loaded, cli_model, model) except ValueError: - return checks() # reported as the failed cli_model check + return unplanned(checks()) # reported as the failed cli_model check effective_deterministic = resolve_deterministic(deterministic, loaded) plan_dicts, prepared_launches = await _prepare_launch_batch( config_path, @@ -422,7 +481,76 @@ def checks( effective_deterministic=effective_deterministic, recorded=_recorded_plans(resume, result_dir or loaded.get("result_dir")), ) - return checks(plan_dicts or None, frozenset(prepared_launches)) + result = checks(plan_dicts or None, frozenset(prepared_launches)) + # ``needs_inference`` stays ``None`` until the tasks are planned. + models: dict[str, dict[str, object]] = { + name: { + "plan": plan_dicts[name], + "launch": [_launch_json(command) for command in commands], + "needs_inference": None, + } + for name, (_, commands) in prepared_launches.items() + } + result["gpu"] = await _plan_gpu_text() if prepared_launches else None + result["models"] = models + if result["n_errors"]: + return unplanned(result) + + # What the run would do: each task's plan, from the call the run makes. + # Planning sets the session up as the run does: it loads every dataset + # (downloading one not yet cached) and starts each vendored service a task + # declares, writing nothing under the result directory. + checks_list = cast(list[dict[str, object]], result["checks"]) + try: + task_models, plans = await _plan_tasks( + config_path, + prepared_launches, + plan_dicts, + model=model, + resume=resume, + result_dir=result_dir, + deterministic=deterministic, + invocation=shlex.join(sys.argv), + cli_model=cli_model, + ) + except Exception as exc: + # Raised before any task would start: the run fails the same way. + detail = cli_error_message(exc) + checks_list.append({"name": "resume_plan", "ok": False, "detail": detail}) + result["n_errors"] = cast(int, result["n_errors"]) + 1 + return unplanned(result) + refused = {t: p for t, p in plans.items() if isinstance(p, Exception)} + result["tasks"] = { + task: _task_json(task_models.get(task), plan) for task, plan in plans.items() + } + if refused: + checks_list.append( + { + "name": "resume_plan", + "ok": False, + "detail": "; ".join( + f"{task}: {cli_error_message(exc)}" + for task, exc in sorted(refused.items()) + ), + } + ) + result["n_errors"] = cast(int, result["n_errors"]) + 1 + else: + checks_list.append( + {"name": "resume_plan", "ok": True, "detail": _plan_summary(plans)} + ) + # The launch decision `_models_to_infer` makes for the run. + launches = ( + _models_to_launch(prepared_launches, task_models, plans) + if _resumes(resume, loaded) + else set(prepared_launches) + ) + for name, entry in models.items(): + entry["needs_inference"] = name in launches + # Whether an engine starts. A task inferring through an `api_base` model + # needs no engine, so it does not count. + result["needs_inference"] = bool(launches) + return result async def _run_all( @@ -730,7 +858,14 @@ def run( ] = False, dry_run: Annotated[ bool, - typer.Option("--dry-run", help="Validate config without running"), + typer.Option( + "--dry-run", + help=( + "Validate the config, then report each model's plan and each " + "task's resume plan, launching no engine. Loads the datasets " + "and starts any vendored service, as the run does." + ), + ), ] = False, output: Annotated[ OutputFormat, diff --git a/tests/integration/test_lazy_launch.py b/tests/integration/test_lazy_launch.py index a7bb9c1d..b5b0cc5c 100644 --- a/tests/integration/test_lazy_launch.py +++ b/tests/integration/test_lazy_launch.py @@ -334,3 +334,200 @@ async def test_only_the_model_with_samples_to_infer_starts( assert engine.requests == ["Q3"] assert reports["echo"]["total"] == 2 assert reports["echo_other"]["total"] == 3 + + +def _sync_run(config: Path, **kwargs: Any) -> dict: + """``_run`` from a test that is not async, as the CLI is not.""" + import anyio + + return anyio.run(lambda: _run(config, **kwargs)) + + +def _dry_run(config: Path, *args: str) -> tuple[int, dict]: + from typer.testing import CliRunner + + from sieval.cli.main import app + + result = CliRunner().invoke( + app, ["run", str(config), "--dry-run", *args, "-o", "json"] + ) + return result.exit_code, json.loads(result.stdout) + + +def _resume_plan(**counts: int) -> dict[str, int]: + """A task's ``resume_plan``: every outcome, zero unless given.""" + keys = ("kept", "rereported", "regraded", "repostprocessed", "restarted") + return {**dict.fromkeys((*keys, "dropped"), 0), **counts} + + +def _tree(root: Path) -> dict[str, bytes]: + return { + str(p.relative_to(root)): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +class TestDryRunReportsWhatARunWouldDo: + """``sieval run --dry-run -o json`` on a CPU node, with declared GPUs.""" + + H100_TEXT = "NVIDIA H100 80GB HBM3:8:81559" + + def test_a_resume_reports_its_plan_and_reads_dirs_read_only( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + with _gpu(H100): + _sync_run(_write(tmp_path, QUESTIONS[:2])) + recorded = _tree(tmp_path / "results") + engine.launches.clear() + config = _write(tmp_path, QUESTIONS) + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + cpu = AsyncMock(return_value=None) + + with ( + _gpu(None), + patch("sieval.infer.topology.resolver.detect_local_gpu", new=cpu), + ): + code, out = _dry_run(config, "--resume") + + cpu.assert_not_awaited() # nvidia-smi is never asked + assert code == 0, out + data = out["data"] + assert {c["name"] for c in data["checks"]} >= {"schema", "capability_reconcile"} + assert data["needs_inference"] is True + assert data["gpu"] == self.H100_TEXT + model = data["models"]["subject"] + assert model["needs_inference"] is True + assert model["plan"]["backend"] == "sglang" + checkpoint = str(tmp_path / "Qwen3-4B") + assert model["launch"] == [ + { + "role": "full", + "argv": [ + "sglang", + "serve", + "--model-path", + checkpoint, + *model["launch"][0]["argv"][4:], + ], + } + ] + assert "--port" in model["launch"][0]["argv"] + assert data["tasks"]["echo"] == { + "model": "subject", + "start": "resume", + "recorded": str(tmp_path / "results" / "echo"), + "samples": 3, + "resume_plan": _resume_plan(kept=2), + "new": 1, + "to_infer": 1, + "to_grade": 1, + } + assert engine.launches == [] + assert _tree(tmp_path / "results") == recorded + + # The plan the dry-run reports is the one the run then records. + with _gpu(H100): + _sync_run(config, resume=True) + plans = yaml.safe_load((tmp_path / "results" / "infer_plans.yaml").read_text()) + assert plans["models"]["subject"] == model["plan"] + + def test_nothing_to_infer( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + with _gpu(H100): + _sync_run(_write(tmp_path, QUESTIONS[:2])) + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + # A task argument: both samples rerun from postprocess. + config = _write(tmp_path, QUESTIONS[:2], strict=True) + + with _gpu(None): + code, out = _dry_run(config, "--resume") + + assert code == 0, out + data = out["data"] + assert data["needs_inference"] is False + assert data["models"]["subject"]["needs_inference"] is False + echo = data["tasks"]["echo"] + assert echo["resume_plan"] == _resume_plan(repostprocessed=2) + assert (echo["to_infer"], echo["to_grade"]) == (0, 2) + + def test_resume_from_another_dir_copies_nothing( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + with _gpu(H100): + _sync_run(_write(tmp_path, QUESTIONS)) + recorded = tmp_path / "results" + before = _tree(recorded) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + config = _write(elsewhere, QUESTIONS) + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + + with _gpu(None): + code, out = _dry_run(config, "--resume", str(recorded)) + + assert code == 0, out + echo = out["data"]["tasks"]["echo"] + assert (echo["start"], echo["recorded"]) == ("resume", str(recorded / "echo")) + assert out["data"]["needs_inference"] is False + assert _tree(recorded) == before + assert not (elsewhere / "results").exists() + + def test_a_refused_resume_fails_the_dry_run( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + with _gpu(H100): + _sync_run(_write(tmp_path, QUESTIONS[:2])) + cfg = yaml.safe_load(_write(tmp_path, QUESTIONS[:2]).read_text()) + cfg["tasks"]["echo"]["infer_args"] = {"temperature": 0.9} # another inference + config = tmp_path / "eval.yaml" + config.write_text(yaml.safe_dump(cfg)) + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + + with _gpu(None): + code, out = _dry_run(config, "--resume") + + assert code == 1 + data = out["data"] + assert "inference" in data["tasks"]["echo"]["error"] + assert [c["name"] for c in data["checks"] if not c["ok"]] == ["resume_plan"] + + def test_a_fresh_run_infers_everything( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + with _gpu(None): + code, out = _dry_run(_write(tmp_path, QUESTIONS)) + assert code == 0, out + data = out["data"] + assert data["needs_inference"] is True + assert data["models"]["subject"]["needs_inference"] is True + # No recorded samples, so no resume plan. + assert data["tasks"]["echo"] == { + "model": "subject", + "start": "fresh", + "recorded": None, + "samples": 3, + "resume_plan": None, + "new": 3, + "to_infer": 3, + "to_grade": 3, + } + assert not (tmp_path / "results").exists() + + def test_a_finished_run_is_reused( + self, tmp_path: Path, engine: Engine, monkeypatch: pytest.MonkeyPatch + ) -> None: + with _gpu(H100): + _sync_run(_write(tmp_path, QUESTIONS)) + monkeypatch.setenv("SIEVAL_TARGET_GPU", self.H100_TEXT) + + with _gpu(None): + code, out = _dry_run(_write(tmp_path, QUESTIONS), "--resume") + + assert code == 0, out + echo = out["data"]["tasks"]["echo"] + assert echo["start"] == "reuse" + assert echo["resume_plan"] == _resume_plan(kept=3) + assert out["data"]["needs_inference"] is False diff --git a/tests/unit/cli/test_output.py b/tests/unit/cli/test_output.py index 7793f87a..6b2a5091 100644 --- a/tests/unit/cli/test_output.py +++ b/tests/unit/cli/test_output.py @@ -396,6 +396,84 @@ def test_dry_run_error_before_any_check_is_shown(self): assert "Dry-run failed" in _collect_log_user(log) +def _rendered_lines(mock_log_user) -> list[str]: + """Each ``log_user`` call as the line it prints.""" + return [c.args[0].format(*c.args[1:]) for c in mock_log_user.call_args_list] + + +def _outcomes(**counts: int) -> dict[str, int]: + keys = ("kept", "rereported", "regraded", "repostprocessed", "restarted") + return {**dict.fromkeys((*keys, "dropped"), 0), **counts} + + +class TestRenderTextDryRunPlan: + """The summary ``sieval run --dry-run`` adds once the tasks are planned.""" + + def _render(self, ok: bool, **data: object) -> list[str]: + result = CommandResult( + command="run.dry_run", + ok=ok, + data={"checks": [], "n_errors": 0 if ok else 1, "n_warnings": 0, **data}, + error=None if ok else "Dry-run failed", + ) + with patch("sieval.cli.output.log_user") as mock: + render(result, OutputFormat.TEXT) + return _rendered_lines(mock) + + def test_each_model_and_task_gets_a_line(self) -> None: + lines = self._render( + True, + models={ + "subject": {"needs_inference": True}, + "judge": {"needs_inference": False}, + }, + tasks={ + "echo": { + "start": "resume", + "resume_plan": _outcomes(kept=2, regraded=1), + "to_infer": 1, + "to_grade": 2, + }, + "fresh": { + "start": "fresh", + "resume_plan": None, + "to_infer": 3, + "to_grade": 3, + }, + }, + needs_inference=True, + ) + assert "model subject: engine starts" in lines + assert "model judge: engine not started" in lines + assert "task echo: resume (2 kept, 1 regraded), 1 to infer, 2 to grade" in lines + assert "task fresh: fresh, 3 to infer, 3 to grade" in lines + assert "needs inference: yes" in lines + + def test_a_refused_task_is_named_on_a_failed_dry_run(self) -> None: + lines = self._render( + False, + models={"subject": {"needs_inference": False}}, + tasks={ + "echo": {"error": "inference changed"}, + "other": { + "start": "reuse", + "resume_plan": _outcomes(kept=3), + "to_infer": 0, + "to_grade": 0, + }, + }, + needs_inference=False, + ) + assert "task echo: refused" in lines + assert "task other: reuse (3 kept), 0 to infer, 0 to grade" in lines + assert "needs inference: no" in lines + assert lines[-1].startswith("\nDry-run failed") + + def test_nothing_is_added_before_the_tasks_are_planned(self) -> None: + lines = self._render(False, tasks=None, needs_inference=None) + assert not any(line.startswith(("task ", "needs inference")) for line in lines) + + class TestRenderTextFallback: """Test the fallback path when no text renderer is registered.""" diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index 300b95be..b52a53d2 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -8,6 +8,7 @@ import re from collections.abc import Callable from pathlib import Path +from typing import cast from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest @@ -279,6 +280,11 @@ async def test_dry_run_reuses_managed_plan_batch_and_cli_overrides( "sieval.cli.validation.run_dry_run", return_value=dry_result, ) as validate, + # The tasks' resume plans, which the CLI tests cover. + patch( + "sieval.cli.run._plan_tasks", + new=AsyncMock(return_value=({}, {})), + ), ): result = await _run_dry_run( config_path, @@ -288,7 +294,16 @@ async def test_dry_run_reuses_managed_plan_batch_and_cli_overrides( deterministic=True, ) - assert result == dry_result + assert result["checks"] == [{"name": "resume_plan", "ok": True, "detail": ANY}] + assert result["n_errors"] == 0 + assert result["models"] == { + "model_a": { + "plan": plan_dicts["model_a"], + "launch": [], + "needs_inference": False, + } + } + assert (result["tasks"], result["needs_inference"]) == ({}, False) prepare.assert_awaited_once() assert prepare.call_args.kwargs["effective_deterministic"] is True assert validate.call_args.kwargs["infer_plans"] == plan_dicts @@ -298,6 +313,108 @@ async def test_dry_run_reuses_managed_plan_batch_and_cli_overrides( assert validate.call_args.kwargs["deterministic_override"] is True assert validate.call_args.kwargs["self_managed_endpoints"] == {"model_a"} + @staticmethod + async def _dry_run( + tmp_path: Path, + *, + n_errors: int = 0, + resume: bool = True, + plan_tasks: AsyncMock | None = None, + ) -> dict[str, object]: + """``_run_dry_run`` with ``model_a`` served and ``api`` not.""" + from sieval.cli.run import _run_dry_run + + config_path = tmp_path / "cfg.yaml" + config_path.write_text(yaml.safe_dump({"models": {}, "tasks": {}})) + plan_dicts = {"model_a": {"backend": "vllm", "assignments": []}} + checks = [{"name": "schema", "ok": not n_errors}] + with ( + patch.dict(os.environ, {"SIEVAL_TARGET_GPU": "NVIDIA H100:8:81559"}), + patch( + "sieval.cli.run._prepare_launch_batch", + new=AsyncMock( + return_value=(plan_dicts, {"model_a": (_fake_plan(), [])}) + ), + ), + patch( + "sieval.cli.validation.run_dry_run", + return_value={"checks": checks, "n_errors": n_errors, "n_warnings": 0}, + ), + patch( + "sieval.cli.run._plan_tasks", + new=plan_tasks or AsyncMock(return_value=({}, {})), + ) as planned, + ): + result = await _run_dry_run(config_path, resume=resume) + result["_planned"] = planned.await_count + return result + + @staticmethod + def _plan(to_infer: int) -> dict[str, object]: + return { + "start": "resume", + "recorded": "r/t", + "samples": 2, + "outcomes": {"kept": 2 - to_infer}, + "new": to_infer, + "to_infer": to_infer, + "to_grade": to_infer, + } + + @pytest.mark.anyio + async def test_needs_inference_is_whether_an_engine_starts( + self, tmp_path: Path + ) -> None: + # `api` infers through an api_base model: no engine starts for it. + plans = AsyncMock( + return_value=( + {"t_api": "api", "t_a": "model_a"}, + {"t_api": self._plan(2), "t_a": self._plan(0)}, + ) + ) + result = await self._dry_run(tmp_path, plan_tasks=plans) + models = cast(dict, result["models"]) + assert models["model_a"]["needs_inference"] is False + assert result["needs_inference"] is False + tasks = cast(dict, result["tasks"]) + assert tasks["t_api"]["to_infer"] == 2 + + @pytest.mark.anyio + async def test_a_fresh_run_starts_every_served_engine(self, tmp_path: Path) -> None: + # `_models_to_infer` does not plan a run where nothing resumes. + result = await self._dry_run(tmp_path, resume=False) + assert cast(dict, result["models"])["model_a"]["needs_inference"] is True + assert result["needs_inference"] is True + + @pytest.mark.anyio + async def test_failed_checks_leave_the_plan_unknown(self, tmp_path: Path) -> None: + result = await self._dry_run(tmp_path, n_errors=1) + assert result["_planned"] == 0 + assert (result["tasks"], result["needs_inference"]) == (None, None) + assert cast(dict, result["models"])["model_a"]["needs_inference"] is None + assert result["gpu"] == "NVIDIA H100:8:81559" + + @pytest.mark.anyio + async def test_a_planning_error_fails_resume_plan(self, tmp_path: Path) -> None: + boom = AsyncMock(side_effect=ValueError("result_dir is not a directory")) + result = await self._dry_run(tmp_path, plan_tasks=boom) + assert result["n_errors"] == 1 + assert cast(list, result["checks"])[-1] == { + "name": "resume_plan", + "ok": False, + "detail": "result_dir is not a directory", + } + assert (result["tasks"], result["needs_inference"]) == (None, None) + + @pytest.mark.anyio + async def test_a_missing_config_still_has_every_key(self, tmp_path: Path) -> None: + from sieval.cli.run import _run_dry_run + + result = await _run_dry_run(tmp_path / "absent.yaml") + assert result["n_errors"] == 1 + for key in ("gpu", "models", "tasks", "needs_inference"): + assert result[key] is None + @pytest.mark.anyio async def test_binding_error_prevents_every_managed_launch(self, tmp_path: Path): from sieval.cli.run import _run_all diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index 4292a2d7..84829cee 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -1485,6 +1485,11 @@ def test_run_dry_run_valid_config(self, tmp_path: Path): "sieval.cli.validation._capability_reconcile_check", return_value={"name": "capability_reconcile", "ok": True}, ), + # Planning builds the run, which these fake classes cannot. + patch( + "sieval.cli.run._plan_tasks", + new=AsyncMock(return_value=({}, {})), + ), ): result = cli_runner.invoke(app, ["run", "--dry-run", str(config)]) assert result.exit_code == 0