refactor!: remove 0.10/0.11 deprecations; cut unit test time 64s to 37s

AGENTS.md

@@ -95,7 +95,7 @@ Bump `RESULT_SCHEMA_VERSION` (`sieval/core/tasks/run_meta.py`) in the PR that ma
     - Tasks: use package-level imports from `sieval.tasks` for official built-in tasks.
     - Custom/community tasks should prefer full module paths (for example `sieval.tasks.<module>.MyTask`) to avoid naming conflicts.
     - `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`).
-    - `session`: canonical path is `sieval.cli.session` (`EvalSession`, `arun_session`). `sieval.cli.leaderboard.session` is a deprecated alias, removed after 0.11.
+    - `session`: canonical path is `sieval.cli.session` (`EvalSession`, `arun_session`).
     - `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`).
     - `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). `cli/infer/` must not import `cli/session.py`.
     - Within the same package/subpackage: use relative imports (`from .foo import Bar`).

CHANGELOG.md

@@ -31,6 +31,13 @@ Planned as 0.12.0.
 - **`sieval run --dry-run` reports what the run would do** (closes #214). `-o json` adds `gpu`, `models` (each served model's plan as `infer_plans.yaml` would record it, each process its launch starts, and `needs_inference`, whether this run starts it), `tasks` (each task's `start`, `resume_plan` with the counts `runs[].resume` will record, `null` for a fresh start, `new`, `to_infer` and `to_grade`) and `needs_inference`, whether any engine starts. The plans come from the call `sieval run` launches by. A resume that would be refused, or planning that raises, fails the new `resume_plan` check; when the dry-run stops before planning, `tasks` and `needs_inference` are `null`. It launches no engine, writes no result directory, reads `--resume` directories in place, and runs on a node without GPUs given `SIEVAL_TARGET_GPU`. Planning loads the datasets and starts each vendored service a task declares, as the run does, so a dry-run that passes its checks costs a session setup. The existing checks are unchanged.
 - `sieval run --dry-run` reports each task's `stages` (per-stage `same`/`differs`/`unknown`, and `rerun_from`, the stage a resume reruns from) and its own `needs_inference`; `RunPlan` gains `stages` (closes #219).
 
+### Removed
+
+- **`sieval.cli.leaderboard.session`**, deprecated in 0.11.0. Import from `sieval.cli.session` (resolvers from `sieval.cli.resolution`).
+- **The resume aliases deprecated in 0.10.0.** `--resume-from DIR` is now an unknown option (use `--resume DIR`). A config setting `runner_config.auto_resume` or `runner_config.resume_from`, top-level or per task, is refused before anything runs, naming the `resume` form to use, rather than ignored, so a resume never silently becomes a fresh run. `TaskRunnerConfig(auto_resume=, resume_from=)`, `EvalSession(resume_from=)`, `arun_session(resume_from=)` / `run_session(resume_from=)` and `sieval.core.runners.fold_resume_aliases` are gone; pass `resume=True` or `resume=<dir>`.
+- The transport import aliases: `sieval.core.models.Transport` (use `sieval.core.models.dialect.Dialect`), and `OpenAIChatTransport` / `OpenAICompletionsTransport` from `sieval.core.models.transports` (use `OpenAIChatDialect` / `OpenAICompletionsDialect` from `sieval.core.models.dialects`). `SglangTransport` stays.
+- `sieval.infer.InferConfig`, unused since `DeploymentPlan` replaced it.
+
 ## [0.11.0] - 2026-10-23
 
 A minor bump: `sieval leaderboard` is removed and SiEval is scoped to an

docs/guide/configuration.md

@@ -461,7 +461,7 @@ declares; neither writes under the result directory. A sample that `feedback`
 sends to another iteration after a rerun from `feedback` is not counted, since
 the plan cannot know it.
 
-Deprecated, removed next release: `--resume-from DIR`, `runner_config.auto_resume`
-and `runner_config.resume_from` (and the same-named Python fields and
-arguments) map to `resume` and warn once; one that disagrees with `resume` is
-refused.
+Removed in 0.12: `--resume-from DIR` (use `--resume DIR`), and
+`runner_config.auto_resume` / `runner_config.resume_from` (use
+`runner_config.resume: true` / `resume: <dir>`). A config that still sets
+either key is refused, naming the `resume` form to use.

scripts/check_layer_imports.py

@@ -63,12 +63,10 @@
 }
 
 # dotted package -> modules it must NOT import: edges *inside* a layer, which
-# FORBIDDEN (keyed on one segment) cannot express. The `sieval` root keeps every
-# module off the deprecated `sieval.cli.leaderboard` alias so it can be deleted.
+# FORBIDDEN (keyed on one segment) cannot express.
 FORBIDDEN_SUBPACKAGE: dict[str, set[str]] = {
     "sieval.cli.infer": {"sieval.cli.session"},
     "sieval.core.models.dialects": {"sieval.core.models.reconcile"},
-    "sieval": {"sieval.cli.leaderboard"},
 }
 
 # Guard: an entry outside the package never matches, so its rule would be dead

sieval/cli/_resume_option.py

@@ -15,7 +15,7 @@
 import typer
 from typer.core import TyperCommand
 
-from sieval.core.runners import ResumeValue, fold_resume_aliases
+from sieval.core.runners import ResumeValue
 
 #: Click's value for a bare ``--resume``: not empty (Typer makes that ``None``)
 #: and never a path (no path holds NUL).
@@ -38,32 +38,10 @@
     ),
 ]
 
-ResumeFromOption = Annotated[
-    Path | None,
-    typer.Option(
-        "--resume-from",
-        metavar="DIR",
-        help="Deprecated: use --resume DIR",
-        hidden=True,
-    ),
-]
-
 
-def cli_resume(resume: str | None, resume_from: Path | None) -> ResumeValue:
-    """``True`` for a bare ``--resume``, a path for ``DIR``, else ``None``.
-
-    Folds in the deprecated ``--resume-from``.
-    """
-    value: ResumeValue = (
-        None if resume is None else True if resume == _IN_PLACE else Path(resume)
-    )
-    return fold_resume_aliases(
-        value,
-        resume_from=resume_from,
-        new="--resume",
-        auto="--resume",
-        source="--resume-from",
-    )
+def cli_resume(resume: str | None) -> ResumeValue:
+    """``True`` for a bare ``--resume``, a path for ``DIR``, else ``None``."""
+    return None if resume is None else True if resume == _IN_PLACE else Path(resume)
 
 
 class ResumeCommand(TyperCommand):

sieval/cli/eval.py

@@ -14,7 +14,6 @@
 
 from sieval.cli._cli_model import ApiBaseOption, CliModel
 from sieval.cli._resume_option import (
-    ResumeFromOption,
     ResumeOption,
     cli_resume,
 )
@@ -38,7 +37,6 @@ def eval_cmd(
         str | None,
         typer.Option("--result-dir", help="Override result directory"),
     ] = None,
-    resume_from: ResumeFromOption = None,
     deterministic: Annotated[
         bool | None,
         typer.Option(
@@ -66,7 +64,7 @@ def eval_cmd(
     from sieval.core.utils.logging import configure_logging
 
     configure_logging(verbose)
-    resumed = cli_resume(resume, resume_from)
+    resumed = cli_resume(resume)
     # No --checkpoint: this command never launches an engine.
     cli_model = CliModel.from_options(checkpoint=None, api_base=api_base, model=model)
 

sieval/cli/run.py

@@ -25,7 +25,6 @@
 )
 from sieval.cli._resume_option import (
     ResumeCommand,
-    ResumeFromOption,
     ResumeOption,
     cli_resume,
 )
@@ -823,7 +822,6 @@ def run(
                 help="Override result directory",
             ),
         ] = None,
-        resume_from: ResumeFromOption = None,
         deterministic: Annotated[
             bool | None,
             typer.Option(
@@ -881,7 +879,7 @@ def run(
     ) -> None:
         """All-in-one: launch inference services, run evaluation, and cleanup."""
         configure_logging(verbose)
-        resumed = cli_resume(resume, resume_from)
+        resumed = cli_resume(resume)
         cli_model = CliModel.from_options(
             checkpoint=checkpoint, api_base=api_base, model=model
         )

sieval/cli/session.py

@@ -128,7 +128,6 @@
     RunPlan,
     TaskRunnerConfig,
     as_resume,
-    fold_resume_aliases,
     resume_source,
     same_dir,
 )
@@ -1259,8 +1258,9 @@ def _configured_client_timeout(
     )
 
 
-#: ``resume`` and its deprecated aliases.
-_RESUME_KEYS = ("resume", "auto_resume", "resume_from")
+#: Resume keys removed in 0.12, refused rather than dropped: a dropped
+#: ``resume_from`` would turn a resume into a fresh run without a word.
+_REMOVED_RESUME_KEYS = {"auto_resume": "resume: true", "resume_from": "resume: <dir>"}
 
 
 def _runner_config_blocks(cfg: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
@@ -1275,41 +1275,29 @@ def _runner_config_blocks(cfg: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
 
 
 def config_resumes(cfg: Mapping[str, Any]) -> bool:
-    """Whether some ``runner_config`` of *cfg* sets ``resume`` or an alias of it."""
-    return any(
-        block.get(key)
-        for block in _runner_config_blocks(cfg).values()
-        for key in _RESUME_KEYS
-    )
+    """Whether some ``runner_config`` of *cfg* sets ``resume``."""
+    return any(block.get("resume") for block in _runner_config_blocks(cfg).values())
 
 
 def _drop_resume_keys(cfg: dict[str, Any]) -> None:
-    """Remove every resume key in place, and a ``runner_config`` left empty."""
+    """Remove ``resume`` in place, and a ``runner_config`` left empty."""
     for block in _runner_config_blocks(cfg).values():
-        for key in _RESUME_KEYS:
-            block.pop(key, None)
+        block.pop("resume", None)
     if cfg.get("runner_config") == {}:
         del cfg["runner_config"]
     for task in (cfg.get("tasks") or {}).values():
         if isinstance(task, dict) and task.get("runner_config") == {}:
             del task["runner_config"]
 
 
-def _fold_runner_resume_aliases(cfg: dict[str, Any]) -> None:
-    """Rewrite ``auto_resume`` / ``resume_from`` as ``resume``, in place."""
+def _refuse_removed_resume_keys(cfg: Mapping[str, Any]) -> None:
+    """Raise on ``auto_resume`` / ``resume_from``, naming the ``resume`` form."""
     for where, block in _runner_config_blocks(cfg).items():
-        if not {"auto_resume", "resume_from"} & block.keys():
-            continue
-        resume = fold_resume_aliases(
-            as_resume(block.get("resume"), f"{where}.resume"),
-            auto_resume=block.pop("auto_resume", None),
-            resume_from=block.pop("resume_from", None),
-            new=f"{where}.resume",
-            auto=f"{where}.auto_resume",
-            source=f"{where}.resume_from",
-        )
-        if resume is not None:
-            block["resume"] = str(resume) if isinstance(resume, Path) else resume
+        for key, use in _REMOVED_RESUME_KEYS.items():
+            if key in block:
+                raise ValueError(
+                    f"{where}.{key} was removed in 0.12; use {where}.{use}"
+                )
 
 
 def read_persisted_config(path: Path) -> dict[str, Any] | None:
@@ -1678,7 +1666,6 @@ def __init__(
         realized_deployments: Mapping[str, Deployment] | None = None,
         model_capability_profiles: Mapping[str, ModelCapabilityProfile] | None = None,
         serving_reconciler: ServingReconciler | None = None,
-        resume_from: str | Path | None = None,
         cli_model: CliModel | None = None,
         infer_gpu: str | None = None,
         unlaunched_deployments: Mapping[str, Deployment] | None = None,
@@ -1687,13 +1674,7 @@ def __init__(
         self.model_override = model_override
         # `True`, or a recorded session dir each task starts from; see
         # `_cli_resume_for`.
-        self.resume_override: ResumeValue = fold_resume_aliases(
-            as_resume(resume),
-            resume_from=resume_from,
-            new="resume",
-            auto="resume",
-            source="resume_from",
-        )
+        self.resume_override: ResumeValue = as_resume(resume)
         self.result_dir_override = result_dir_override
         self.deterministic_override = deterministic_override
         legacy_endpoint_map = dict(endpoint_map or {})
@@ -1794,8 +1775,7 @@ def __init__(
             model=model_override,
             result_dir=result_dir_override,
         )
-        # Before persisting, so effective_config.yaml records only `resume`.
-        _fold_runner_resume_aliases(reified)
+        _refuse_removed_resume_keys(reified)
         # Before the copy, so the digests reach both the persisted view (and
         # therefore --resume) and the runtime view below. Not where the
         # operation runs: `arun` persists before `_prepare_execution`, so by
@@ -5393,7 +5373,6 @@ async def arun_session(
     invocation: str | None = None,
     self_managed_endpoints: frozenset[str] | set[str] = frozenset(),
     realized_deployments: Mapping[str, Deployment] | None = None,
-    resume_from: str | Path | None = None,
     cli_model: CliModel | None = None,
     infer_gpu: str | None = None,
     unlaunched_deployments: Mapping[str, Deployment] | None = None,
@@ -5423,7 +5402,6 @@ async def arun_session(
             warning to genuinely external endpoints.
         realized_deployments: Typed realized deployments keyed by root model
             config name.
-        resume_from: Deprecated alias of ``resume=<dir>``; warns.
         cli_model: ``--checkpoint``/``--api-base``, added as the ``models:``
             entry of a config that has none.
         infer_gpu: The GPUs *infer_plans* were resolved for, as
@@ -5448,7 +5426,6 @@ async def arun_session(
         invocation=invocation,
         self_managed_endpoints=self_managed_endpoints,
         realized_deployments=realized_deployments,
-        resume_from=resume_from,
         cli_model=cli_model,
         infer_gpu=infer_gpu,
         unlaunched_deployments=unlaunched_deployments,
@@ -5503,7 +5480,6 @@ def run_session(
     invocation: str | None = None,
     self_managed_endpoints: frozenset[str] | set[str] = frozenset(),
     realized_deployments: Mapping[str, Deployment] | None = None,
-    resume_from: str | Path | None = None,
 ) -> dict[str, Any]:
     return anyio.run(
         arun_session,
@@ -5517,5 +5493,4 @@ def run_session(
         invocation,
         self_managed_endpoints,
         realized_deployments,
-        resume_from,
     )

sieval/core/models/__init__.py

@@ -110,7 +110,6 @@
     reconcile,
 )
 from .sglang_gen_model import SglangGenModel
-from .transport import Transport
 
 __all__ = [
     "BINDING_RESOURCE_KEYS",
@@ -199,7 +198,6 @@
     "ToolParams",
     "ToolResultPart",
     "TopKEntry",
-    "Transport",
     "UsageStats",
     "check_extra",
     "sequence_wrapper",