diff --git a/AGENTS.md b/AGENTS.md index 39d80f4f..0d196a0f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ Bump `RESULT_SCHEMA_VERSION` (`sieval/core/tasks/run_meta.py`) in the PR that ma - Tasks: use package-level imports from `sieval.tasks` for official built-in tasks. - Custom/community tasks should prefer full module paths (for example `sieval.tasks..MyTask`) to avoid naming conflicts. - `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`). - - `session`: canonical path is `sieval.cli.session` (`EvalSession`, `arun_session`). `sieval.cli.leaderboard.session` is a deprecated alias, removed after 0.11. + - `session`: canonical path is `sieval.cli.session` (`EvalSession`, `arun_session`). - `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`). - `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). `cli/infer/` must not import `cli/session.py`. - Within the same package/subpackage: use relative imports (`from .foo import Bar`). diff --git a/CHANGELOG.md b/CHANGELOG.md index c18e4a82..3fc44e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ Planned as 0.12.0. - **`sieval run --dry-run` reports what the run would do** (closes #214). `-o json` adds `gpu`, `models` (each served model's plan as `infer_plans.yaml` would record it, each process its launch starts, and `needs_inference`, whether this run starts it), `tasks` (each task's `start`, `resume_plan` with the counts `runs[].resume` will record, `null` for a fresh start, `new`, `to_infer` and `to_grade`) and `needs_inference`, whether any engine starts. The plans come from the call `sieval run` launches by. A resume that would be refused, or planning that raises, fails the new `resume_plan` check; when the dry-run stops before planning, `tasks` and `needs_inference` are `null`. It launches no engine, writes no result directory, reads `--resume` directories in place, and runs on a node without GPUs given `SIEVAL_TARGET_GPU`. Planning loads the datasets and starts each vendored service a task declares, as the run does, so a dry-run that passes its checks costs a session setup. The existing checks are unchanged. - `sieval run --dry-run` reports each task's `stages` (per-stage `same`/`differs`/`unknown`, and `rerun_from`, the stage a resume reruns from) and its own `needs_inference`; `RunPlan` gains `stages` (closes #219). +### Removed + +- **`sieval.cli.leaderboard.session`**, deprecated in 0.11.0. Import from `sieval.cli.session` (resolvers from `sieval.cli.resolution`). +- **The resume aliases deprecated in 0.10.0.** `--resume-from DIR` is now an unknown option (use `--resume DIR`). A config setting `runner_config.auto_resume` or `runner_config.resume_from`, top-level or per task, is refused before anything runs, naming the `resume` form to use, rather than ignored, so a resume never silently becomes a fresh run. `TaskRunnerConfig(auto_resume=, resume_from=)`, `EvalSession(resume_from=)`, `arun_session(resume_from=)` / `run_session(resume_from=)` and `sieval.core.runners.fold_resume_aliases` are gone; pass `resume=True` or `resume=`. +- The transport import aliases: `sieval.core.models.Transport` (use `sieval.core.models.dialect.Dialect`), and `OpenAIChatTransport` / `OpenAICompletionsTransport` from `sieval.core.models.transports` (use `OpenAIChatDialect` / `OpenAICompletionsDialect` from `sieval.core.models.dialects`). `SglangTransport` stays. +- `sieval.infer.InferConfig`, unused since `DeploymentPlan` replaced it. + ## [0.11.0] - 2026-10-23 A minor bump: `sieval leaderboard` is removed and SiEval is scoped to an diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e88e459e..c0697e59 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -461,7 +461,7 @@ declares; neither writes under the result directory. A sample that `feedback` sends to another iteration after a rerun from `feedback` is not counted, since the plan cannot know it. -Deprecated, removed next release: `--resume-from DIR`, `runner_config.auto_resume` -and `runner_config.resume_from` (and the same-named Python fields and -arguments) map to `resume` and warn once; one that disagrees with `resume` is -refused. +Removed in 0.12: `--resume-from DIR` (use `--resume DIR`), and +`runner_config.auto_resume` / `runner_config.resume_from` (use +`runner_config.resume: true` / `resume: `). A config that still sets +either key is refused, naming the `resume` form to use. diff --git a/scripts/check_layer_imports.py b/scripts/check_layer_imports.py index e00f9d97..4f825853 100644 --- a/scripts/check_layer_imports.py +++ b/scripts/check_layer_imports.py @@ -63,12 +63,10 @@ } # dotted package -> modules it must NOT import: edges *inside* a layer, which -# FORBIDDEN (keyed on one segment) cannot express. The `sieval` root keeps every -# module off the deprecated `sieval.cli.leaderboard` alias so it can be deleted. +# FORBIDDEN (keyed on one segment) cannot express. FORBIDDEN_SUBPACKAGE: dict[str, set[str]] = { "sieval.cli.infer": {"sieval.cli.session"}, "sieval.core.models.dialects": {"sieval.core.models.reconcile"}, - "sieval": {"sieval.cli.leaderboard"}, } # Guard: an entry outside the package never matches, so its rule would be dead diff --git a/sieval/cli/_resume_option.py b/sieval/cli/_resume_option.py index 47d99fb0..58e947b9 100644 --- a/sieval/cli/_resume_option.py +++ b/sieval/cli/_resume_option.py @@ -15,7 +15,7 @@ import typer from typer.core import TyperCommand -from sieval.core.runners import ResumeValue, fold_resume_aliases +from sieval.core.runners import ResumeValue #: Click's value for a bare ``--resume``: not empty (Typer makes that ``None``) #: and never a path (no path holds NUL). @@ -38,32 +38,10 @@ ), ] -ResumeFromOption = Annotated[ - Path | None, - typer.Option( - "--resume-from", - metavar="DIR", - help="Deprecated: use --resume DIR", - hidden=True, - ), -] - -def cli_resume(resume: str | None, resume_from: Path | None) -> ResumeValue: - """``True`` for a bare ``--resume``, a path for ``DIR``, else ``None``. - - Folds in the deprecated ``--resume-from``. - """ - value: ResumeValue = ( - None if resume is None else True if resume == _IN_PLACE else Path(resume) - ) - return fold_resume_aliases( - value, - resume_from=resume_from, - new="--resume", - auto="--resume", - source="--resume-from", - ) +def cli_resume(resume: str | None) -> ResumeValue: + """``True`` for a bare ``--resume``, a path for ``DIR``, else ``None``.""" + return None if resume is None else True if resume == _IN_PLACE else Path(resume) class ResumeCommand(TyperCommand): diff --git a/sieval/cli/eval.py b/sieval/cli/eval.py index a5455aa2..53da151b 100644 --- a/sieval/cli/eval.py +++ b/sieval/cli/eval.py @@ -14,7 +14,6 @@ from sieval.cli._cli_model import ApiBaseOption, CliModel from sieval.cli._resume_option import ( - ResumeFromOption, ResumeOption, cli_resume, ) @@ -38,7 +37,6 @@ def eval_cmd( str | None, typer.Option("--result-dir", help="Override result directory"), ] = None, - resume_from: ResumeFromOption = None, deterministic: Annotated[ bool | None, typer.Option( @@ -66,7 +64,7 @@ def eval_cmd( from sieval.core.utils.logging import configure_logging configure_logging(verbose) - resumed = cli_resume(resume, resume_from) + resumed = cli_resume(resume) # No --checkpoint: this command never launches an engine. cli_model = CliModel.from_options(checkpoint=None, api_base=api_base, model=model) diff --git a/sieval/cli/leaderboard/__init__.py b/sieval/cli/leaderboard/__init__.py deleted file mode 100644 index a6c0e432..00000000 --- a/sieval/cli/leaderboard/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Deprecated: holds only the ``session`` alias. Delete after 0.11. - -AI-Generated Code - Claude Opus 5.5 (Anthropic) -""" diff --git a/sieval/cli/leaderboard/session.py b/sieval/cli/leaderboard/session.py deleted file mode 100644 index cec22054..00000000 --- a/sieval/cli/leaderboard/session.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Deprecated alias of :mod:`sieval.cli.session` (and the class resolvers of -:mod:`sieval.cli.resolution`), re-exporting the same objects. Delete after 0.11. - -Other engine globals (e.g. ``DETERMINISTIC_DEFAULT_SEED``) are forwarded on -access. Assigning here does not reach the engine; patch :mod:`sieval.cli.session`. - -AI-Generated Code - Claude Opus 5.5 (Anthropic) -""" - -import warnings - -import sieval.cli.resolution as _resolution -import sieval.cli.session as _session -from sieval import __version__ as __version__ -from sieval.cli.resolution import ( - resolve_class, - resolve_dataset_class, - resolve_task_class, -) -from sieval.cli.session import ( - EvalSession, - ModelConfigDict, - RootConfigDict, - arun_session, - resolve_deterministic, - run_session, - unwrap_proxies, -) - -_DEPRECATION = ( - "sieval.cli.leaderboard.session is deprecated and will be removed after " - "0.11; import from sieval.cli.session (resolvers: sieval.cli.resolution)." -) - -warnings.warn(_DEPRECATION, DeprecationWarning, stacklevel=2) - - -def __getattr__(name: str): - if name.startswith("__"): # keep protocol probes (`__path__`, `__wrapped__`) local - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - for home in (_session, _resolution): - try: - value = getattr(home, name) - except AttributeError: - continue - warnings.warn(_DEPRECATION, DeprecationWarning, stacklevel=2) - return value - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - "EvalSession", - "ModelConfigDict", - "RootConfigDict", - "arun_session", - "resolve_class", - "resolve_dataset_class", - "resolve_deterministic", - "resolve_task_class", - "run_session", - "unwrap_proxies", -] diff --git a/sieval/cli/run.py b/sieval/cli/run.py index 38e89189..5765edb6 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -25,7 +25,6 @@ ) from sieval.cli._resume_option import ( ResumeCommand, - ResumeFromOption, ResumeOption, cli_resume, ) @@ -823,7 +822,6 @@ def run( help="Override result directory", ), ] = None, - resume_from: ResumeFromOption = None, deterministic: Annotated[ bool | None, typer.Option( @@ -881,7 +879,7 @@ def run( ) -> None: """All-in-one: launch inference services, run evaluation, and cleanup.""" configure_logging(verbose) - resumed = cli_resume(resume, resume_from) + resumed = cli_resume(resume) cli_model = CliModel.from_options( checkpoint=checkpoint, api_base=api_base, model=model ) diff --git a/sieval/cli/session.py b/sieval/cli/session.py index 9b891720..88e01be9 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -128,7 +128,6 @@ RunPlan, TaskRunnerConfig, as_resume, - fold_resume_aliases, resume_source, same_dir, ) @@ -1259,8 +1258,9 @@ def _configured_client_timeout( ) -#: ``resume`` and its deprecated aliases. -_RESUME_KEYS = ("resume", "auto_resume", "resume_from") +#: Resume keys removed in 0.12, refused rather than dropped: a dropped +#: ``resume_from`` would turn a resume into a fresh run without a word. +_REMOVED_RESUME_KEYS = {"auto_resume": "resume: true", "resume_from": "resume: "} def _runner_config_blocks(cfg: Mapping[str, Any]) -> dict[str, dict[str, Any]]: @@ -1275,19 +1275,14 @@ def _runner_config_blocks(cfg: Mapping[str, Any]) -> dict[str, dict[str, Any]]: def config_resumes(cfg: Mapping[str, Any]) -> bool: - """Whether some ``runner_config`` of *cfg* sets ``resume`` or an alias of it.""" - return any( - block.get(key) - for block in _runner_config_blocks(cfg).values() - for key in _RESUME_KEYS - ) + """Whether some ``runner_config`` of *cfg* sets ``resume``.""" + return any(block.get("resume") for block in _runner_config_blocks(cfg).values()) def _drop_resume_keys(cfg: dict[str, Any]) -> None: - """Remove every resume key in place, and a ``runner_config`` left empty.""" + """Remove ``resume`` in place, and a ``runner_config`` left empty.""" for block in _runner_config_blocks(cfg).values(): - for key in _RESUME_KEYS: - block.pop(key, None) + block.pop("resume", None) if cfg.get("runner_config") == {}: del cfg["runner_config"] for task in (cfg.get("tasks") or {}).values(): @@ -1295,21 +1290,14 @@ def _drop_resume_keys(cfg: dict[str, Any]) -> None: del task["runner_config"] -def _fold_runner_resume_aliases(cfg: dict[str, Any]) -> None: - """Rewrite ``auto_resume`` / ``resume_from`` as ``resume``, in place.""" +def _refuse_removed_resume_keys(cfg: Mapping[str, Any]) -> None: + """Raise on ``auto_resume`` / ``resume_from``, naming the ``resume`` form.""" for where, block in _runner_config_blocks(cfg).items(): - if not {"auto_resume", "resume_from"} & block.keys(): - continue - resume = fold_resume_aliases( - as_resume(block.get("resume"), f"{where}.resume"), - auto_resume=block.pop("auto_resume", None), - resume_from=block.pop("resume_from", None), - new=f"{where}.resume", - auto=f"{where}.auto_resume", - source=f"{where}.resume_from", - ) - if resume is not None: - block["resume"] = str(resume) if isinstance(resume, Path) else resume + for key, use in _REMOVED_RESUME_KEYS.items(): + if key in block: + raise ValueError( + f"{where}.{key} was removed in 0.12; use {where}.{use}" + ) def read_persisted_config(path: Path) -> dict[str, Any] | None: @@ -1678,7 +1666,6 @@ def __init__( realized_deployments: Mapping[str, Deployment] | None = None, model_capability_profiles: Mapping[str, ModelCapabilityProfile] | None = None, serving_reconciler: ServingReconciler | None = None, - resume_from: str | Path | None = None, cli_model: CliModel | None = None, infer_gpu: str | None = None, unlaunched_deployments: Mapping[str, Deployment] | None = None, @@ -1687,13 +1674,7 @@ def __init__( self.model_override = model_override # `True`, or a recorded session dir each task starts from; see # `_cli_resume_for`. - self.resume_override: ResumeValue = fold_resume_aliases( - as_resume(resume), - resume_from=resume_from, - new="resume", - auto="resume", - source="resume_from", - ) + self.resume_override: ResumeValue = as_resume(resume) self.result_dir_override = result_dir_override self.deterministic_override = deterministic_override legacy_endpoint_map = dict(endpoint_map or {}) @@ -1794,8 +1775,7 @@ def __init__( model=model_override, result_dir=result_dir_override, ) - # Before persisting, so effective_config.yaml records only `resume`. - _fold_runner_resume_aliases(reified) + _refuse_removed_resume_keys(reified) # Before the copy, so the digests reach both the persisted view (and # therefore --resume) and the runtime view below. Not where the # operation runs: `arun` persists before `_prepare_execution`, so by @@ -5393,7 +5373,6 @@ async def arun_session( invocation: str | None = None, self_managed_endpoints: frozenset[str] | set[str] = frozenset(), realized_deployments: Mapping[str, Deployment] | None = None, - resume_from: str | Path | None = None, cli_model: CliModel | None = None, infer_gpu: str | None = None, unlaunched_deployments: Mapping[str, Deployment] | None = None, @@ -5423,7 +5402,6 @@ async def arun_session( warning to genuinely external endpoints. realized_deployments: Typed realized deployments keyed by root model config name. - resume_from: Deprecated alias of ``resume=``; warns. cli_model: ``--checkpoint``/``--api-base``, added as the ``models:`` entry of a config that has none. infer_gpu: The GPUs *infer_plans* were resolved for, as @@ -5448,7 +5426,6 @@ async def arun_session( invocation=invocation, self_managed_endpoints=self_managed_endpoints, realized_deployments=realized_deployments, - resume_from=resume_from, cli_model=cli_model, infer_gpu=infer_gpu, unlaunched_deployments=unlaunched_deployments, @@ -5503,7 +5480,6 @@ def run_session( invocation: str | None = None, self_managed_endpoints: frozenset[str] | set[str] = frozenset(), realized_deployments: Mapping[str, Deployment] | None = None, - resume_from: str | Path | None = None, ) -> dict[str, Any]: return anyio.run( arun_session, @@ -5517,5 +5493,4 @@ def run_session( invocation, self_managed_endpoints, realized_deployments, - resume_from, ) diff --git a/sieval/core/models/__init__.py b/sieval/core/models/__init__.py index 08280753..b18592f0 100644 --- a/sieval/core/models/__init__.py +++ b/sieval/core/models/__init__.py @@ -110,7 +110,6 @@ reconcile, ) from .sglang_gen_model import SglangGenModel -from .transport import Transport __all__ = [ "BINDING_RESOURCE_KEYS", @@ -199,7 +198,6 @@ "ToolParams", "ToolResultPart", "TopKEntry", - "Transport", "UsageStats", "check_extra", "sequence_wrapper", diff --git a/sieval/core/models/transport.py b/sieval/core/models/transport.py deleted file mode 100644 index 97a249f2..00000000 --- a/sieval/core/models/transport.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Deprecated compatibility name for the provider dialect protocol. - -New production code imports :class:`sieval.core.models.dialect.Dialect`. - -AI-Generated Code - GPT-5.6 (OpenAI) -""" - -from .dialect import Dialect - -Transport = Dialect - -__all__ = ["Transport"] diff --git a/sieval/core/models/transports/__init__.py b/sieval/core/models/transports/__init__.py index ac2fa9d0..b0e2126b 100644 --- a/sieval/core/models/transports/__init__.py +++ b/sieval/core/models/transports/__init__.py @@ -1,21 +1,12 @@ -"""Deprecated transport names and the temporary SGLang legacy executor. +"""The SGLang legacy executor. -The two OpenAI names re-export their canonical -:class:`~sieval.core.models.dialect.Dialect` implementations for one -compatibility cycle. ``SglangTransport`` is different: -it remains the unchanged native ``/generate`` executor behind +``SglangTransport`` is the native ``/generate`` executor behind ``SglangGenModel``'s explicit ``sglang_legacy`` bypass. It is not a canonical Dialect and must not be registered as ``sglang_native`` before PR 5. AI-Generated Code - Claude Opus 4.8 (Anthropic) """ -from .openai_chat import OpenAIChatTransport -from .openai_completions import OpenAICompletionsTransport from .sglang import SglangTransport -__all__ = [ - "OpenAIChatTransport", - "OpenAICompletionsTransport", - "SglangTransport", -] +__all__ = ["SglangTransport"] diff --git a/sieval/core/models/transports/openai_chat.py b/sieval/core/models/transports/openai_chat.py deleted file mode 100644 index 88985c87..00000000 --- a/sieval/core/models/transports/openai_chat.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Deprecated import path for :class:`OpenAIChatDialect`. - -The alias retains the old direct ``arun(Request)`` entry point for one -compatibility cycle; canonical execution goes through ``Model`` and its pool. - -AI-Generated Code - GPT-5.6 (OpenAI) -""" - -from sieval.core.models.dialects.openai_chat import OpenAIChatDialect - -OpenAIChatTransport = OpenAIChatDialect - -__all__ = ["OpenAIChatTransport"] diff --git a/sieval/core/models/transports/openai_completions.py b/sieval/core/models/transports/openai_completions.py deleted file mode 100644 index 4631bcb6..00000000 --- a/sieval/core/models/transports/openai_completions.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Deprecated import path for :class:`OpenAICompletionsDialect`. - -The alias retains the old direct ``arun(Request)`` entry point for one -compatibility cycle; canonical execution goes through ``Model`` and its pool. - -AI-Generated Code - GPT-5.6 (OpenAI) -""" - -from sieval.core.models.dialects.openai_completions import OpenAICompletionsDialect - -OpenAICompletionsTransport = OpenAICompletionsDialect - -__all__ = ["OpenAICompletionsTransport"] diff --git a/sieval/core/runners/__init__.py b/sieval/core/runners/__init__.py index c4c1514a..a5e9f27d 100644 --- a/sieval/core/runners/__init__.py +++ b/sieval/core/runners/__init__.py @@ -3,7 +3,6 @@ from sieval.core.runners.resume_option import ( ResumeValue, as_resume, - fold_resume_aliases, resume_source, same_dir, ) @@ -39,7 +38,6 @@ "UnfinishedCopyError", "UnreadableIntervalError", "as_resume", - "fold_resume_aliases", "plan_resume", "read_run_version", "resume_source", diff --git a/sieval/core/runners/resume_option.py b/sieval/core/runners/resume_option.py index 0f61267e..6ed773c6 100644 --- a/sieval/core/runners/resume_option.py +++ b/sieval/core/runners/resume_option.py @@ -1,8 +1,7 @@ """The ``resume`` option: ``True`` resumes in place, a directory is copied from. A directory that is the task's own ``result_dir`` resumes in place; -:func:`resume_source` is the one place that decides. The deprecated -``auto_resume`` / ``resume_from`` fold in via :func:`fold_resume_aliases`. +:func:`resume_source` is the one place that decides. AI-Generated Code - Claude Opus 5.5 (Anthropic) """ @@ -10,8 +9,6 @@ import os from pathlib import Path -from loguru import logger - type ResumeValue = bool | Path | None @@ -39,71 +36,3 @@ def resume_source(resume: object, own_dir: Path) -> Path | None: if not isinstance(resume, Path): return None return None if same_dir(resume, own_dir) else resume - - -# Deprecated aliases, removed next release. - - -def fold_resume_aliases( - resume: ResumeValue, - *, - auto_resume: object = None, - resume_from: object = None, - new: str, - auto: str, - source: str, -) -> ResumeValue: - """Fold the deprecated aliases into *resume*, warning once each. - - *new* / *auto* / *source* are the names the user wrote, for messages. - ``resume_from`` wins over ``auto_resume``. The result must agree with - *resume*: ``true`` with a directory (kept), a directory only with itself, - ``false`` only with ``false``; otherwise refused. - """ - alias: ResumeValue = None - name = "" - if auto_resume is not None: - if not isinstance(auto_resume, bool): - raise TypeError(f"{auto} must be true or false") - _warn_once(f"{auto} is deprecated; use {new}: {_show(auto_resume)}") - alias, name = auto_resume, auto - if resume_from is not None and resume_from != "": - if isinstance(resume_from, bool): - raise TypeError(f"{source} must be a directory") - _warn_once(f"{source} is deprecated; use {new} ") - alias, name = as_resume(resume_from, source), source - if alias is None: - return resume - if resume is None: - return alias - merged = _merge(resume, alias) - if merged is None: - raise ValueError( - f"{new} ({_show(resume)}) and its deprecated alias {name} " - f"({_show(alias)}) disagree; set only {new}" - ) - return merged - - -def _merge(a: bool | Path, b: bool | Path) -> bool | Path | None: - """What *a* and *b* mean together, or ``None`` when they disagree.""" - if isinstance(a, Path) and isinstance(b, Path): - return a if same_dir(a, b) else None - if isinstance(a, Path) or isinstance(b, Path): - other = b if isinstance(a, Path) else a - return (a if isinstance(a, Path) else b) if other is True else None - return a if a == b else None - - -#: Warned already: a config is read twice per process (prelaunch, then run). -_warned: set[str] = set() - - -def _warn_once(message: str) -> None: - if message not in _warned: - _warned.add(message) - logger.warning("{}", message) - - -def _show(value: object) -> str: - return str(value).lower() if isinstance(value, bool) else str(value) diff --git a/sieval/core/runners/runner.py b/sieval/core/runners/runner.py index 5c3f8fc2..203a7208 100644 --- a/sieval/core/runners/runner.py +++ b/sieval/core/runners/runner.py @@ -106,7 +106,7 @@ format_reject_message, resume_schema_verdict, ) -from .resume_option import as_resume, fold_resume_aliases, resume_source +from .resume_option import as_resume, resume_source from .stage_resume import ( LegacyStageMatch, ResumePlan, @@ -265,16 +265,12 @@ class TaskRunnerConfig: ``result_dir`` and resumed; ``result_dir`` itself resumes in place. Once ``result_dir`` holds a run the copy is skipped, so the same config resumes an interrupted copy. - auto_resume: Deprecated; folded into ``resume`` and cleared. - resume_from: Deprecated; folded into ``resume`` and cleared. effective_config: The session's ``effective_config.yaml``, recorded in ``meta.json`` as ``config``. Not part of any identity. """ result_dir: str | None = None resume: bool | Path | None = None - auto_resume: bool | None = None - resume_from: str | None = None effective_config: str | None = None profile_io: bool = False # timings @@ -325,16 +321,7 @@ class TaskRunnerConfig: deterministic: bool = False def __post_init__(self) -> None: - self.resume = fold_resume_aliases( - as_resume(self.resume), - auto_resume=self.auto_resume, - resume_from=self.resume_from, - new="resume", - auto="auto_resume", - source="resume_from", - ) - self.auto_resume = None - self.resume_from = None + self.resume = as_resume(self.resume) def _read_run_meta(root_dir: Path) -> dict | None: diff --git a/sieval/infer/__init__.py b/sieval/infer/__init__.py index 826b0b04..ccddb4fe 100644 --- a/sieval/infer/__init__.py +++ b/sieval/infer/__init__.py @@ -14,7 +14,6 @@ from sieval.infer.config import ( InferCommand, InferCondition, - InferConfig, InferConfigDict, InferEnv, InferHandle, @@ -50,7 +49,6 @@ __all__ = [ # Config types (kept for compatibility) "InferCommand", - "InferConfig", "InferConfigDict", "InferEnv", "InferHandle", diff --git a/sieval/infer/config.py b/sieval/infer/config.py index 710a8a44..837d19ea 100644 --- a/sieval/infer/config.py +++ b/sieval/infer/config.py @@ -10,7 +10,7 @@ # Infer param values are YAML scalars ParamValue = str | int | float | bool -# Metadata values used across InferHandle and InferConfig +# Metadata values used across InferHandle MetadataValue = str | int | list[str] @@ -106,18 +106,3 @@ class InferEnv: gpu_topo: str = "" python_version: str = "" extra: dict[str, str] = field(default_factory=dict) - - -# Legacy: InferConfig kept for backward compatibility during migration. -# New code should use DeploymentPlan instead. -@dataclass -class InferConfig: - """Resolved infer configuration ready to pass to a backend. - - Deprecated: use DeploymentPlan from sieval.infer.topology instead. - """ - - backend: str - checkpoint: str - params: dict[str, ParamValue] = field(default_factory=dict) - metadata: dict[str, MetadataValue] = field(default_factory=dict) diff --git a/sieval/infer/topology/resolver.py b/sieval/infer/topology/resolver.py index 0f5f3c40..5fe4e465 100644 --- a/sieval/infer/topology/resolver.py +++ b/sieval/infer/topology/resolver.py @@ -381,8 +381,8 @@ async def auto_resolve_plan( ) -> ResolveResult: """Auto-resolve a DeploymentPlan from a checkpoint path. - Replaces resolve.auto_resolve(). Same 5-step flow but returns - ResolveResult instead of InferConfig. + Same 5-step flow as the old resolve.auto_resolve(), returning a + ResolveResult. Args: capability: Recipe capability key (``"instruct"`` / ``"base"``). Base diff --git a/tests/README.md b/tests/README.md index 657ad69e..380e0e58 100644 --- a/tests/README.md +++ b/tests/README.md @@ -31,7 +31,6 @@ tests/ ├── unit/ # Unit tests — mirrors sieval/ structure │ ├── conftest.py │ ├── cli/ # sieval/cli/ -│ │ ├── leaderboard/ # sieval/cli/leaderboard/ (deprecated alias, removed after 0.11) │ │ │ └── test_session.py # The alias re-exports the engine and warns │ │ ├── infer/ # sieval/cli/infer/ │ │ │ ├── test_display.py # Infer display formatting @@ -79,7 +78,7 @@ tests/ │ │ │ └── test_texts.py │ │ └── test_datasets.py │ ├── infer/ # sieval/infer/ -│ │ ├── test_config.py # InferConfig, InferHandle, InferStatus +│ │ ├── test_config.py # InferHandle, InferPhase, config dicts │ │ ├── test_deployer.py # LocalDeployer launch orchestration │ │ ├── test_introspect.py # Checkpoint introspection and GPU detection │ │ ├── test_recipes.py # Recipe loading and merging diff --git a/tests/conftest.py b/tests/conftest.py index db10ac3a..718d4672 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,8 +15,16 @@ AI-Generated Code - Claude Opus 4.6 (Anthropic) """ -import gc -import random +import os + +# `datasets` imports torch whenever it is installed, for a formatter nothing +# here uses: about a second per interpreter, and the suite starts many (xdist +# workers, the offload pool's spawned workers, every subprocess test). Set +# before the first `datasets` import, so those children inherit it too. +os.environ.setdefault("USE_TORCH", "0") + +import gc # noqa: E402 +import random # noqa: E402 import sys import threading import time diff --git a/tests/integration/resume/test_across_releases.py b/tests/integration/resume/test_across_releases.py index 717aff90..9be7d834 100644 --- a/tests/integration/resume/test_across_releases.py +++ b/tests/integration/resume/test_across_releases.py @@ -23,6 +23,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import openai._base_client import orjson import pytest import yaml @@ -273,14 +274,11 @@ def test_a_changed_prompt_restarts_every_sample( upgrade restarts every sample instead of keeping its inference.""" evaluator(_recorded_service(release)) root = _copy(release, tmp_path) - original = HumanEvalZeroShotGenTask.preprocess - - async def preprocess(self, raw, ctx): - record = await original(self, raw, ctx) - record["prompt"][0]["content"] += "\n" - return record - - monkeypatch.setattr(HumanEvalZeroShotGenTask, "preprocess", preprocess) + _patch_preprocess(tmp_path, monkeypatch) + # The refused connections retry; skip the SDK's backoff, not the retries. + monkeypatch.setattr( + openai._base_client.BaseClient, "_calculate_retry_timeout", lambda *_: 0 + ) result = CliRunner().invoke( app, [ @@ -363,28 +361,50 @@ def test_the_dry_run_plan_names_what_a_resume_does(self, release, tmp_path): def _patch_report(tmp_path: Path, monkeypatch) -> None: - """Replace `report` from a package on disk, as a plugin patch does.""" - root = tmp_path / "patches" - (root / "report_patch").mkdir(parents=True) - (root / "report_patch" / "__init__.py").write_text( - textwrap.dedent( - """ - from sieval.tasks import HumanEvalZeroShotGenTask + _patch( + tmp_path, + monkeypatch, + "report", + """ + async def report(self, finals, fails): + return {**(await _original(self, finals, fails)), "patched": True} + """, + ) - _original = HumanEvalZeroShotGenTask.report +def _patch_preprocess(tmp_path: Path, monkeypatch) -> None: + _patch( + tmp_path, + monkeypatch, + "preprocess", + """ + async def preprocess(self, raw, ctx): + record = await _original(self, raw, ctx) + record["prompt"][0]["content"] += "\\n" + return record + """, + ) - async def report(self, finals, fails): - return {**(await _original(self, finals, fails)), "patched": True} - """ - ) + +def _patch(tmp_path: Path, monkeypatch, stage: str, body: str) -> None: + """Replace *stage* from a package on disk, as a plugin patch does. + + A function defined in this test module would make the stage's identity + walk this module and everything it imports, the CLI included. + """ + package = f"{stage}_patch" + root = tmp_path / "patches" + (root / package).mkdir(parents=True) + (root / package / "__init__.py").write_text( + "from sieval.tasks import HumanEvalZeroShotGenTask\n\n" + f"_original = HumanEvalZeroShotGenTask.{stage}\n\n" + textwrap.dedent(body) ) monkeypatch.syspath_prepend(str(root)) - monkeypatch.delitem(sys.modules, "report_patch", raising=False) + monkeypatch.delitem(sys.modules, package, raising=False) import importlib - patch = importlib.import_module("report_patch") - monkeypatch.setattr(HumanEvalZeroShotGenTask, "report", patch.report) + patch = importlib.import_module(package) + monkeypatch.setattr(HumanEvalZeroShotGenTask, stage, getattr(patch, stage)) def _dataset(): diff --git a/tests/integration/test_request_records.py b/tests/integration/test_request_records.py index 07cfe945..66494848 100644 --- a/tests/integration/test_request_records.py +++ b/tests/integration/test_request_records.py @@ -17,7 +17,7 @@ import httpx import pytest -from sieval.cli.leaderboard.session import arun_session +from sieval.cli.session import arun_session from sieval.core.tasks import requests as requests_module from sieval.core.tasks.requests import REQUESTS_FILE, TaskRequestLog from tests.integration.test_api_key_env import ( diff --git a/tests/unit/cli/leaderboard/__init__.py b/tests/unit/cli/leaderboard/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py deleted file mode 100644 index bf82eda3..00000000 --- a/tests/unit/cli/leaderboard/test_session.py +++ /dev/null @@ -1,116 +0,0 @@ -"""The deprecated ``sieval.cli.leaderboard.session`` alias (#194). - -AI-Generated Code - Claude Opus 5.5 (Anthropic) -""" - -import importlib -import sys - -import pytest - -import sieval.cli.resolution as resolution -import sieval.cli.session as session - -_SHIM = "sieval.cli.leaderboard.session" - - -def _import_shim(): - """Import the shim afresh: the warning fires once, at module execution.""" - sys.modules.pop(_SHIM, None) - return importlib.import_module(_SHIM) - - -def test_old_path_warns_with_the_new_one(): - with pytest.warns(DeprecationWarning, match=r"sieval\.cli\.session"): - _import_shim() - - -@pytest.mark.parametrize( - ("name", "home"), - [ - ("EvalSession", session), - ("arun_session", session), - ("run_session", session), - ("resolve_deterministic", session), - ("unwrap_proxies", session), - ("RootConfigDict", session), - ("ModelConfigDict", session), - ("resolve_task_class", resolution), - ("resolve_dataset_class", resolution), - ("resolve_class", resolution), - ], -) -def test_old_path_hands_out_the_same_objects(name, home): - """Identity, not equality: ``isinstance`` and patching need the very same - class and function the new path holds.""" - with pytest.warns(DeprecationWarning): - shim = _import_shim() - assert getattr(shim, name) is getattr(home, name) - - -def test_shim_exports_exactly_the_documented_names(): - with pytest.warns(DeprecationWarning): - shim = _import_shim() - assert set(shim.__all__) == { - "EvalSession", - "ModelConfigDict", - "RootConfigDict", - "arun_session", - "resolve_class", - "resolve_dataset_class", - "resolve_deterministic", - "resolve_task_class", - "run_session", - "unwrap_proxies", - } - - -def test_package_no_longer_exports_aggregation_api(): - import sieval.cli.leaderboard as package - - for gone in ("AlignmentCard", "scan_runs", "build_matrix", "leaderboard_app"): - assert not hasattr(package, gone) - - -def test_a_global_the_old_module_held_is_forwarded(): - """A ``None`` here silently drops the deterministic seed (#204).""" - with pytest.warns(DeprecationWarning): - shim = _import_shim() - with pytest.warns(DeprecationWarning, match=r"sieval\.cli\.session"): - seed = getattr(shim, "DETERMINISTIC_DEFAULT_SEED", None) - assert seed is not None - assert seed is session.DETERMINISTIC_DEFAULT_SEED - - -def test_every_engine_global_resolves_through_the_alias(): - with pytest.warns(DeprecationWarning): - shim = _import_shim() - names = [n for n in vars(session) if not n.startswith("__")] - with pytest.warns(DeprecationWarning): - for name in names: - assert getattr(shim, name) is getattr(session, name), name - - -def test_a_resolver_not_in_all_is_forwarded_from_resolution(): - with pytest.warns(DeprecationWarning): - shim = _import_shim() - name = next(n for n in vars(resolution) if not hasattr(session, n)) - with pytest.warns(DeprecationWarning): - assert getattr(shim, name) is getattr(resolution, name) - - -def test_an_unknown_name_still_raises_attribute_error(): - with pytest.warns(DeprecationWarning): - shim = _import_shim() - assert getattr(shim, "no_such_name", None) is None - with pytest.raises(AttributeError, match="no_such_name"): - shim.no_such_name # noqa: B018 - - -def test_a_dunder_the_engine_holds_is_not_forwarded(monkeypatch): - """``inspect.unwrap`` and the import system probe dunders; they must not - see the engine's.""" - monkeypatch.setattr(session, "__wrapped__", object(), raising=False) - with pytest.warns(DeprecationWarning): - shim = _import_shim() - assert not hasattr(shim, "__wrapped__") diff --git a/tests/unit/cli/test_resume_option.py b/tests/unit/cli/test_resume_option.py index 36c1be68..caffc59d 100644 --- a/tests/unit/cli/test_resume_option.py +++ b/tests/unit/cli/test_resume_option.py @@ -1,6 +1,5 @@ """ -``--resume [DIR]`` on ``sieval run`` and ``sieval eval``: every parse form, and -``--resume-from`` as its alias. +``--resume [DIR]`` on ``sieval run`` and ``sieval eval``: every parse form. AI-Generated Code - Claude Opus 5.5 (Anthropic) """ @@ -162,31 +161,8 @@ def test_help_shows_the_optional_value(self, command): @pytest.mark.parametrize("command", COMMANDS, ids=" ".join) -class TestResumeFromAlias: - def test_maps_to_resume_dir_and_warns(self, command, config, warnings): - code, kw, _ = _invoke(command, [config, "--resume-from", "/runs/old"]) - assert code == 0 - assert kw["resume"] == Path("/runs/old") - assert "resume_from" not in kw - assert warnings == ["--resume-from is deprecated; use --resume "] - - def test_beside_the_same_resume_dir(self, command, config, warnings): - code, kw, _ = _invoke( - command, [config, "--resume", "/runs/old", "--resume-from", "/runs/old"] - ) - assert code == 0 - assert kw["resume"] == Path("/runs/old") - - def test_beside_resume_alone_keeps_the_directory(self, command, config, warnings): - """``--resume --resume-from DIR`` meant ``DIR`` before.""" - code, kw, _ = _invoke(command, [config, "--resume", "--resume-from", "/d"]) - assert code == 0 - assert kw["resume"] == Path("/d") - - def test_beside_another_resume_dir_is_refused(self, command, config, warnings): - code, kw, output = _invoke( - command, [config, "--resume", "/a", "--resume-from", "/b"] - ) - assert code == 1 - assert kw == {} - assert "disagree; set only --resume" in output +def test_resume_from_was_removed(command, config): + code, kw, output = _invoke(command, [config, "--resume-from", "/runs/old"]) + assert code == 2 + assert kw == {} + assert "--resume-from" in output diff --git a/tests/unit/cli/test_session.py b/tests/unit/cli/test_session.py index 9ec54b6e..4f06352b 100644 --- a/tests/unit/cli/test_session.py +++ b/tests/unit/cli/test_session.py @@ -7437,7 +7437,6 @@ async def test_arun_session_delegates_to_runner(self): invocation=None, self_managed_endpoints=frozenset(), realized_deployments=None, - resume_from=None, cli_model=None, infer_gpu=None, unlaunched_deployments=None, @@ -7472,7 +7471,6 @@ async def test_arun_session_passes_deterministic(self): invocation=None, self_managed_endpoints=frozenset(), realized_deployments=None, - resume_from=None, cli_model=None, infer_gpu=None, unlaunched_deployments=None, @@ -7504,7 +7502,6 @@ def test_run_session_delegates_to_anyio_run(self): None, frozenset(), None, - None, ) assert result == {"task_b": {"ok": True}} @@ -7536,7 +7533,6 @@ async def test_arun_session_forwards_legacy_external_endpoint_adapter(self): invocation=None, self_managed_endpoints=frozenset(), realized_deployments=None, - resume_from=None, cli_model=None, infer_gpu=None, unlaunched_deployments=None, @@ -7567,7 +7563,6 @@ def test_run_session_forwards_legacy_external_endpoint_adapter_positionally( None, frozenset(), None, - None, ) def test_eval_session_run_calls_anyio_run(self): diff --git a/tests/unit/cli/test_session_persistence.py b/tests/unit/cli/test_session_persistence.py index df0e8f45..2327dbe4 100644 --- a/tests/unit/cli/test_session_persistence.py +++ b/tests/unit/cli/test_session_persistence.py @@ -1051,8 +1051,7 @@ def test_yaml_true_resumes_without_the_cli(self, tmp_path: Path): assert rc.resume is True def test_a_cli_resume_wins_over_yaml_false(self, tmp_path: Path): - """`resume: false` in YAML is the default; `--resume` turns it on, as - `--resume` did over `auto_resume: false`.""" + """`resume: false` in YAML is the default; `--resume` turns it on.""" session = self._session(tmp_path, resume=True) rc = session._build_runner_config({}, {"resume": False}, "qa") assert rc.resume is True @@ -1154,8 +1153,8 @@ def test_a_fresh_run_has_nothing_to_compare(self, tmp_path: Path): assert session._legacy_match_for("qa", rc) is None -class TestDeprecatedResumeKeys: - """`runner_config.auto_resume` and `.resume_from` fold into `.resume`.""" +class TestRemovedResumeKeys: + """`runner_config.auto_resume` and `.resume_from` are refused, naming `resume`.""" BASE = TestResumeFromSurface.CONFIG @@ -1172,82 +1171,18 @@ def _session(self, tmp_path: Path, top: str = "", task: str = "", **kw): config_path=str(cfg), result_dir_override=str(tmp_path / "out"), **kw ) - def test_auto_resume_becomes_resume(self, tmp_path: Path, warnings): - session = self._session(tmp_path, top=" auto_resume: true\n") - assert session._reified_config["runner_config"] == {"resume": True} - rc = session._build_runner_config({}, session.config["runner_config"], "qa") - assert rc.resume is True - assert warnings == [ - "runner_config.auto_resume is deprecated; use runner_config.resume: true" - ] - - def test_resume_from_becomes_resume(self, tmp_path: Path, warnings): - session = self._session(tmp_path, task=" resume_from: /mnt/old/qa\n") - task_rc = session._reified_config["tasks"]["qa"]["runner_config"] - assert task_rc == {"resume": "/mnt/old/qa"} - rc = session._build_runner_config(session.config["tasks"]["qa"], {}, "qa") - assert rc.resume == Path("/mnt/old/qa") - assert warnings == [ - "tasks.qa.runner_config.resume_from is deprecated; " - "use tasks.qa.runner_config.resume " - ] - - @pytest.mark.anyio - async def test_effective_config_records_only_resume(self, tmp_path: Path): - session = self._session( - tmp_path, - top=" auto_resume: false\n", - task=" resume_from: /mnt/old/qa\n", - ) - await session._persist_effective_config() - persisted = yaml.safe_load( - (tmp_path / "out" / "effective_config.yaml").read_text() - ) - assert persisted["runner_config"] == {"resume": False} - assert persisted["tasks"]["qa"]["runner_config"] == {"resume": "/mnt/old/qa"} - - def test_an_alias_beside_a_different_resume_is_refused(self, tmp_path: Path): - with pytest.raises(ValueError, match="disagree; set only"): - self._session( - tmp_path, - task=" resume: /mnt/a/qa\n resume_from: /mnt/b/qa\n", - ) - - def test_auto_resume_false_beside_resume_true_is_refused(self, tmp_path: Path): - with pytest.raises(ValueError, match="runner_config.auto_resume"): - self._session(tmp_path, top=" resume: true\n auto_resume: false\n") - - def test_a_per_task_alias_beats_the_cli(self, tmp_path: Path, warnings): - session = self._session( - tmp_path, task=" resume_from: /mnt/old/qa\n", resume=True - ) - rc = session._build_runner_config(session.config["tasks"]["qa"], {}, "qa") - assert rc.resume == Path("/mnt/old/qa") - - def test_the_session_alias_maps_to_resume(self, tmp_path: Path, warnings): - (tmp_path / "old" / "qa").mkdir(parents=True) - session = self._session(tmp_path, resume_from=tmp_path / "old") - assert session.resume_override == tmp_path / "old" - rc = session._build_runner_config({}, {}, "qa") - assert rc.resume == tmp_path / "old" / "qa" - assert warnings == ["resume_from is deprecated; use resume "] - - def test_run_session_takes_the_alias_alone(self, tmp_path: Path, warnings): - """``resume`` left at its default must not read as ``false``, which - disagrees with any ``resume_from``.""" - from sieval.cli import session as session_mod - - built: list[EvalSession] = [] - - def build(*args, **kwargs): - built.append(EvalSession(*args, **kwargs)) - raise KeyboardInterrupt # stop before anything runs + def test_auto_resume_is_refused(self, tmp_path: Path): + with pytest.raises( + ValueError, + match=r"runner_config\.auto_resume was removed in 0\.12; " + r"use runner_config\.resume: true", + ): + self._session(tmp_path, top=" auto_resume: true\n") - cfg = _write_yaml(tmp_path, "cfg.yaml", self.BASE) - with ( - patch.object(session_mod, "EvalSession", build), - pytest.raises(KeyboardInterrupt), + def test_resume_from_is_refused(self, tmp_path: Path): + with pytest.raises( + ValueError, + match=r"tasks\.qa\.runner_config\.resume_from was removed in 0\.12; " + r"use tasks\.qa\.runner_config\.resume: ", ): - session_mod.run_session(cfg, resume_from=tmp_path / "old") - assert built[0].resume_override == tmp_path / "old" - assert warnings == ["resume_from is deprecated; use resume "] + self._session(tmp_path, task=" resume_from: /mnt/old/qa\n") diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8b432f46..047e6ba0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,12 +7,9 @@ import os import pytest -from loguru import logger from sieval.core.models import ModelMeta, ModelOutput, ModelUsage -from sieval.core.runners import resume_option from sieval.core.tasks.context import TaskContext, TaskStageMeta -from sieval.core.utils.logging import configure_logging # scripts/ tests are repo-hygiene checks (preflight, layer imports) that run # tooling against the LIVE repo tree. Inside mutmut's mutants/ sandbox that @@ -68,19 +65,3 @@ def base_context() -> TaskContext: return TaskContext( sample_id=0, raw_sample={"question": "What is 1+1?", "answer": "2"} ) - - -@pytest.fixture -def warnings(): - """Warning messages logged during the test; resume deprecations warn anew. - - ``configure_logging()`` first: its first call in a process removes every - sink, so a CLI under test would otherwise drop this one. - """ - configure_logging() - resume_option._warned.clear() - messages: list[str] = [] - sink = logger.add(lambda m: messages.append(m.record["message"]), level="WARNING") - yield messages - logger.remove(sink) - resume_option._warned.clear() diff --git a/tests/unit/core/models/test_chat_model.py b/tests/unit/core/models/test_chat_model.py index 4cc15b73..6a58779d 100644 --- a/tests/unit/core/models/test_chat_model.py +++ b/tests/unit/core/models/test_chat_model.py @@ -1,9 +1,9 @@ """Shell tests: backend selector wiring for ChatModel. RFC #25 moved the chat-completions wire logic (streaming accumulation, -reasoning extraction, logprob parsing) into ``OpenAIChatTransport``; the +reasoning extraction, logprob parsing) into ``OpenAIChatDialect``; the former ``_agenerate_impl`` / ``_alogprobs_impl`` coverage moved with it to -tests/unit/core/models/transports/test_openai_chat.py, and the request-builder +tests/unit/core/models/dialects/test_openai_chat.py, and the request-builder validation (n/stream types, alogprobs n=1) lives on ``Model`` in tests/unit/core/models/test_model.py. What remains here is the selector contract: ChatModel pairs the shared client with that transport, which @@ -21,18 +21,18 @@ ChatModel, TextPart, ) -from sieval.core.models.transports.openai_chat import OpenAIChatTransport +from sieval.core.models.dialects.openai_chat import OpenAIChatDialect class TestDefaultTransport: def test_builds_openai_chat_transport(self): m = ChatModel(model="m", api_key="k") - assert isinstance(m._transport, OpenAIChatTransport) + assert isinstance(m._transport, OpenAIChatDialect) assert Capability.Chat in m.capabilities def test_transport_bound_to_shared_client_and_model(self): m = ChatModel(model="m", api_key="k") - assert isinstance(m._transport, OpenAIChatTransport) + assert isinstance(m._transport, OpenAIChatDialect) assert m._transport._client is m._client assert m._transport._requested_model_id == "m" diff --git a/tests/unit/core/models/test_gen_model.py b/tests/unit/core/models/test_gen_model.py index f51c85bf..75c23f2d 100644 --- a/tests/unit/core/models/test_gen_model.py +++ b/tests/unit/core/models/test_gen_model.py @@ -1,9 +1,9 @@ """Shell tests: backend selector wiring for GenModel. RFC #25 moved the completions wire logic (streaming accumulation, echo split, -``_completion_top_logprobs`` sanitizing) into ``OpenAICompletionsTransport``; +``_completion_top_logprobs`` sanitizing) into ``OpenAICompletionsDialect``; the former ``_agenerate_impl`` / ``_alogprobs_impl`` coverage moved with it to -tests/unit/core/models/transports/test_openai_completions.py, and the +tests/unit/core/models/dialects/test_openai_completions.py, and the request-builder validation (n/stream types, alogprobs n=1) lives on ``Model`` in tests/unit/core/models/test_model.py. What remains here is the selector contract: GenModel pairs the shared client with that transport, which supplies @@ -13,20 +13,18 @@ """ from sieval.core.models import Capability, CompletionInput, GenModel -from sieval.core.models.transports.openai_completions import ( - OpenAICompletionsTransport, -) +from sieval.core.models.dialects.openai_completions import OpenAICompletionsDialect class TestDefaultTransport: def test_builds_openai_completions_transport(self): m = GenModel(model="m", api_key="k") - assert isinstance(m._transport, OpenAICompletionsTransport) + assert isinstance(m._transport, OpenAICompletionsDialect) assert Capability.InputScoring in m.capabilities def test_transport_bound_to_shared_client_and_model(self): m = GenModel(model="m", api_key="k") - assert isinstance(m._transport, OpenAICompletionsTransport) + assert isinstance(m._transport, OpenAICompletionsDialect) assert m._transport._client is m._client assert m._transport._model == "m" diff --git a/tests/unit/core/models/transports/test_openai_chat.py b/tests/unit/core/models/transports/test_openai_chat.py deleted file mode 100644 index b5af0b8a..00000000 --- a/tests/unit/core/models/transports/test_openai_chat.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Compatibility checks for the deprecated OpenAI chat transport path. - -Wire behavior is covered by ``tests/unit/core/models/dialects/test_openai_chat.py``; -this module only protects the one-cycle import alias. - -AI-Generated Code - GPT-5.6 (OpenAI) -""" - -from sieval.core.models.dialects.openai_chat import OpenAIChatDialect -from sieval.core.models.transports.openai_chat import OpenAIChatTransport - - -def test_openai_chat_transport_is_dialect_alias() -> None: - assert OpenAIChatTransport is OpenAIChatDialect diff --git a/tests/unit/core/models/transports/test_openai_completions.py b/tests/unit/core/models/transports/test_openai_completions.py deleted file mode 100644 index 787da093..00000000 --- a/tests/unit/core/models/transports/test_openai_completions.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Compatibility checks for the deprecated completions transport path. - -Wire behavior is covered by -``tests/unit/core/models/dialects/test_openai_completions.py``; this module -protects the one-cycle import alias and its direct-call entry point. - -AI-Generated Code - GPT-5.6 (OpenAI) -""" - -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from sieval.core.models.dialects.openai_completions import OpenAICompletionsDialect -from sieval.core.models.ir import CompletionInput, Request -from sieval.core.models.transports.openai_completions import ( - OpenAICompletionsTransport, -) - - -def test_openai_completions_transport_is_dialect_alias() -> None: - assert OpenAICompletionsTransport is OpenAICompletionsDialect - - -@pytest.mark.anyio -async def test_openai_completions_transport_retains_direct_arun() -> None: - raw = SimpleNamespace( - choices=[ - SimpleNamespace( - index=0, - text="answer", - finish_reason="stop", - logprobs=None, - ) - ], - usage=None, - model="served-model", - system_fingerprint=None, - ) - create = AsyncMock(return_value=raw) - client = SimpleNamespace(completions=SimpleNamespace(create=create)) - transport = OpenAICompletionsTransport(client, "requested-model") - - response = await transport.arun(Request(input=CompletionInput("prompt"))) - - assert response.texts == ("answer",) - create.assert_awaited_once_with( - model="requested-model", - prompt="prompt", - stream=False, - ) diff --git a/tests/unit/core/runners/test_resume_option.py b/tests/unit/core/runners/test_resume_option.py index 87cb4328..1a00a526 100644 --- a/tests/unit/core/runners/test_resume_option.py +++ b/tests/unit/core/runners/test_resume_option.py @@ -1,6 +1,5 @@ """ -``resume``: one option for an in-place resume and a copy, and its deprecated -aliases ``auto_resume`` and ``resume_from``. +``resume``: one option for an in-place resume and a copy. AI-Generated Code - Claude Opus 5.5 (Anthropic) """ @@ -9,20 +8,10 @@ import pytest -from sieval.core.runners.resume_option import ( - as_resume, - fold_resume_aliases, - resume_source, -) +from sieval.core.runners.resume_option import as_resume, resume_source from sieval.core.runners.runner import TaskRunnerConfig -def _fold(resume=None, **aliases): - return fold_resume_aliases( - resume, new="resume", auto="auto_resume", source="resume_from", **aliases - ) - - class TestAsResume: @pytest.mark.parametrize("value", [None, True, False]) def test_bools_and_none_stay(self, value): @@ -58,72 +47,6 @@ def test_the_own_directory_resumes_in_place(self, tmp_path): assert resume_source(tmp_path / "a" / ".." / "own", tmp_path / "own") is None -class TestAliases: - def test_none_given_leaves_resume(self, warnings): - assert _fold(True) is True - assert _fold(None) is None - assert warnings == [] - - @pytest.mark.parametrize("value", [True, False]) - def test_auto_resume_maps_to_resume_and_warns(self, warnings, value): - assert _fold(auto_resume=value) is value - assert warnings == [ - f"auto_resume is deprecated; use resume: {str(value).lower()}" - ] - - def test_resume_from_maps_to_a_directory_and_warns(self, warnings, tmp_path): - assert _fold(resume_from=str(tmp_path)) == tmp_path - assert warnings == ["resume_from is deprecated; use resume "] - - def test_each_alias_warns_once_per_process(self, warnings, tmp_path): - _fold(resume_from=str(tmp_path)) - _fold(resume_from=str(tmp_path)) - assert len(warnings) == 1 - - def test_resume_from_wins_over_auto_resume(self, warnings, tmp_path): - assert _fold(auto_resume=True, resume_from=str(tmp_path)) == tmp_path - assert len(warnings) == 2 - - def test_an_empty_resume_from_is_unset(self, warnings): - assert _fold(resume_from="") is None - assert warnings == [] - - @pytest.mark.parametrize( - ("resume", "aliases", "expected"), - [ - (True, {"auto_resume": True}, True), - (False, {"auto_resume": False}, False), - (Path("/d"), {"auto_resume": True}, Path("/d")), - (True, {"resume_from": "/d"}, Path("/d")), - (Path("/d"), {"resume_from": "/d/"}, Path("/d")), - ], - ) - def test_agreeing_values_are_one(self, warnings, resume, aliases, expected): - assert _fold(resume, **aliases) == expected - - @pytest.mark.parametrize( - ("resume", "aliases"), - [ - (True, {"auto_resume": False}), - (False, {"auto_resume": True}), - (Path("/d"), {"auto_resume": False}), - (False, {"resume_from": "/d"}), - (Path("/d"), {"resume_from": "/e"}), - ], - ) - def test_disagreeing_values_are_refused(self, warnings, resume, aliases): - with pytest.raises(ValueError, match="disagree; set only resume"): - _fold(resume, **aliases) - - def test_a_non_bool_auto_resume_is_refused(self): - with pytest.raises(TypeError, match="auto_resume must be true or false"): - _fold(auto_resume="yes") - - def test_a_bool_resume_from_is_refused(self): - with pytest.raises(TypeError, match="resume_from must be a directory"): - _fold(resume_from=True) - - class TestTaskRunnerConfig: def test_resume_defaults_to_none(self): assert TaskRunnerConfig().resume is None @@ -132,19 +55,3 @@ def test_a_string_resume_is_a_path(self, tmp_path): """YAML hands the runner a string; the field is typed for Python.""" cfg = TaskRunnerConfig(resume=str(tmp_path)) # ty: ignore[invalid-argument-type] assert cfg.resume == tmp_path - - def test_auto_resume_folds_into_resume(self, warnings): - cfg = TaskRunnerConfig(auto_resume=True) - assert cfg.resume is True - assert cfg.auto_resume is None - assert warnings == ["auto_resume is deprecated; use resume: true"] - - def test_resume_from_folds_into_resume(self, warnings, tmp_path): - cfg = TaskRunnerConfig(resume_from=str(tmp_path)) - assert cfg.resume == tmp_path - assert cfg.resume_from is None - assert warnings == ["resume_from is deprecated; use resume "] - - def test_an_alias_disagreeing_with_resume_is_refused(self, tmp_path): - with pytest.raises(ValueError, match="disagree"): - TaskRunnerConfig(resume=tmp_path / "a", resume_from=str(tmp_path / "b")) diff --git a/tests/unit/core/runners/test_stage_resume.py b/tests/unit/core/runners/test_stage_resume.py index 25d68661..6c703442 100644 --- a/tests/unit/core/runners/test_stage_resume.py +++ b/tests/unit/core/runners/test_stage_resume.py @@ -443,22 +443,6 @@ async def test_true_and_its_own_directory_record_alike(self, tmp_path): r["start"] for r in _meta(tmp_path / "b")["runs"] ] - @pytest.mark.anyio - async def test_the_deprecated_resume_from_still_copies(self, tmp_path): - src = await _first(tmp_path) - task, model = _task(cls=StrictQATask) - runner, _ = await _run(tmp_path, "second", task, resume_from=str(src)) - assert model.call_count == 0 - assert _meta(runner.root_dir)["runs"][0]["start"] == "from" - - @pytest.mark.anyio - async def test_the_deprecated_auto_resume_still_resumes(self, tmp_path): - src = await _first(tmp_path) - task, model = _task() - runner, _ = await _run(tmp_path, "first", task, auto_resume=True) - assert model.call_count == 0 - assert runner.root_dir == src - def test_the_copy_holds_only_shards(self, tmp_path): src = tmp_path / "src" (src / "0" / "final").mkdir(parents=True) diff --git a/tests/unit/core/tasks/test_task.py b/tests/unit/core/tasks/test_task.py index d3e45ff1..b589ec6d 100644 --- a/tests/unit/core/tasks/test_task.py +++ b/tests/unit/core/tasks/test_task.py @@ -218,10 +218,6 @@ def test_gen_task_with_chat_model_raises(self): with pytest.raises(TypeError, match="completion"): _GenOnlyTask(_SimpleDataset(), _MockChatModel()) - def test_no_model_type_restriction_accepts_both(self): - _ConcreteTask(_SimpleDataset(), _MockChatModel()) - _ConcreteTask(_SimpleDataset(), _MockGenModel()) - def test_unrecognized_model_type_raises(self): class _CustomModel(Model): """Bare model with no bound dialect shape.""" diff --git a/tests/unit/infer/test_config.py b/tests/unit/infer/test_config.py index 9f07acf7..ba08f1ef 100644 --- a/tests/unit/infer/test_config.py +++ b/tests/unit/infer/test_config.py @@ -7,7 +7,6 @@ from sieval.infer.config import ( InferCondition, - InferConfig, InferConfigDict, InferHandle, InferMetaDict, @@ -88,44 +87,6 @@ def test_custom_metadata(self) -> None: assert handle.metadata == meta -class TestInferConfig: - """Tests for InferConfig dataclass.""" - - def test_required_fields(self) -> None: - cfg = InferConfig(backend="vllm", checkpoint="/models/qwen3-8b") - assert cfg.backend == "vllm" - assert cfg.checkpoint == "/models/qwen3-8b" - - def test_params_default_factory(self) -> None: - cfg = InferConfig(backend="vllm", checkpoint="/models/qwen3-8b") - assert cfg.params == {} - assert isinstance(cfg.params, dict) - - def test_metadata_default_factory(self) -> None: - cfg = InferConfig(backend="vllm", checkpoint="/models/qwen3-8b") - assert cfg.metadata == {} - assert isinstance(cfg.metadata, dict) - - def test_defaults_not_shared(self) -> None: - """Each instance should get its own default dicts.""" - c1 = InferConfig(backend="vllm", checkpoint="/a") - c2 = InferConfig(backend="vllm", checkpoint="/b") - c1.params["key"] = "value" - c1.metadata["key"] = "value" - assert "key" not in c2.params - assert "key" not in c2.metadata - - def test_custom_params_and_metadata(self) -> None: - cfg = InferConfig( - backend="sglang", - checkpoint="/models/qwen2.5-72b", - params={"dtype": "bfloat16", "tp": 2}, - metadata={"framework": "sglang==0.3.0"}, - ) - assert cfg.params["dtype"] == "bfloat16" - assert cfg.metadata["framework"] == "sglang==0.3.0" - - class TestInferConfigDict: """Tests for InferConfigDict TypedDict.""" diff --git a/tests/unit/scripts/test_check_layer_imports.py b/tests/unit/scripts/test_check_layer_imports.py index cdc4fb4b..d563ad77 100644 --- a/tests/unit/scripts/test_check_layer_imports.py +++ b/tests/unit/scripts/test_check_layer_imports.py @@ -1047,18 +1047,13 @@ def test_reaches_the_verdict_through_check_file(self, tmp_path: Path): assert any("must not import sieval.cli.session" in e for e in errors) def test_forbidden_subpackages_for_matches_subtree_not_prefix(self): - shim = "sieval.cli.leaderboard" - assert _forbidden_subpackages_for("sieval.cli.infer") == { - "sieval.cli.session", - shim, - } + assert _forbidden_subpackages_for("sieval.cli.infer") == {"sieval.cli.session"} assert _forbidden_subpackages_for("sieval.cli.infer.sub") == { - "sieval.cli.session", - shim, + "sieval.cli.session" } # A sibling whose name merely starts with the root string must not match. - assert _forbidden_subpackages_for("sieval.cli.inference") == {shim} - assert _forbidden_subpackages_for("sieval.cli") == {shim} + assert _forbidden_subpackages_for("sieval.cli.inference") == set() + assert _forbidden_subpackages_for("sieval.cli") == set() def test_rule_table_entries_are_dotted_sieval_packages(self): """A non-dotted entry can never match, so the rule would pass vacuously.""" @@ -1066,24 +1061,6 @@ def test_rule_table_entries_are_dotted_sieval_packages(self): assert root == "sieval" or root.startswith("sieval.") assert all(t.startswith("sieval.") for t in targets) - @pytest.mark.parametrize( - ("rel", "body"), - [ - ("sieval/cli/run.py", "from sieval.cli.leaderboard.session import x\n"), - ("sieval/cli/run.py", "from sieval.cli import leaderboard\n"), - ("sieval/core/foo.py", "import sieval.cli.leaderboard.session\n"), - ], - ) - def test_nothing_under_sieval_imports_the_deprecated_shim( - self, tmp_path: Path, rel: str, body: str - ): - """The shim is deleted in the release after 0.11; no internal caller - may come to depend on it first (#194).""" - f = self._write(tmp_path, rel, body) - errors = _check_subpackage_imports(f, ast.parse(body)) - assert len(errors) == 1 - assert "must not import sieval.cli.leaderboard" in errors[0] - @pytest.mark.parametrize( "body", [ diff --git a/tests/unit/tasks/conftest.py b/tests/unit/tasks/conftest.py index 86c4978c..35922ea8 100644 --- a/tests/unit/tasks/conftest.py +++ b/tests/unit/tasks/conftest.py @@ -1,4 +1,5 @@ -"""Task-test collection tweaks: NLTK staging under xdist, and mutation testing. +"""Task-test collection tweaks (NLTK staging under xdist, mutation testing), +and one shared fixture. **NLTK staging.** ``sieval/community/ifbench/instructions_util.py`` calls ``download_nltk_resources()`` at module scope, so the corpora arrive with whoever @@ -33,10 +34,15 @@ Skipping happens before fixture setup, so the probe interpreter never starts. +**BigQuery credentials.** ``no_bigquery_credentials`` is shared by the backend +and the task tests. Deleting the variable alone is not enough: google-auth then +probes the GCE metadata server, which on a host off GCP costs ~12s of timeouts. + AI-Generated Code - Claude Fable 5 (Anthropic) """ import contextlib +import importlib import os import tempfile from pathlib import Path @@ -44,6 +50,8 @@ import pytest from filelock import FileLock +from sieval.tasks._spider2_backends import BIGQUERY_CREDENTIAL_ENV + def _warm_stem() -> Path | None: """Where this run's lock and sentinel live, or ``None`` outside xdist.""" @@ -96,3 +104,30 @@ def pytest_collection_modifyitems(items): for item in items: if item.name.startswith("test_import_does_not_pull_"): item.add_marker(skip) + + +@pytest.fixture +def no_bigquery_credentials(monkeypatch): + """No environment key *and* no application default credentials. + + Both halves are needed since `missing_credentials` learned to accept ADC: + deleting the variable alone leaves the answer dependent on whether the host + running the suite happens to have `gcloud auth application-default login` + in its state — green in CI, red on a developer box that uses GCP. + """ + monkeypatch.delenv(BIGQUERY_CREDENTIAL_ENV, raising=False) + try: + # `import_module`, not a plain `import`: CI installs eight dependency + # groups and `spider2` is not one of them, so a static import here is an + # `unresolved-import` in the typecheck job — which the scoped override + # for `_spider2_backends.py` does not cover, because this is a test. + google_auth = importlib.import_module("google.auth") + except ImportError: + # The `spider2` group is absent, so the probe's own ImportError branch + # already reports "unreachable" — nothing to patch. + return + + def _nothing_configured(*_args, **_kwargs): + raise RuntimeError("no application default credentials") + + monkeypatch.setattr(google_auth, "default", _nothing_configured) diff --git a/tests/unit/tasks/test__spider2_backends.py b/tests/unit/tasks/test__spider2_backends.py index fbbf6984..453249e7 100644 --- a/tests/unit/tasks/test__spider2_backends.py +++ b/tests/unit/tasks/test__spider2_backends.py @@ -13,7 +13,6 @@ AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) """ -import importlib import sqlite3 import pandas as pd @@ -120,33 +119,6 @@ def test_unknown_backend_is_rejected(): # --- remote engines: credentials -------------------------------------------- -@pytest.fixture -def no_bigquery_credentials(monkeypatch): - """No environment key *and* no application default credentials. - - Both halves are needed since `missing_credentials` learned to accept ADC: - deleting the variable alone leaves the answer dependent on whether the host - running the suite happens to have `gcloud auth application-default login` - in its state — green in CI, red on a developer box that uses GCP. - """ - monkeypatch.delenv(BIGQUERY_CREDENTIAL_ENV, raising=False) - try: - # `import_module`, not a plain `import`: CI installs eight dependency - # groups and `spider2` is not one of them, so a static import here is an - # `unresolved-import` in the typecheck job — which the scoped override - # for `_spider2_backends.py` does not cover, because this is a test. - google_auth = importlib.import_module("google.auth") - except ImportError: - # The `spider2` group is absent, so the probe's own ImportError branch - # already reports "unreachable" — nothing to patch. - return - - def _nothing_configured(*_args, **_kwargs): - raise RuntimeError("no application default credentials") - - monkeypatch.setattr(google_auth, "default", _nothing_configured) - - @pytest.mark.usefixtures("no_bigquery_credentials") def test_bigquery_without_credentials_raises_missing_credentials(): """Distinguishable from a wrong answer, and it names the variable.""" diff --git a/tests/unit/tasks/test_spider2_lite_0shot_gen.py b/tests/unit/tasks/test_spider2_lite_0shot_gen.py index e2736a24..2ad245a2 100644 --- a/tests/unit/tasks/test_spider2_lite_0shot_gen.py +++ b/tests/unit/tasks/test_spider2_lite_0shot_gen.py @@ -281,10 +281,8 @@ async def test_unrunnable_sql_is_recorded_as_an_error(staged): @pytest.mark.anyio -async def test_missing_cloud_credentials_are_flagged_not_silently_wrong( - staged, monkeypatch -): - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) +@pytest.mark.usefixtures("no_bigquery_credentials") +async def test_missing_cloud_credentials_are_flagged_not_silently_wrong(staged): row = dict(_ROW, instance_id="bq001", db="warehouse") task = _task(staged, rows=[row]) _, judgement = await _judge(task, row)