From 928f115757f3ce094cac5c6793e0d4605d2d9370 Mon Sep 17 00:00:00 2001 From: Ethan Date: Wed, 28 Oct 2026 02:47:01 +0800 Subject: [PATCH 1/2] feat(core)!: count the served model's numerics in the inference identity The inference identity now keeps `numerics`: the effective dtype, runtime quantization, kv_cache_dtype and model overrides (json_model_override_args / hf_overrides) the served model computes with. They sit beside the digest as values, so a key unknown on either side (an endpoint that does not state it) is no difference, while a known one that differs refuses a resume. Every other engine parameter stays serving. - NUMERICS_PARAMS is the one list of which engine params count. - Unset values resolve to the effective ones: dtype to the checkpoint's torch_dtype, quantization to its quant_method, kv_cache_dtype to auto; aliases (bf16, fp8) are normalized. - Endpoints state numerics under the root model's serving.numerics. - IDENTITY_SCHEMA 4 -> 5. A schema-4 directory's numerics are read from the plan of its latest run in runs[].serving, resolved against this run's checkpoint, so recorded results keep resuming. Closes #258 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/guide/configuration.md | 32 ++- sieval/cli/session.py | 18 ++ sieval/cli/validation.py | 12 +- sieval/core/runners/multi_runner.py | 8 +- sieval/core/runners/runner.py | 40 ++- sieval/core/runners/stage_resume.py | 21 +- sieval/core/tasks/run_meta.py | 241 +++++++++++++++++- sieval/infer/introspect.py | 24 ++ .../resume/test_across_releases.py | 2 +- tests/unit/cli/test_session.py | 17 ++ tests/unit/cli/test_validation.py | 10 + .../unit/core/runners/test_runner_run_meta.py | 156 +++++++++++- tests/unit/core/runners/test_stage_resume.py | 22 ++ tests/unit/core/tasks/test_run_meta.py | 212 ++++++++++++++- tests/unit/infer/test_introspect.py | 28 ++ 16 files changed, 808 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6e3871b4..6f21c0f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Bump `RESULT_SCHEMA_VERSION` (`sieval/core/tasks/run_meta.py`) in the PR that ma - a change to a task result directory's layout (shards, `.idx`, `meta.json`, the `report.json` shape) that older directories cannot be read under; - a core change that alters results. The grading identities do not walk `sieval.core` or `sieval.datasets`, so nothing else catches it. -`IDENTITY_SCHEMA` versions only the `identity` block. Bumping it does not refuse a resume, so a new schema must read the old one: schema 2 reads schema 1's `grader` through `legacy_grader_keys`, which keeps the old walks unchanged; schema 3 reads the rendered-prompt `preprocess` of schemas 1 and 2 by rendering the recorded samples once. +`IDENTITY_SCHEMA` versions only the `identity` block. Bumping it does not refuse a resume, so a new schema must read the old one: schema 2 reads schema 1's `grader` through `legacy_grader_keys`, which keeps the old walks unchanged; schema 3 reads the rendered-prompt `preprocess` of schemas 1 and 2 by rendering the recorded samples once; schema 5 reads schema 4's missing `inference.numerics` from the plan in its `runs[].serving`. `Model.extra` is the capability contract between tasks and models (`sieval/core/models/extra.py`): a task reads a key in `preprocess`, an absent key changes nothing, and `core/` only checks the shape of documented keys. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4b9d1e90..047f5977 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -364,7 +364,7 @@ run's, and reruns each sample from the earliest stage that changed: | --- | --- | | `data` (the selected samples) | Those samples rerun from the start; the rest are kept | | `preprocess` (prompt rendering: its code and the arguments it reads) | Every sample reruns from the start | -| `inference` (what the model is asked: request arguments, dialect, the task's inference arguments, `deterministic`, `max_iterations`, `max_retries`, context limit) | Refused, naming the stage | +| `inference` (what the model is asked: request arguments, dialect, the task's inference arguments, `deterministic`, `max_iterations`, `max_retries`, context limit; and what the served model computes with: `dtype`, `quantization`, `kv_cache_dtype`, model overrides) | Refused, naming the stage | | `postprocess` (answer extraction) | `postprocess`, `feedback` and `report` rerun on the recorded inference | | `feedback` (scoring: judges, their prompts and sampling, services) | `feedback` and `report` rerun on the recorded `postprocess` | | `report` (metric aggregation) | `report.json` is recomputed from the recorded samples: no engine starts, no judge is called | @@ -379,6 +379,31 @@ entry records its own `serving` (`engine`, `engine_version`, `launch`: `in_run` or `external`, and `recipe`: the plan without checkpoint or host), so a reader sees what changed. +What the served model computes with does count. These engine parameters +change the logits, so they are `identity.inference.numerics`, compared on +resume; every other engine parameter is serving +(`sieval.core.tasks.run_meta.NUMERICS_PARAMS` is the list): + +| `numerics` key | Engine parameters | Unset means | +| --- | --- | --- | +| `dtype` | `dtype` | The checkpoint's `torch_dtype` (`auto` too) | +| `quantization` | `quantization` | The checkpoint's `quantization_config.quant_method`, else unknown | +| `kv_cache_dtype` | `kv_cache_dtype` | `auto` | +| `model_overrides` | `json_model_override_args` (SGLang), `hf_overrides` (vLLM) | None | + +Each is compared by the value the engine uses, so stating a default +(`dtype: auto`, `kv_cache_dtype: auto`, `bf16` for `bfloat16`) is no change. +Out of it, among others: tp, dp, ep, pp, replicas, prefill/decode roles, memory +fraction, chunked prefill, CUDA graphs, scheduler and port settings. Roles that +disagree on a key record each value. + +A key unknown on either side is no difference: an endpoint's numerics are +what its root model states under `serving.numerics`, and a key it does not +state is unknown. A `meta.json` written before numerics counted (identity +schema 4) has its numerics read from the plan of its latest run in +`runs[].serving`, unset values resolved against this run's checkpoint, so it +resumes against a run with the same effective values. + The context limit does count, since it decides which prompts overflow. A launched engine's is the plan's `max_model_len` (vLLM) or `context_length` (SGLang), the smallest across roles; a plan with a role that pins neither @@ -396,6 +421,7 @@ models: engine_version: 0.5.4 topology: {prefill: 1, decode: 3} hardware: 4x H200 + numerics: {dtype: bfloat16, kv_cache_dtype: fp8_e4m3} # compared ``` - `weights`: a resume compares the model under test by it when both runs @@ -404,7 +430,7 @@ models: - `serving`: `engine_version`, `topology` and `hardware`, any value; never compared. Refused on a model `sieval run` launches, which records its own. So a single-node run can resume on a prefill/decode deployment of the same - weights. + weights. `numerics` takes the keys of the table above and is compared. None of the three is allowed on a derived model. @@ -514,7 +540,7 @@ in `SIEVAL_TARGET_GPU` ([Resolving for declared GPUs](infer.md#resolving-for-dec "samples": 3, "resume_plan": {"kept": 2, "rereported": 0, "regraded": 0, "repostprocessed": 0, "restarted": 0, "dropped": 0}, "stages": { - "identity_schema": 4, + "identity_schema": 5, "stages": {"data": "differs", "preprocess": "same", "inference": "same", "postprocess": "same", "feedback": "same", "report": "same"}, "rerun_from": null, "refused": null, diff --git a/sieval/cli/session.py b/sieval/cli/session.py index d7c27f83..748fa1f2 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -149,6 +149,7 @@ from sieval.core.utils.paths import resolve_data_dir from sieval.datasets.downloaders.staged import probe_source, staged_dir from sieval.infer import deployment_plan_projection +from sieval.infer.introspect import checkpoint_numerics from sieval.infer.params import merge_params from sieval.infer.topology.models import DETERMINISTIC_DEFAULT_SEED @@ -4880,6 +4881,9 @@ def _setup_tasks(self) -> None: } or None, stated_serving=self._stated_serving_for(candidate.config_name), + checkpoint_numerics=self._checkpoint_numerics_for( + candidate.config_name + ), resume_config_diff=list(self._resume_config_diff), legacy_match=self._legacy_match_for(task_name, runner_config), unserved_candidate=( @@ -5087,6 +5091,20 @@ def _judge_weights_for(self, task_name: str) -> dict[str, str] | None: out[role] = weights return out or None + def _checkpoint_numerics_for(self, model_name: str) -> dict[str, JSONValue] | None: + """The ``dtype`` and ``quantization`` of the checkpoint *model_name*'s + root launches, which its engine uses where the plan leaves them unset; + ``None`` for a model SiEval does not serve or an unreadable checkpoint. + """ + plan = (self._infer_plans or {}).get(self._root_model(model_name)) + if plan is None: + return None + checkpoint = plan.get("checkpoint") + if not isinstance(checkpoint, str): + return None + numerics = checkpoint_numerics(checkpoint) + return dict(numerics) if numerics is not None else None + def _stated_serving_for(self, model_name: str) -> dict[str, JSONValue] | None: """The root model's ``serving``; ``None`` when absent or empty.""" root = self._root_model(model_name) diff --git a/sieval/cli/validation.py b/sieval/cli/validation.py index c3a69deb..648748ad 100644 --- a/sieval/cli/validation.py +++ b/sieval/cli/validation.py @@ -56,7 +56,7 @@ from sieval.core.runners.barriers import check_barrier_stage from sieval.core.tasks.consts import TaskAction from sieval.core.tasks.python_requires import MissingPythonRequirementsError -from sieval.core.tasks.run_meta import STATED_SERVING_KEYS +from sieval.core.tasks.run_meta import NUMERICS_PARAMS, STATED_SERVING_KEYS # Known top-level keys from RootConfigDict _ROOT_KEYS: set[str] = set(RootConfigDict.__annotations__) @@ -166,6 +166,16 @@ def _check_stated_serving( version = stated.get("engine_version") if version is not None and not _is_nonempty_str(version): result.errors.append(f"{where}.engine_version must be a non-empty string") + numerics = stated.get("numerics") + if numerics is not None and not isinstance(numerics, Mapping): + result.errors.append(f"{where}.numerics must be a mapping") + elif numerics is not None: + unknown = sorted(set(numerics) - set(NUMERICS_PARAMS)) + if unknown: + result.errors.append( + f"{where}.numerics has unknown key(s) {', '.join(unknown)}; " + f"expected {', '.join(NUMERICS_PARAMS)}" + ) def _validate_models(cfg: dict, result: ValidationResult) -> None: diff --git a/sieval/core/runners/multi_runner.py b/sieval/core/runners/multi_runner.py index 8d3d6f0f..23300bd2 100644 --- a/sieval/core/runners/multi_runner.py +++ b/sieval/core/runners/multi_runner.py @@ -138,6 +138,7 @@ def add_task( judge_weights: dict[str, str] | None = None, lifecycles: dict[str, dict[str, str]] | None = None, stated_serving: dict[str, JSONValue] | None = None, + checkpoint_numerics: dict[str, JSONValue] | None = None, resume_config_diff: list[str] | None = None, legacy_match: LegacyStageMatch | None = None, unserved_candidate: bool = False, @@ -147,9 +148,9 @@ def add_task( """Register a task for parallel execution. *inference_recipe*, *context_limit*, *weights*, *judge_weights*, - *lifecycles*, *stated_serving*, *serving_group*, *resume_config_diff*, - *legacy_match*, *unserved_candidate* and *engine_guard* pass through to - :class:`TaskRunner`. + *lifecycles*, *stated_serving*, *checkpoint_numerics*, *serving_group*, + *resume_config_diff*, *legacy_match*, *unserved_candidate* and + *engine_guard* pass through to :class:`TaskRunner`. Auto-sets ``result_dir`` to ``/`` when a base was provided and the task's config lacks its own. @@ -193,6 +194,7 @@ def add_task( judge_weights=judge_weights, lifecycles=lifecycles, stated_serving=stated_serving, + checkpoint_numerics=checkpoint_numerics, resume_config_diff=resume_config_diff, legacy_match=legacy_match, unserved_candidate=unserved_candidate, diff --git a/sieval/core/runners/runner.py b/sieval/core/runners/runner.py index efd1921e..93c961fd 100644 --- a/sieval/core/runners/runner.py +++ b/sieval/core/runners/runner.py @@ -76,6 +76,7 @@ GradingIdentity, InferenceIdentity, JudgeIdentity, + Numerics, PreprocessIdentity, ResumeOutcomes, RunFrom, @@ -94,9 +95,12 @@ judge_identities, legacy_grader_keys, legacy_inference_digest, + numerics_differences, preprocess_key, prompt_of, + read_recorded_numerics, request_digest, + served_numerics, serving_fingerprint, stage_code, stage_key, @@ -522,6 +526,7 @@ def __init__( judge_weights: dict[str, str] | None = None, lifecycles: dict[str, dict[str, str]] | None = None, stated_serving: dict[str, JSONValue] | None = None, + checkpoint_numerics: dict[str, JSONValue] | None = None, resume_config_diff: list[str] | None = None, legacy_match: LegacyStageMatch | None = None, unserved_candidate: bool = False, @@ -544,7 +549,9 @@ def __init__( that needs a lost engine (:mod:`.engine_loss`). *judge_weights* names, by judge role, the weights a launched judge serves, compared instead of its name. *lifecycles* are the launched models' lifecycles, recorded in - ``runs[].serving``. + ``runs[].serving``. *checkpoint_numerics* is the launched checkpoint's + own ``dtype`` and ``quantization``, which the inference identity's + ``numerics`` resolve unset engine params to (:func:`served_numerics`). """ # A malformed judge reference would leave every grader identity # unknown; refuse it before any work. @@ -557,6 +564,7 @@ def __init__( self._judge_weights = dict(judge_weights or {}) self._lifecycles = lifecycles self._stated_serving = stated_serving + self._checkpoint_numerics = checkpoint_numerics self._resume_config_diff = list(resume_config_diff or []) self._legacy_match = LegacyStageMatch(**(legacy_match or {})) self._unserved_candidate = unserved_candidate @@ -2253,11 +2261,15 @@ def _check_inference(self, current: InferenceIdentity | None) -> None: if self._inference_matches(current): return recorded = identity["inference"].get("digest") - reason = ( - "this run's inference identity is unknown" - if current is None - else f"recorded {recorded}, now {current['digest']}" + numerics = numerics_differences( + self._recorded_numerics(), (current or {}).get("numerics") ) + if current is None: + reason = "this run's inference identity is unknown" + elif recorded == current["digest"] and numerics: + reason = "the served model's numerics: " + "; ".join(numerics) + else: + reason = f"recorded {recorded}, now {current['digest']}" elif identity is None and ( self._trusts_unrecorded() or self._legacy_match.get("inference") is True ): @@ -2272,7 +2284,9 @@ def _check_inference(self, current: InferenceIdentity | None) -> None: f" recorded: {self._recorded_dir()}\n" " Inference is reused, never rerun, by a resume. Start fresh in a " "new result_dir, or match the model's request arguments, the " - "task's inference arguments and the context limit.", + "task's inference arguments, the context limit and the served " + "model's numerics (dtype, quantization, kv_cache_dtype, model " + "overrides).", task=self._task.name, ) @@ -2368,6 +2382,7 @@ def _stage_plan(self) -> ResumePlan: current, self._legacy_grader_keys(), self._legacy_inference_digest(), + self._checkpoint_numerics, ) if plan["legacy_preprocess"] and self._legacy_prompts_match is not None: plan["stages"]["preprocess"] = ( @@ -2832,6 +2847,11 @@ def _inference_identity(self) -> InferenceIdentity | None: task_args=self._task.inference_args(), runner_keys=self._runner_keys(), context_limit=self._context_limit, + numerics=served_numerics( + self._inference_recipe, + self._stated_serving, + self._checkpoint_numerics, + ), ) except (TypeError, ValueError, AttributeError) as exc: logger.warning("Inference identity unavailable: {}", exc) @@ -2862,8 +2882,14 @@ def _inference_matches(self, current: InferenceIdentity | None) -> bool | None: (self._recorded or {}).get("identity"), current["digest"] if current is not None else None, self._legacy_inference_digest(), + numerics=current.get("numerics") if current is not None else None, + recorded_numerics=self._recorded_numerics(), ) + def _recorded_numerics(self) -> Numerics | None: + """The recorded run's :func:`read_recorded_numerics`.""" + return read_recorded_numerics(self._recorded, self._checkpoint_numerics) + def _serving(self) -> ServingFingerprint | None: try: return serving_fingerprint( @@ -3113,7 +3139,7 @@ async def _end_run( if inference is not None: await self._saver.set_identity( inference={ - "digest": inference["digest"], + **inference, "requests": self._requests_identity(), } ) diff --git a/sieval/core/runners/stage_resume.py b/sieval/core/runners/stage_resume.py index 485e25c8..1c136c05 100644 --- a/sieval/core/runners/stage_resume.py +++ b/sieval/core/runners/stage_resume.py @@ -63,6 +63,8 @@ preprocess_key, prompt_of, read_identity, + read_recorded_numerics, + served_numerics, split_digests, stage_code, stage_key, @@ -194,6 +196,7 @@ def plan_resume( current: Mapping[str, Any], legacy_grader_keys: Iterable[str] = (), legacy_inference: str | None = None, + checkpoint_numerics: Mapping[str, JSONValue] | None = None, ) -> ResumePlan: """Compare a recorded ``meta.json`` with this run's identities, running nothing. @@ -203,7 +206,10 @@ def plan_resume( :func:`~sieval.core.tasks.run_meta.legacy_grader_keys` for this task, read only for a schema-1 directory, and *legacy_inference* :func:`~sieval.core.tasks.run_meta.legacy_inference_digest`, only for one - before identity schema 4. The runner plans per sample and is + before identity schema 4. *checkpoint_numerics* resolves the unset + numerics of a schema-4 directory's plan + (:func:`~sieval.core.tasks.run_meta.read_recorded_numerics`). The runner + plans per sample and is authoritative; this says what a resume of a finished directory does. """ block = (recorded_meta or {}).get("identity") @@ -221,6 +227,8 @@ def plan_resume( block if isinstance(block, Mapping) else None, mine.get("digest") if isinstance(mine, Mapping) else None, legacy_inference, + numerics=mine.get("numerics") if isinstance(mine, Mapping) else None, + recorded_numerics=read_recorded_numerics(recorded_meta, checkpoint_numerics), ) stages["inference"] = "unknown" if match is None else _verdict(match) stages = {stage: stages[stage] for stage in STAGES} @@ -285,6 +293,8 @@ def task_resume_plan( runner_keys: Mapping[str, JSONValue], context_limit: int | None = None, recipe: Mapping[str, JSONValue] | None = None, + stated_serving: Mapping[str, JSONValue] | None = None, + checkpoint_numerics: Mapping[str, JSONValue] | None = None, ) -> ResumePlan: """:func:`plan_resume` for *task* against a recorded ``meta.json``. @@ -292,8 +302,11 @@ def task_resume_plan( :class:`~sieval.core.runners.runner.TaskRunner` does, so call it after ``task.setup()``, which may resolve an argument the grading identities read. *runner_keys* holds :data:`~sieval.core.tasks.run_meta.INFERENCE_RUNNER_KEYS` - and *context_limit* and *recipe* as the runner is given them; *recipe* is - read only for a directory before identity schema 4. Reads files, + and *context_limit*, *recipe*, *stated_serving* and + *checkpoint_numerics* as the runner is given them; they decide the + served numerics (:func:`~sieval.core.tasks.run_meta.served_numerics`), + and *recipe* is compared whole for a directory before identity schema 4. + Reads files, starts nothing and calls no model. """ task_cls = type(task) @@ -311,6 +324,7 @@ def task_resume_plan( task_args=task.inference_args(), runner_keys=runner_keys, context_limit=context_limit, + numerics=served_numerics(recipe, stated_serving, checkpoint_numerics), ), **grading, } @@ -324,6 +338,7 @@ def task_resume_plan( runner_keys=runner_keys, recipe=recipe, ), + checkpoint_numerics, ) diff --git a/sieval/core/tasks/run_meta.py b/sieval/core/tasks/run_meta.py index 798c008b..0f4caad1 100644 --- a/sieval/core/tasks/run_meta.py +++ b/sieval/core/tasks/run_meta.py @@ -38,6 +38,15 @@ compared apart from ``inference``: by ``weights`` when both runs state them, else by ``model``. +What the served model computes with is not how it is served: ``inference`` +keeps ``numerics`` (:func:`served_numerics`), the effective ``dtype``, +runtime ``quantization``, ``kv_cache_dtype`` and model overrides, by the +engine parameters :data:`NUMERICS_PARAMS` lists. They sit beside the digest, +not in it, because a key may be unknown: for an endpoint, one the caller does +not state. A key known on both sides must be equal; one unknown on either +side is no difference. A schema-4 block recorded none: its numerics are read +from its plan in ``runs[].serving`` (:func:`read_recorded_numerics`). + ``preprocess`` is code, not the prompts it renders: a prompt is code, arguments, data and the model's ``extra``, and ``extra`` reaches the request, which ``inference`` digests per sample. So a wrapping model and a plain one @@ -95,8 +104,8 @@ #: Version of the ``identity`` block's layout. A digest computed differently is #: not a layout change: it reruns one stage, a new schema would discard them all. #: Schema 1 is still read (:func:`read_identity`). Schema 4 took serving out of -#: ``inference``. -IDENTITY_SCHEMA = 4 +#: ``inference``; schema 5 put the served model's numerics back in. +IDENTITY_SCHEMA = 5 #: Identity schemas whose ``inference`` digest covers the engine and serving #: recipe (:func:`legacy_inference_digest`). @@ -174,11 +183,19 @@ class DigestedStage(TypedDict): unknown: NotRequired[int] +#: What the served model computes with, by ``inference.numerics`` key: one +#: value each, keyed as :data:`NUMERICS_PARAMS` lists them. +type Numerics = dict[str, JSONValue] + + class InferenceIdentity(TypedDict): - """``digest`` covers what is asked; ``requests`` what was sent.""" + """``digest`` covers what is asked; ``requests`` what was sent; + ``numerics`` what the served model computes with (:func:`served_numerics`), + absent when none of it is known.""" digest: str requests: NotRequired[DigestedStage] + numerics: NotRequired[Numerics] class ServingFingerprint(TypedDict): @@ -202,7 +219,27 @@ class ServingFingerprint(TypedDict): #: What a caller may state about how an external endpoint serves the model. -STATED_SERVING_KEYS: tuple[str, ...] = ("engine_version", "topology", "hardware") +#: ``numerics`` alone is compared, as ``inference.numerics``. +STATED_SERVING_KEYS: tuple[str, ...] = ( + "engine_version", + "topology", + "hardware", + "numerics", +) + +#: The engine parameters that change what the served model computes, by the +#: ``inference.numerics`` key they set. Every other engine parameter is how +#: the model is served (parallelism, replicas, disaggregation, memory, +#: scheduling, ports): recorded in ``runs[].serving``, never compared. Names +#: are in underscore form; :func:`numerics_key` reads either form. +NUMERICS_PARAMS: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "dtype": ("dtype",), + "quantization": ("quantization",), + "kv_cache_dtype": ("kv_cache_dtype",), + "model_overrides": ("json_model_override_args", "hf_overrides"), + } +) class JudgeIdentity(TypedDict): @@ -278,8 +315,10 @@ class RunIdentity(TypedDict): Two runs share a stage only when every digest in it matches: for ``inference`` that is ``digest`` *and* ``requests.digest``, and a ``None`` - digest on either side matches nothing. A schema-1 or schema-2 - ``preprocess`` is a :class:`DigestedStage` of rendered prompts. + digest on either side matches nothing. ``inference.numerics`` must not + differ on a key both sides know (:func:`numerics_differences`). A + schema-1 or schema-2 ``preprocess`` is a :class:`DigestedStage` of rendered + prompts. """ schema: int @@ -587,11 +626,14 @@ def inference_identity( task_args: Mapping[str, JSONValue], runner_keys: Mapping[str, JSONValue], context_limit: int | None, + numerics: Numerics | None = None, ) -> InferenceIdentity: """Digest what the candidate is asked, leaving out which model and how served. *context_limit* is its context window in tokens, ``None`` when unknown. - What was actually sent is folded per sample into ``requests`` at the end. + *numerics* is :func:`served_numerics`, kept beside the digest as values, + since a key may be unknown. What was actually sent is folded per sample + into ``requests`` at the end. """ payload = { "model_args": model_request_args(model), @@ -599,7 +641,10 @@ def inference_identity( "runner": {key: runner_keys.get(key) for key in INFERENCE_RUNNER_KEYS}, "context_limit": context_limit, } - return {"digest": digest(payload)} + identity: InferenceIdentity = {"digest": digest(payload)} + if numerics: + identity["numerics"] = dict(numerics) + return identity def legacy_inference_digest( @@ -627,12 +672,17 @@ def inference_matches( recorded: Mapping[str, Any] | None, current: str | None, legacy: str | None = None, + *, + numerics: Numerics | None = None, + recorded_numerics: Numerics | None = None, ) -> bool | None: """Whether a recorded ``identity`` block's ``inference`` is this run's. *legacy* is this run's :func:`legacy_inference_digest`, used for a block - of a :data:`LEGACY_INFERENCE_SCHEMAS` schema. ``None`` when either side is - unknown. + of a :data:`LEGACY_INFERENCE_SCHEMAS` schema. ``None`` when either digest + is unknown. Equal digests still differ when *numerics*, this run's + :func:`served_numerics`, and *recorded_numerics* + (:func:`read_recorded_numerics`) differ (:func:`numerics_differences`). """ if not isinstance(recorded, Mapping): return None @@ -641,7 +691,172 @@ def inference_matches( mine = legacy if recorded.get("schema") in LEGACY_INFERENCE_SCHEMAS else current if not isinstance(theirs, str) or mine is None: return None - return theirs == mine + return theirs == mine and not numerics_differences(recorded_numerics, numerics) + + +def numerics_key(param: str) -> str | None: + """The ``inference.numerics`` key engine parameter *param* sets, in dash or + underscore form; ``None`` for a parameter that only says how the model is + served (:data:`NUMERICS_PARAMS`).""" + name = param.replace("-", "_") + return next((k for k, names in NUMERICS_PARAMS.items() if name in names), None) + + +#: Spellings the engines accept for one ``dtype``. +_DTYPE_ALIASES: Mapping[str, str] = MappingProxyType( + { + "bf16": "bfloat16", + "half": "float16", + "fp16": "float16", + "float": "float32", + "fp32": "float32", + } +) + +#: Spellings the engines accept for one ``kv_cache_dtype``. +_KV_CACHE_DTYPE_ALIASES: Mapping[str, str] = MappingProxyType({"fp8": "fp8_e4m3"}) + + +def _numerics_value(key: str, value: object) -> JSONValue: + """*value* of numerics *key*, spelled one way: an alias by its name, model + overrides as an object (an empty one is none).""" + if key == "dtype" and isinstance(value, str): + return _DTYPE_ALIASES.get(value, value) + if key == "kv_cache_dtype" and isinstance(value, str): + return _KV_CACHE_DTYPE_ALIASES.get(value, value) + if key == "model_overrides" and isinstance(value, str): + try: + value = json.loads(value) + except ValueError: + return value + if key == "model_overrides" and not value: + return None + return cast(JSONValue, value) + + +def _role_numerics( + params: Mapping[str, Any], checkpoint: Mapping[str, JSONValue] | None +) -> Numerics: + """One role's effective numerics: what its engine *params* state, else the + engine's default. An unset or ``auto`` ``dtype`` and an unset + ``quantization`` are the *checkpoint*'s, unknown without it.""" + stated: dict[str, object] = {} + for param, value in params.items(): + key = numerics_key(param) + if key is not None and value is not None and key not in stated: + stated[key] = value + out: Numerics = {} + dtype = stated.get("dtype") + if dtype is not None and dtype != "auto": + out["dtype"] = _numerics_value("dtype", dtype) + elif checkpoint is not None and "dtype" in checkpoint: + out["dtype"] = _numerics_value("dtype", checkpoint["dtype"]) + if "quantization" in stated: + out["quantization"] = _numerics_value("quantization", stated["quantization"]) + elif checkpoint is not None and "quantization" in checkpoint: + out["quantization"] = checkpoint["quantization"] + out["kv_cache_dtype"] = _numerics_value( + "kv_cache_dtype", stated.get("kv_cache_dtype", "auto") + ) + out["model_overrides"] = _numerics_value( + "model_overrides", stated.get("model_overrides") + ) + return out + + +def served_numerics( + recipe: Mapping[str, Any] | None, + stated: Mapping[str, Any] | None = None, + checkpoint: Mapping[str, JSONValue] | None = None, +) -> Numerics | None: + """What the served model computes with, keyed by :data:`NUMERICS_PARAMS`. + + For an engine the run launched, the effective value of each key from the + ``engine_params`` of every role in *recipe*; *checkpoint* holds the + checkpoint's own ``dtype`` and ``quantization`` (``None`` when it has + none), which an unset one resolves to. Roles that disagree on a key give + their distinct values, sorted. For an endpoint (no *recipe*), the + ``numerics`` the caller *stated*, as given. A key left out is unknown, and + ``None`` is returned when every key is. + """ + if recipe is None: + given = (stated or {}).get("numerics") + if not isinstance(given, Mapping) or not given: + return None + return { + key: _numerics_value(key, given[key]) + for key in NUMERICS_PARAMS + if key in given and not (key == "dtype" and given[key] == "auto") + } or None + roles = [ + _role_numerics(a.get("engine_params") or {}, checkpoint) + for a in recipe.get("assignments") or () + if isinstance(a, Mapping) + ] + if not roles: + return None + out: Numerics = {} + for key in NUMERICS_PARAMS: + if not all(key in role for role in roles): + continue + values = {canonical_json(role[key]): role[key] for role in roles} + out[key] = ( + next(iter(values.values())) + if len(values) == 1 + else [values[text] for text in sorted(values)] + ) + return out or None + + +def read_recorded_numerics( + meta: Mapping[str, Any] | None, + checkpoint: Mapping[str, JSONValue] | None = None, +) -> Numerics | None: + """The numerics a recorded ``meta.json``'s inference ran with, ``None`` + when unknown. + + This schema records them in ``inference.numerics``. Schema 4 recorded none: + they are read from the plan of its latest run with this model, in + ``runs[].serving``, as :func:`served_numerics` would have, its unset values + resolved against this run's *checkpoint*. Schemas 1 to 3 compare the plan + whole (:func:`legacy_inference_digest`), so they need none. + """ + block = (meta or {}).get("identity") + if not isinstance(block, Mapping): + return None + schema = block.get("schema") + if schema == IDENTITY_SCHEMA: + stage = block.get("inference") + numerics = stage.get("numerics") if isinstance(stage, Mapping) else None + return dict(numerics) if isinstance(numerics, Mapping) else None + if schema != 4: + return None + runs = cast(Mapping[str, Any], meta).get("runs") + for run in reversed(runs if isinstance(runs, list) else []): + if not isinstance(run, Mapping) or "model" in run: + continue # another model's run did not infer these samples + serving = run.get("serving") + if isinstance(serving, Mapping): + recipe = serving.get("recipe") + return served_numerics( + recipe if isinstance(recipe, Mapping) else None, None, checkpoint + ) + return None + + +def numerics_differences( + recorded: Mapping[str, Any] | None, current: Mapping[str, Any] | None +) -> list[str]: + """Each numerics key known on both sides whose values differ, as + ``key: recorded -> current``. A key unknown on either side is no + difference.""" + if recorded is None or current is None: + return [] + return [ + f"{key}: {recorded[key]!r} -> {current[key]!r}" + for key in NUMERICS_PARAMS + if key in recorded and key in current and recorded[key] != current[key] + ] def serving_fingerprint( @@ -2402,7 +2617,7 @@ def read_identity( ) -> tuple[RunIdentity, Mapping[str, Any] | None] | None: """A recorded ``identity`` block as this schema, with a schema-1 ``grader``. - Schemas 2 to 4 are read as written; schema 2's ``preprocess`` is a digest + Schemas 2 to 5 are read as written; schema 2's ``preprocess`` is a digest of rendered prompts, which :func:`is_code_identity` tells apart. Schema 1 maps ``data`` and ``inference`` across and ``prompt`` to ``preprocess`` (:data:`SCHEMA_1_STAGES`); its ``grader`` cannot be split, so the grading @@ -2415,7 +2630,7 @@ def read_identity( block = cast(Mapping[str, Any], block) out = empty_identity() schema = block.get("schema") - if schema in (2, 3, IDENTITY_SCHEMA): + if schema in (2, 3, 4, IDENTITY_SCHEMA): for stage in STAGES: if isinstance(block.get(stage), dict): out[stage] = block[stage] # type: ignore[literal-required] diff --git a/sieval/infer/introspect.py b/sieval/infer/introspect.py index daea4d62..ea6da92d 100644 --- a/sieval/infer/introspect.py +++ b/sieval/infer/introspect.py @@ -422,6 +422,30 @@ def native_context_window(checkpoint: str) -> int | None: return _positive_config_int(checkpoint, "max_position_embeddings") +def checkpoint_numerics(checkpoint: str) -> dict[str, str] | None: + """The ``dtype`` and ``quantization`` an engine serving *checkpoint* uses + when its params leave them unset. + + ``dtype`` is ``torch_dtype`` (``dtype`` in newer configs) from + ``config.json`` or its ``text_config``, ``quantization`` the + ``quantization_config.quant_method``; each absent when unstated. ``None`` + when ``config.json`` cannot be read. + """ + sources = _text_configs(checkpoint) + if not sources: + return None + out: dict[str, str] = {} + for source in sources: + for key in ("torch_dtype", "dtype"): + value = source.get(key) + if isinstance(value, str) and "dtype" not in out: + out["dtype"] = value + quantization = _extract_quantization(source) + if quantization is not None and "quantization" not in out: + out["quantization"] = quantization.quant_method + return out + + def logit_shape(checkpoint: str) -> tuple[int, float] | None: """*checkpoint*'s output vocabulary and the bytes of one logit in its dtype. diff --git a/tests/integration/resume/test_across_releases.py b/tests/integration/resume/test_across_releases.py index b159be60..d96bdf5c 100644 --- a/tests/integration/resume/test_across_releases.py +++ b/tests/integration/resume/test_across_releases.py @@ -229,7 +229,7 @@ def test_an_unchanged_task_keeps_every_verdict_and_upgrades_the_identity( assert resumed["resume"]["repostprocessed"] == 0 assert resumed["resume"]["restarted"] == 0 identity = meta["identity"] - assert identity["schema"] == IDENTITY_SCHEMA == 4 + assert identity["schema"] == IDENTITY_SCHEMA == 5 assert "grader" not in identity and "prompt" not in identity old = recorded["identity"] assert identity["data"] == old["data"] diff --git a/tests/unit/cli/test_session.py b/tests/unit/cli/test_session.py index af470a95..849c49ed 100644 --- a/tests/unit/cli/test_session.py +++ b/tests/unit/cli/test_session.py @@ -8221,6 +8221,23 @@ def test_stated_serving_is_the_root_models(self): runner.config["models"] = cast(Any, {"m": {"api_base": "http://x/v1"}}) assert runner._stated_serving_for("m") is None + def test_checkpoint_numerics_are_the_launched_checkpoints(self, tmp_path): + """Read from the root's plan checkpoint; none for an endpoint or an + unreadable checkpoint.""" + (tmp_path / "config.json").write_text( + json.dumps({"torch_dtype": "bfloat16", "architectures": ["X"]}) + ) + runner = self._make_runner({}, models={"m": MockChatModel()}) + runner.config["models"] = cast( + Any, {"base": {"path": str(tmp_path)}, "m": {"base": "base"}} + ) + runner._infer_plans = {"base": {**self._plan(), "checkpoint": str(tmp_path)}} + assert runner._checkpoint_numerics_for("m") == {"dtype": "bfloat16"} + runner._infer_plans = {"base": {**self._plan(), "checkpoint": "/nowhere"}} + assert runner._checkpoint_numerics_for("m") is None + runner._infer_plans = None + assert runner._checkpoint_numerics_for("m") is None + def test_weights_are_the_root_models(self): runner = self._make_runner({}, models={"m": MockChatModel()}) runner.config["models"] = cast( diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index 3dc3af6a..2dccc280 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -311,6 +311,16 @@ def errors(model): "engine_version must be" in e for e in errors({"name": "x", "serving": {"engine_version": 5}}) ) + numerics = {"dtype": "bfloat16", "kv_cache_dtype": "fp8_e5m2"} + assert errors({"name": "x", "serving": {"numerics": numerics}}) == [] + assert any( + "numerics must be a mapping" in e + for e in errors({"name": "x", "serving": {"numerics": "bf16"}}) + ) + assert any( + "numerics has unknown key(s) tp" in e + for e in errors({"name": "x", "serving": {"numerics": {"tp": 2}}}) + ) # An engine the run launches records its own serving. assert any( "describes an external endpoint" in e diff --git a/tests/unit/core/runners/test_runner_run_meta.py b/tests/unit/core/runners/test_runner_run_meta.py index 0fab144a..8d4f5433 100644 --- a/tests/unit/core/runners/test_runner_run_meta.py +++ b/tests/unit/core/runners/test_runner_run_meta.py @@ -337,14 +337,21 @@ async def test_n_counts(self, tmp_path): @pytest.mark.anyio async def test_the_recipe_is_recorded_not_digested(self, tmp_path): - """The serving recipe is the run's serving fingerprint, not identity.""" + """The serving recipe is the run's serving fingerprint; only the + numerics it implies join the identity, beside the digest.""" task = RecordTask(dataset=MockDataset(SAMPLES), model=MockChatModel(), name="t") recipe = {"backend": "vllm", "assignments": [{"role": "full"}]} runner = TaskRunner(task, make_config(tmp_path / "a"), inference_recipe=recipe) await runner.arun() meta = _meta(runner.root_dir) plain = _meta(await self._root(tmp_path / "b")) - assert meta["identity"]["inference"] == plain["identity"]["inference"] + inference = meta["identity"]["inference"] + assert inference.pop("numerics") == { + "kv_cache_dtype": "auto", + "model_overrides": None, + } + assert inference == plain["identity"]["inference"] + assert "numerics" not in plain["identity"]["inference"] assert meta["runs"][-1]["serving"]["recipe"] == recipe @pytest.mark.anyio @@ -526,6 +533,151 @@ async def test_another_context_limit_is_refused(self, tmp_path): assert (root / "meta.json").read_bytes() == before +def _engine(**params) -> dict: + return { + "backend": "sglang", + "assignments": [{"role": "full", "engine_params": params}], + } + + +class TestServedNumericsAreIdentity: + """What the served model computes with gates a resume; how it is served + does not (issue #258).""" + + BF16 = {"dtype": "bfloat16"} + + @pytest.mark.anyio + async def test_another_kv_cache_dtype_is_refused(self, tmp_path): + task, _ = _counting_task() + await TaskRunner( + task, + make_config(tmp_path), + inference_recipe=_engine(tp=1), + checkpoint_numerics=self.BF16, + ).arun() + root = tmp_path / "results" + _forget_last_sample(root) + before = (root / "meta.json").read_bytes() + + task, model = _counting_task() + runner = TaskRunner( + task, + make_config(tmp_path, resume=True), + inference_recipe=_engine(tp=1, kv_cache_dtype="fp8_e5m2"), + checkpoint_numerics=self.BF16, + ) + with pytest.raises(StageResumeError, match="kv_cache_dtype: 'auto'"): + await runner.arun() + assert model.call_count == 0 + assert (root / "meta.json").read_bytes() == before + + @pytest.mark.anyio + async def test_a_stated_default_and_another_layout_resume(self, tmp_path): + task, _ = _counting_task() + await TaskRunner( + task, + make_config(tmp_path), + inference_recipe=_engine(tp=1), + checkpoint_numerics=self.BF16, + ).arun() + root = tmp_path / "results" + _forget_last_sample(root) + + task, model = _counting_task() + await TaskRunner( + task, + make_config(tmp_path, resume=True), + inference_recipe=_engine(tp=8, dtype="bf16", kv_cache_dtype="auto"), + checkpoint_numerics=self.BF16, + ).arun() + assert model.call_count == 1 + inference = _meta(root)["identity"]["inference"] + assert inference["numerics"]["dtype"] == "bfloat16" + assert inference["requests"]["digest"] is not None + + @pytest.mark.anyio + async def test_a_schema_4_directory_reads_its_plan_as_effective(self, tmp_path): + """Recorded before numerics counted: its plan in `runs[].serving` says + what it ran with, so a run with the same effective values resumes and + one with another dtype is refused.""" + task, _ = _counting_task() + await TaskRunner( + task, make_config(tmp_path), inference_recipe=_engine(tp=1) + ).arun() + root = tmp_path / "results" + meta = _meta(root) + meta["identity"]["schema"] = 4 + meta["identity"]["inference"].pop("numerics") + (root / "meta.json").write_bytes(orjson.dumps(meta)) + _forget_last_sample(root) + + task, model = _counting_task() + with pytest.raises(StageResumeError, match="dtype: 'bfloat16' -> 'float16'"): + await TaskRunner( + task, + make_config(tmp_path, resume=True), + inference_recipe=_engine(tp=2, dtype="float16"), + checkpoint_numerics=self.BF16, + ).arun() + assert model.call_count == 0 + + await TaskRunner( + task, + make_config(tmp_path, resume=True), + inference_recipe=_engine(tp=2, dtype="bfloat16"), + checkpoint_numerics=self.BF16, + ).arun() + assert model.call_count == 1 + identity = _meta(root)["identity"] + assert identity["schema"] == IDENTITY_SCHEMA + assert identity["inference"]["numerics"]["dtype"] == "bfloat16" + + @pytest.mark.anyio + async def test_an_endpoint_that_states_none_is_not_a_mismatch(self, tmp_path): + task, _ = _counting_task() + await TaskRunner( + task, + make_config(tmp_path), + inference_recipe=_engine(tp=1), + checkpoint_numerics=self.BF16, + ).arun() + root = tmp_path / "results" + _forget_last_sample(root) + + task, model = _counting_task() + await TaskRunner(task, make_config(tmp_path, resume=True)).arun() + assert model.call_count == 1 + + @pytest.mark.anyio + async def test_an_endpoint_stating_other_numerics_is_refused(self, tmp_path): + task, _ = _counting_task() + await TaskRunner( + task, + make_config(tmp_path), + inference_recipe=_engine(tp=1), + checkpoint_numerics=self.BF16, + ).arun() + root = tmp_path / "results" + _forget_last_sample(root) + + task, model = _counting_task() + with pytest.raises(StageResumeError, match="dtype: 'bfloat16' -> 'float16'"): + await TaskRunner( + task, + make_config(tmp_path, resume=True), + stated_serving={"numerics": {"dtype": "fp16"}}, + ).arun() + assert model.call_count == 0 + # The recorded run's quantization is unknown (an unquantized + # checkpoint), so stating one is not a difference. + await TaskRunner( + task, + make_config(tmp_path, resume=True), + stated_serving={"numerics": {"dtype": "bf16", "quantization": "fp8"}}, + ).arun() + assert model.call_count == 1 + + class TestUnavailableFingerprints: @pytest.mark.anyio async def test_a_serving_fingerprint_that_cannot_be_read_is_left_out( diff --git a/tests/unit/core/runners/test_stage_resume.py b/tests/unit/core/runners/test_stage_resume.py index 12e0655d..d0d7904f 100644 --- a/tests/unit/core/runners/test_stage_resume.py +++ b/tests/unit/core/runners/test_stage_resume.py @@ -1375,6 +1375,28 @@ def test_plan_resume_rolls_back_to_the_earliest_changed_stage(): assert plan_resume(None, same)["identity_schema"] is None +def test_plan_resume_refuses_other_served_numerics(): + """Equal digests with another known numerics value refuse; a schema-4 + directory's are read from its plan, resolved against this checkpoint.""" + inference = {"digest": "sha256:i", "numerics": {"kv_cache_dtype": "auto"}} + recorded = {"identity": {"schema": IDENTITY_SCHEMA, "inference": inference}} + same = {"inference": inference} + fp8 = {"inference": {**inference, "numerics": {"kv_cache_dtype": "fp8_e4m3"}}} + assert plan_resume(recorded, same)["refused"] is None + assert plan_resume(recorded, fp8)["refused"] == "inference" + + schema_4 = { + "identity": {"schema": 4, "inference": {"digest": "sha256:i"}}, + "runs": [{"serving": {"recipe": {"assignments": [{"engine_params": {}}]}}}], + } + bf16 = {"inference": {"digest": "sha256:i", "numerics": {"dtype": "bfloat16"}}} + ckpt = {"dtype": "bfloat16"} + assert plan_resume(schema_4, bf16, checkpoint_numerics=ckpt)["refused"] is None + fp16 = {"dtype": "float16"} + refused = plan_resume(schema_4, bf16, checkpoint_numerics=fp16)["refused"] + assert refused == "inference" + + # =================================================================== # preprocess is code: a change restarts every sample, and a directory that # recorded rendered prompts (identity schema 1 or 2) is upgraded in place diff --git a/tests/unit/core/tasks/test_run_meta.py b/tests/unit/core/tasks/test_run_meta.py index 0c0d660f..8be7beeb 100644 --- a/tests/unit/core/tasks/test_run_meta.py +++ b/tests/unit/core/tasks/test_run_meta.py @@ -32,6 +32,7 @@ GRADING_STAGES, IDENTITY_SCHEMA, JUDGE_MODEL_REF_ENV, + NUMERICS_PARAMS, STAGES, JudgeIdentity, _file_key, @@ -56,11 +57,15 @@ legacy_inference_digest, model_request_args, new_run_record, + numerics_differences, + numerics_key, ordered_stages, prompt_of, read_identity, + read_recorded_numerics, request_definition, request_digest, + served_numerics, serving_fingerprint, stage_key, validate_judge_model_ref, @@ -434,6 +439,200 @@ def test_an_unknown_side_is_unknown(self): assert inference_matches(block, None) is None +def _plan(*roles: Mapping[str, Any]) -> dict[str, Any]: + return {"assignments": [{"role": "full", "engine_params": dict(r)} for r in roles]} + + +BF16 = {"dtype": "bfloat16"} + + +class TestServedNumerics: + def test_engine_params_split_into_numerics_and_serving(self): + """Only the computation keys count; tp, memory and the rest do not.""" + params = { + "dtype": "bfloat16", + "quantization": "fp8", + "kv-cache-dtype": "fp8_e5m2", + "tensor_parallel_size": 4, + "mem_fraction_static": 0.8, + "enable_dp_attention": True, + } + assert served_numerics(_plan(params)) == { + "dtype": "bfloat16", + "quantization": "fp8", + "kv_cache_dtype": "fp8_e5m2", + "model_overrides": None, + } + moved = {**params, "tensor_parallel_size": 8, "mem_fraction_static": 0.9} + assert served_numerics(_plan(moved)) == served_numerics(_plan(params)) + + def test_every_numerics_param_maps_to_one_key(self): + assert {p: numerics_key(p) for p in ("dtype", "hf-overrides", "tp")} == { + "dtype": "dtype", + "hf-overrides": "model_overrides", + "tp": None, + } + for key, names in NUMERICS_PARAMS.items(): + assert all(numerics_key(name) == key for name in names) + + def test_a_stated_default_equals_an_unset_one(self): + unset = served_numerics(_plan({}), checkpoint=BF16) + stated = served_numerics( + _plan({"dtype": "auto", "kv_cache_dtype": "auto"}), checkpoint=BF16 + ) + aliased = served_numerics(_plan({"dtype": "bf16"}), checkpoint=BF16) + assert unset == stated == aliased + assert unset == { + "dtype": "bfloat16", + "kv_cache_dtype": "auto", + "model_overrides": None, + } + + def test_quantization_unset_is_the_checkpoints(self): + checkpoint = {"dtype": "bfloat16", "quantization": "fp8"} + on_disk = served_numerics(_plan({}), checkpoint=checkpoint) + assert on_disk is not None and on_disk["quantization"] == "fp8" + runtime = served_numerics(_plan({"quantization": "awq"}), checkpoint=checkpoint) + assert runtime is not None and runtime["quantization"] == "awq" + # Unquantized checkpoint: no key, so unknown, not "none". + plain = served_numerics(_plan({}), checkpoint=BF16) + assert plain is not None and "quantization" not in plain + + def test_an_unreadable_checkpoint_leaves_dtype_unknown(self): + numerics = served_numerics(_plan({})) + assert numerics is not None and "dtype" not in numerics + + def test_model_overrides_compare_as_objects(self): + text = served_numerics( + _plan({"json_model_override_args": '{"b": 2, "a": 1}'}), checkpoint=BF16 + ) + obj = served_numerics( + _plan({"json_model_override_args": {"a": 1, "b": 2}}), checkpoint=BF16 + ) + assert text == obj + assert text is not None and text["model_overrides"] == {"a": 1, "b": 2} + empty = served_numerics( + _plan({"json_model_override_args": "{}"}), checkpoint=BF16 + ) + assert empty == served_numerics(_plan({}), checkpoint=BF16) + # Text that is not JSON is compared as given. + raw = served_numerics(_plan({"hf_overrides": "{bad"}), checkpoint=BF16) + assert raw is not None and raw["model_overrides"] == "{bad" + + def test_a_plan_without_roles_is_unknown(self): + assert served_numerics({"assignments": []}) is None + assert served_numerics({}) is None + + def test_roles_that_disagree_record_each_value(self): + numerics = served_numerics( + _plan({"kv_cache_dtype": "fp8_e5m2"}, {}), checkpoint=BF16 + ) + assert numerics is not None + assert numerics["kv_cache_dtype"] == ["auto", "fp8_e5m2"] + assert numerics["dtype"] == "bfloat16" + + def test_an_endpoint_has_what_the_caller_states(self): + assert served_numerics(None) is None + assert served_numerics(None, {"hardware": "8x H200"}) is None + stated = {"numerics": {"dtype": "bf16", "kv_cache_dtype": "fp8"}} + assert served_numerics(None, stated) == { + "dtype": "bfloat16", + "kv_cache_dtype": "fp8_e4m3", + } + + +class TestNumericsInTheInferenceIdentity: + KEYS = {"deterministic": False, "max_iterations": None, "max_retries": None} + + def _identity(self, numerics): + return inference_identity( + model=MockChatModel(), + task_args={}, + runner_keys=self.KEYS, + context_limit=None, + numerics=numerics, + ) + + def test_numerics_sit_beside_the_digest(self): + fp8 = self._identity({"kv_cache_dtype": "fp8_e5m2"}) + auto = self._identity({"kv_cache_dtype": "auto"}) + assert fp8["digest"] == auto["digest"] + assert fp8["numerics"] == {"kv_cache_dtype": "fp8_e5m2"} + assert "numerics" not in self._identity(None) + + def test_a_known_difference_refuses(self): + block = {"schema": IDENTITY_SCHEMA, "inference": {"digest": "sha256:a"}} + bf16, fp8 = {"dtype": "bfloat16"}, {"dtype": "float16"} + assert inference_matches( + block, "sha256:a", numerics=bf16, recorded_numerics=bf16 + ) + assert ( + inference_matches(block, "sha256:a", numerics=fp8, recorded_numerics=bf16) + is False + ) + + def test_a_key_unknown_on_either_side_is_no_difference(self): + block = {"schema": IDENTITY_SCHEMA, "inference": {"digest": "sha256:a"}} + stated = {"dtype": "bfloat16"} + launched = {"dtype": "bfloat16", "kv_cache_dtype": "auto"} + assert inference_matches( + block, "sha256:a", numerics=launched, recorded_numerics=stated + ) + assert inference_matches(block, "sha256:a", numerics=launched) + assert numerics_differences(None, launched) == [] + assert numerics_differences({"dtype": "float16"}, launched) == [ + "dtype: 'float16' -> 'bfloat16'" + ] + + +class TestReadRecordedNumerics: + def test_this_schema_reads_the_block(self): + meta = { + "identity": { + "schema": IDENTITY_SCHEMA, + "inference": {"digest": "x", "numerics": {"dtype": "float16"}}, + } + } + assert read_recorded_numerics(meta) == {"dtype": "float16"} + meta["identity"]["inference"] = {"digest": "x"} + assert read_recorded_numerics(meta) is None + + def test_schema_4_reads_its_latest_own_plan_as_effective(self): + old = _plan({"dtype": "float16"}) + latest = _plan({"tensor_parallel_size": 2}) + meta = { + "identity": {"schema": 4, "inference": {"digest": "x"}}, + "runs": [ + {"serving": {"recipe": old}}, + {"serving": {"recipe": latest}}, + # Another model's run regraded; it inferred nothing. + {"model": "other", "serving": {"recipe": _plan({"dtype": "x"})}}, + ], + } + # Unset dtype resolves to this run's checkpoint: a bf16 run of it, + # recorded without numerics, equals a new bf16 run. + assert read_recorded_numerics(meta, BF16) == served_numerics( + _plan({}), checkpoint=BF16 + ) + + def test_schema_4_endpoint_or_no_runs_is_unknown(self): + endpoint = { + "identity": {"schema": 4}, + "runs": [{"serving": {"recipe": None, "stated": {"hardware": "x"}}}], + } + assert read_recorded_numerics(endpoint) is None + assert read_recorded_numerics({"identity": {"schema": 4}}) is None + + @pytest.mark.parametrize("schema", [1, 2, 3, 99]) + def test_other_schemas_record_none(self, schema): + meta = { + "identity": {"schema": schema}, + "runs": [{"serving": {"recipe": _plan(BF16)}}], + } + assert read_recorded_numerics(meta) is None + assert read_recorded_numerics(None) is None + + class TestLifecycleAndJudgeWeights: def test_lifecycles_are_recorded_sorted_and_copied(self): lifecycles = { @@ -1339,7 +1538,7 @@ def test_a_schema_1_block_reads_as_this_schema(self): read = read_identity(block) assert read is not None identity, legacy = read - assert identity["schema"] == IDENTITY_SCHEMA == 4 + assert identity["schema"] == IDENTITY_SCHEMA == 5 assert identity["data"] == data assert identity["preprocess"] == prompt assert identity["inference"] == {"digest": "sha256:i"} @@ -1359,10 +1558,17 @@ def test_a_schema_2_block_is_read_as_written(self): assert not is_code_identity(identity["preprocess"]) assert legacy is None + def test_a_schema_4_block_is_read_as_written(self): + inference = {"digest": "sha256:i"} + read = read_identity({"schema": 4, "inference": inference}) + assert read is not None + assert read[0]["schema"] == IDENTITY_SCHEMA + assert read[0]["inference"] == inference + def test_another_schema_is_not_read(self): - assert read_identity({"schema": 5, "data": {"digest": "x"}}) is None + assert read_identity({"schema": 6, "data": {"digest": "x"}}) is None assert read_identity("nope") is None - assert carried_identity({"identity": {"schema": 5}}) == empty_identity() + assert carried_identity({"identity": {"schema": 6}}) == empty_identity() def test_carried_runs_tolerates_old_and_malformed(self): assert carried_runs(None) == [] diff --git a/tests/unit/infer/test_introspect.py b/tests/unit/infer/test_introspect.py index 24fe7945..33aef553 100644 --- a/tests/unit/infer/test_introspect.py +++ b/tests/unit/infer/test_introspect.py @@ -23,6 +23,7 @@ _refine_family_from_name, _resolve_param_billions, bytes_per_param, + checkpoint_numerics, checkpoint_weight_bytes, detect_local_gpu, dtype_bytes, @@ -735,6 +736,33 @@ def test_missing_or_unreadable_config_is_none(self, tmp_path: Path) -> None: assert native_context_window(str(tmp_path)) is None +class TestCheckpointNumerics: + @pytest.mark.parametrize( + ("config", "expected"), + [ + ({"torch_dtype": "bfloat16"}, {"dtype": "bfloat16"}), + ({"dtype": "float16"}, {"dtype": "float16"}), + ({"text_config": {"torch_dtype": "bfloat16"}}, {"dtype": "bfloat16"}), + ( + { + "torch_dtype": "bfloat16", + "quantization_config": {"quant_method": "fp8"}, + }, + {"dtype": "bfloat16", "quantization": "fp8"}, + ), + ({}, {}), + ], + ) + def test_reads_dtype_and_quantization( + self, tmp_path: Path, config: object, expected: dict[str, str] + ) -> None: + (tmp_path / "config.json").write_text(json.dumps(config)) + assert checkpoint_numerics(str(tmp_path)) == expected + + def test_missing_config_is_none(self, tmp_path: Path) -> None: + assert checkpoint_numerics(str(tmp_path)) is None + + class TestLogitShape: @pytest.mark.parametrize( ("config", "expected"), From d12b2d2929accc7e7f04438e372d0b7b05b6950c Mon Sep 17 00:00:00 2001 From: Ethan Date: Wed, 28 Oct 2026 02:59:30 +0800 Subject: [PATCH 2/2] fix(core): treat a readable unquantized checkpoint as unquantized Runtime `quantization: fp8` on a bf16 checkpoint resumed into the bf16 run: an unquantized checkpoint left `quantization` unknown, and an unknown key never differs. A readable checkpoint now records `None`; only an unreadable one stays unknown. - `dtype: auto` on a float32 checkpoint is unknown: engines downcast it. - A schema-4 directory whose latest own run recorded no serving has unknown numerics, instead of falling back to an older run's plan. - Trim the numerics docs and docstrings. Co-Authored-By: Claude Opus 5.5 --- docs/guide/configuration.md | 27 ++---- sieval/cli/session.py | 6 +- sieval/core/runners/runner.py | 5 +- sieval/core/runners/stage_resume.py | 16 +--- sieval/core/tasks/run_meta.py | 96 +++++++++---------- sieval/infer/introspect.py | 11 +-- .../unit/core/runners/test_runner_run_meta.py | 35 ++++++- tests/unit/core/tasks/test_run_meta.py | 29 +++++- 8 files changed, 123 insertions(+), 102 deletions(-) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 047f5977..85b049d0 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -379,30 +379,21 @@ entry records its own `serving` (`engine`, `engine_version`, `launch`: `in_run` or `external`, and `recipe`: the plan without checkpoint or host), so a reader sees what changed. -What the served model computes with does count. These engine parameters -change the logits, so they are `identity.inference.numerics`, compared on -resume; every other engine parameter is serving -(`sieval.core.tasks.run_meta.NUMERICS_PARAMS` is the list): +What the served model computes with does count: these engine parameters +change the logits, so they are compared as `identity.inference.numerics`. | `numerics` key | Engine parameters | Unset means | | --- | --- | --- | -| `dtype` | `dtype` | The checkpoint's `torch_dtype` (`auto` too) | -| `quantization` | `quantization` | The checkpoint's `quantization_config.quant_method`, else unknown | +| `dtype` | `dtype` | The checkpoint's `torch_dtype`; unknown for float32, which engines downcast | +| `quantization` | `quantization` | The checkpoint's `quantization_config.quant_method`, else none | | `kv_cache_dtype` | `kv_cache_dtype` | `auto` | | `model_overrides` | `json_model_override_args` (SGLang), `hf_overrides` (vLLM) | None | -Each is compared by the value the engine uses, so stating a default -(`dtype: auto`, `kv_cache_dtype: auto`, `bf16` for `bfloat16`) is no change. -Out of it, among others: tp, dp, ep, pp, replicas, prefill/decode roles, memory -fraction, chunked prefill, CUDA graphs, scheduler and port settings. Roles that -disagree on a key record each value. - -A key unknown on either side is no difference: an endpoint's numerics are -what its root model states under `serving.numerics`, and a key it does not -state is unknown. A `meta.json` written before numerics counted (identity -schema 4) has its numerics read from the plan of its latest run in -`runs[].serving`, unset values resolved against this run's checkpoint, so it -resumes against a run with the same effective values. +Values are compared as the engine uses them, so stating a default (`dtype: +auto`, `bf16`) is no change. A key unknown on either side is no difference: +an endpoint's numerics are only what its root model states under +`serving.numerics`. A schema-4 `meta.json` has its numerics read from its +latest run's plan. The context limit does count, since it decides which prompts overflow. A launched engine's is the plan's `max_model_len` (vLLM) or `context_length` diff --git a/sieval/cli/session.py b/sieval/cli/session.py index 748fa1f2..6931ac2b 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -5092,10 +5092,8 @@ def _judge_weights_for(self, task_name: str) -> dict[str, str] | None: return out or None def _checkpoint_numerics_for(self, model_name: str) -> dict[str, JSONValue] | None: - """The ``dtype`` and ``quantization`` of the checkpoint *model_name*'s - root launches, which its engine uses where the plan leaves them unset; - ``None`` for a model SiEval does not serve or an unreadable checkpoint. - """ + """:func:`checkpoint_numerics` of the checkpoint *model_name*'s root + launches; ``None`` when SiEval does not serve it.""" plan = (self._infer_plans or {}).get(self._root_model(model_name)) if plan is None: return None diff --git a/sieval/core/runners/runner.py b/sieval/core/runners/runner.py index 93c961fd..6e53ca5b 100644 --- a/sieval/core/runners/runner.py +++ b/sieval/core/runners/runner.py @@ -549,9 +549,8 @@ def __init__( that needs a lost engine (:mod:`.engine_loss`). *judge_weights* names, by judge role, the weights a launched judge serves, compared instead of its name. *lifecycles* are the launched models' lifecycles, recorded in - ``runs[].serving``. *checkpoint_numerics* is the launched checkpoint's - own ``dtype`` and ``quantization``, which the inference identity's - ``numerics`` resolve unset engine params to (:func:`served_numerics`). + ``runs[].serving``. *checkpoint_numerics* is the launched + checkpoint's own ``dtype`` and ``quantization`` (:func:`served_numerics`). """ # A malformed judge reference would leave every grader identity # unknown; refuse it before any work. diff --git a/sieval/core/runners/stage_resume.py b/sieval/core/runners/stage_resume.py index 1c136c05..bffd79ac 100644 --- a/sieval/core/runners/stage_resume.py +++ b/sieval/core/runners/stage_resume.py @@ -206,11 +206,9 @@ def plan_resume( :func:`~sieval.core.tasks.run_meta.legacy_grader_keys` for this task, read only for a schema-1 directory, and *legacy_inference* :func:`~sieval.core.tasks.run_meta.legacy_inference_digest`, only for one - before identity schema 4. *checkpoint_numerics* resolves the unset - numerics of a schema-4 directory's plan - (:func:`~sieval.core.tasks.run_meta.read_recorded_numerics`). The runner - plans per sample and is - authoritative; this says what a resume of a finished directory does. + before identity schema 4; *checkpoint_numerics* resolves a schema-4 plan's + unset numerics. The runner plans per sample and is authoritative; this + says what a resume of a finished directory does. """ block = (recorded_meta or {}).get("identity") schema = block.get("schema") if isinstance(block, Mapping) else None @@ -302,12 +300,8 @@ def task_resume_plan( :class:`~sieval.core.runners.runner.TaskRunner` does, so call it after ``task.setup()``, which may resolve an argument the grading identities read. *runner_keys* holds :data:`~sieval.core.tasks.run_meta.INFERENCE_RUNNER_KEYS` - and *context_limit*, *recipe*, *stated_serving* and - *checkpoint_numerics* as the runner is given them; they decide the - served numerics (:func:`~sieval.core.tasks.run_meta.served_numerics`), - and *recipe* is compared whole for a directory before identity schema 4. - Reads files, - starts nothing and calls no model. + and the rest as the runner is given them. Reads files, starts nothing and + calls no model. """ task_cls = type(task) judges = judge_identities(task.judge_models(), task.judge_prompts()) diff --git a/sieval/core/tasks/run_meta.py b/sieval/core/tasks/run_meta.py index 0f4caad1..81c0f5e3 100644 --- a/sieval/core/tasks/run_meta.py +++ b/sieval/core/tasks/run_meta.py @@ -38,14 +38,9 @@ compared apart from ``inference``: by ``weights`` when both runs state them, else by ``model``. -What the served model computes with is not how it is served: ``inference`` -keeps ``numerics`` (:func:`served_numerics`), the effective ``dtype``, -runtime ``quantization``, ``kv_cache_dtype`` and model overrides, by the -engine parameters :data:`NUMERICS_PARAMS` lists. They sit beside the digest, -not in it, because a key may be unknown: for an endpoint, one the caller does -not state. A key known on both sides must be equal; one unknown on either -side is no difference. A schema-4 block recorded none: its numerics are read -from its plan in ``runs[].serving`` (:func:`read_recorded_numerics`). +What the served model computes with is compared, though: ``inference`` +keeps ``numerics`` (:func:`served_numerics`) beside the digest, not in it, +since a key may be unknown, and an unknown key is no difference. ``preprocess`` is code, not the prompts it renders: a prompt is code, arguments, data and the model's ``extra``, and ``extra`` reaches the request, @@ -227,11 +222,8 @@ class ServingFingerprint(TypedDict): "numerics", ) -#: The engine parameters that change what the served model computes, by the -#: ``inference.numerics`` key they set. Every other engine parameter is how -#: the model is served (parallelism, replicas, disaggregation, memory, -#: scheduling, ports): recorded in ``runs[].serving``, never compared. Names -#: are in underscore form; :func:`numerics_key` reads either form. +#: The engine parameters that change the logits, by the ``inference.numerics`` +#: key they set. Every other one is serving, never compared. NUMERICS_PARAMS: Mapping[str, tuple[str, ...]] = MappingProxyType( { "dtype": ("dtype",), @@ -695,9 +687,8 @@ def inference_matches( def numerics_key(param: str) -> str | None: - """The ``inference.numerics`` key engine parameter *param* sets, in dash or - underscore form; ``None`` for a parameter that only says how the model is - served (:data:`NUMERICS_PARAMS`).""" + """The ``inference.numerics`` key engine parameter *param* (dash or + underscore form) sets; ``None`` for a serving-only parameter.""" name = param.replace("-", "_") return next((k for k, names in NUMERICS_PARAMS.items() if name in names), None) @@ -718,8 +709,8 @@ def numerics_key(param: str) -> str | None: def _numerics_value(key: str, value: object) -> JSONValue: - """*value* of numerics *key*, spelled one way: an alias by its name, model - overrides as an object (an empty one is none).""" + """*value* of numerics *key*, spelled one way: aliases by name, model + overrides as an object (empty is none).""" if key == "dtype" and isinstance(value, str): return _DTYPE_ALIASES.get(value, value) if key == "kv_cache_dtype" and isinstance(value, str): @@ -728,7 +719,7 @@ def _numerics_value(key: str, value: object) -> JSONValue: try: value = json.loads(value) except ValueError: - return value + return value # the engine refuses it at launch; compare the text if key == "model_overrides" and not value: return None return cast(JSONValue, value) @@ -737,9 +728,13 @@ def _numerics_value(key: str, value: object) -> JSONValue: def _role_numerics( params: Mapping[str, Any], checkpoint: Mapping[str, JSONValue] | None ) -> Numerics: - """One role's effective numerics: what its engine *params* state, else the - engine's default. An unset or ``auto`` ``dtype`` and an unset - ``quantization`` are the *checkpoint*'s, unknown without it.""" + """One role's effective numerics from its engine *params*. + + An unset or ``auto`` ``dtype`` is the *checkpoint*'s, except float32, + which engines downcast by version: unknown. An unset ``quantization`` is + the *checkpoint*'s, ``None`` for an unquantized one. Both are unknown when + *checkpoint* is (its config unreadable). + """ stated: dict[str, object] = {} for param, value in params.items(): key = numerics_key(param) @@ -750,11 +745,13 @@ def _role_numerics( if dtype is not None and dtype != "auto": out["dtype"] = _numerics_value("dtype", dtype) elif checkpoint is not None and "dtype" in checkpoint: - out["dtype"] = _numerics_value("dtype", checkpoint["dtype"]) + native = _numerics_value("dtype", checkpoint["dtype"]) + if native != "float32": + out["dtype"] = native if "quantization" in stated: out["quantization"] = _numerics_value("quantization", stated["quantization"]) - elif checkpoint is not None and "quantization" in checkpoint: - out["quantization"] = checkpoint["quantization"] + elif checkpoint is not None: + out["quantization"] = checkpoint.get("quantization") out["kv_cache_dtype"] = _numerics_value( "kv_cache_dtype", stated.get("kv_cache_dtype", "auto") ) @@ -771,13 +768,11 @@ def served_numerics( ) -> Numerics | None: """What the served model computes with, keyed by :data:`NUMERICS_PARAMS`. - For an engine the run launched, the effective value of each key from the - ``engine_params`` of every role in *recipe*; *checkpoint* holds the - checkpoint's own ``dtype`` and ``quantization`` (``None`` when it has - none), which an unset one resolves to. Roles that disagree on a key give - their distinct values, sorted. For an endpoint (no *recipe*), the - ``numerics`` the caller *stated*, as given. A key left out is unknown, and - ``None`` is returned when every key is. + A launched engine's come from every role's ``engine_params`` in *recipe*, + unset values resolved against *checkpoint* (:func:`_role_numerics`); roles + that disagree on a key give their distinct values, sorted. An endpoint's + (no *recipe*) are the ``numerics`` the caller *stated*. A key left out is + unknown; ``None`` when all are. """ if recipe is None: given = (stated or {}).get("numerics") @@ -812,14 +807,12 @@ def read_recorded_numerics( meta: Mapping[str, Any] | None, checkpoint: Mapping[str, JSONValue] | None = None, ) -> Numerics | None: - """The numerics a recorded ``meta.json``'s inference ran with, ``None`` + """The numerics a recorded ``meta.json``'s inference ran with; ``None`` when unknown. - This schema records them in ``inference.numerics``. Schema 4 recorded none: - they are read from the plan of its latest run with this model, in - ``runs[].serving``, as :func:`served_numerics` would have, its unset values - resolved against this run's *checkpoint*. Schemas 1 to 3 compare the plan - whole (:func:`legacy_inference_digest`), so they need none. + Schema 4 recorded none: they are read from the plan of its latest run with + this model, resolved against this run's *checkpoint*; unknown when that + run recorded no serving. Schemas 1 to 3 compare the plan whole instead. """ block = (meta or {}).get("identity") if not isinstance(block, Mapping): @@ -832,24 +825,25 @@ def read_recorded_numerics( if schema != 4: return None runs = cast(Mapping[str, Any], meta).get("runs") - for run in reversed(runs if isinstance(runs, list) else []): - if not isinstance(run, Mapping) or "model" in run: - continue # another model's run did not infer these samples - serving = run.get("serving") - if isinstance(serving, Mapping): - recipe = serving.get("recipe") - return served_numerics( - recipe if isinstance(recipe, Mapping) else None, None, checkpoint - ) - return None + own = ( + run + for run in reversed(runs if isinstance(runs, list) else []) + if isinstance(run, Mapping) and "model" not in run + ) + serving = next(own, {}).get("serving") + if not isinstance(serving, Mapping): + return None + recipe = serving.get("recipe") + return served_numerics( + recipe if isinstance(recipe, Mapping) else None, None, checkpoint + ) def numerics_differences( recorded: Mapping[str, Any] | None, current: Mapping[str, Any] | None ) -> list[str]: - """Each numerics key known on both sides whose values differ, as - ``key: recorded -> current``. A key unknown on either side is no - difference.""" + """Each key known on both sides whose values differ, as + ``key: recorded -> current``.""" if recorded is None or current is None: return [] return [ diff --git a/sieval/infer/introspect.py b/sieval/infer/introspect.py index ea6da92d..27ee2a6e 100644 --- a/sieval/infer/introspect.py +++ b/sieval/infer/introspect.py @@ -423,14 +423,9 @@ def native_context_window(checkpoint: str) -> int | None: def checkpoint_numerics(checkpoint: str) -> dict[str, str] | None: - """The ``dtype`` and ``quantization`` an engine serving *checkpoint* uses - when its params leave them unset. - - ``dtype`` is ``torch_dtype`` (``dtype`` in newer configs) from - ``config.json`` or its ``text_config``, ``quantization`` the - ``quantization_config.quant_method``; each absent when unstated. ``None`` - when ``config.json`` cannot be read. - """ + """*checkpoint*'s ``torch_dtype`` (or ``dtype``) and + ``quantization_config.quant_method``, from ``config.json`` or its + ``text_config``; each absent when unstated, ``None`` when unreadable.""" sources = _text_configs(checkpoint) if not sources: return None diff --git a/tests/unit/core/runners/test_runner_run_meta.py b/tests/unit/core/runners/test_runner_run_meta.py index 8d4f5433..6c9c6a6c 100644 --- a/tests/unit/core/runners/test_runner_run_meta.py +++ b/tests/unit/core/runners/test_runner_run_meta.py @@ -668,15 +668,44 @@ async def test_an_endpoint_stating_other_numerics_is_refused(self, tmp_path): stated_serving={"numerics": {"dtype": "fp16"}}, ).arun() assert model.call_count == 0 - # The recorded run's quantization is unknown (an unquantized - # checkpoint), so stating one is not a difference. + # The recorded checkpoint is unquantized, so a stated fp8 differs. + with pytest.raises(StageResumeError, match="quantization: None -> 'fp8'"): + await TaskRunner( + task, + make_config(tmp_path, resume=True), + stated_serving={"numerics": {"dtype": "bf16", "quantization": "fp8"}}, + ).arun() + assert model.call_count == 0 await TaskRunner( task, make_config(tmp_path, resume=True), - stated_serving={"numerics": {"dtype": "bf16", "quantization": "fp8"}}, + stated_serving={"numerics": {"dtype": "bf16"}}, ).arun() assert model.call_count == 1 + @pytest.mark.anyio + async def test_runtime_quantization_of_an_unquantized_checkpoint_is_refused( + self, tmp_path + ): + task, _ = _counting_task() + await TaskRunner( + task, + make_config(tmp_path), + inference_recipe=_engine(tp=1), + checkpoint_numerics=self.BF16, + ).arun() + _forget_last_sample(tmp_path / "results") + + task, model = _counting_task() + with pytest.raises(StageResumeError, match="quantization: None -> 'fp8'"): + await TaskRunner( + task, + make_config(tmp_path, resume=True), + inference_recipe=_engine(tp=1, quantization="fp8"), + checkpoint_numerics=self.BF16, + ).arun() + assert model.call_count == 0 + class TestUnavailableFingerprints: @pytest.mark.anyio diff --git a/tests/unit/core/tasks/test_run_meta.py b/tests/unit/core/tasks/test_run_meta.py index 8be7beeb..7785e9d1 100644 --- a/tests/unit/core/tasks/test_run_meta.py +++ b/tests/unit/core/tasks/test_run_meta.py @@ -484,6 +484,7 @@ def test_a_stated_default_equals_an_unset_one(self): assert unset == stated == aliased assert unset == { "dtype": "bfloat16", + "quantization": None, "kv_cache_dtype": "auto", "model_overrides": None, } @@ -494,13 +495,25 @@ def test_quantization_unset_is_the_checkpoints(self): assert on_disk is not None and on_disk["quantization"] == "fp8" runtime = served_numerics(_plan({"quantization": "awq"}), checkpoint=checkpoint) assert runtime is not None and runtime["quantization"] == "awq" - # Unquantized checkpoint: no key, so unknown, not "none". + # A readable, unquantized checkpoint is known to be unquantized, so + # runtime fp8 on it differs. plain = served_numerics(_plan({}), checkpoint=BF16) - assert plain is not None and "quantization" not in plain + fp8 = served_numerics(_plan({"quantization": "fp8"}), checkpoint=BF16) + assert plain is not None and plain["quantization"] is None + assert numerics_differences(plain, fp8) == ["quantization: None -> 'fp8'"] - def test_an_unreadable_checkpoint_leaves_dtype_unknown(self): + def test_an_unreadable_checkpoint_leaves_dtype_and_quantization_unknown(self): numerics = served_numerics(_plan({})) - assert numerics is not None and "dtype" not in numerics + assert numerics is not None + assert "dtype" not in numerics and "quantization" not in numerics + + def test_a_float32_checkpoint_under_auto_is_unknown(self): + """Engines downcast float32 under ``auto``, by version.""" + fp32 = {"dtype": "float32"} + auto = served_numerics(_plan({}), checkpoint=fp32) + assert auto is not None and "dtype" not in auto + stated = served_numerics(_plan({"dtype": "fp32"}), checkpoint=fp32) + assert stated is not None and stated["dtype"] == "float32" def test_model_overrides_compare_as_objects(self): text = served_numerics( @@ -623,6 +636,14 @@ def test_schema_4_endpoint_or_no_runs_is_unknown(self): assert read_recorded_numerics(endpoint) is None assert read_recorded_numerics({"identity": {"schema": 4}}) is None + def test_schema_4_latest_run_without_serving_is_unknown(self): + """An older run's plan may not be what the latest samples ran with.""" + meta = { + "identity": {"schema": 4}, + "runs": [{"serving": {"recipe": _plan({"dtype": "float16"})}}, {}], + } + assert read_recorded_numerics(meta, BF16) is None + @pytest.mark.parametrize("schema", [1, 2, 3, 99]) def test_other_schemas_record_none(self, schema): meta = {