From 12889da57ff45b71664c91bd7a5e361e9abbf501 Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Mon, 28 Sep 2026 17:22:32 +0200 Subject: [PATCH 1/3] Add fastermodels.measurement: one on-device measurement row, validated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A measurement row records one session of one model variant on one device and software stack, at one precision and one batch size, on one date: latency percentiles and their spread over repeats, device conditions (power mode, clock locking, temperature, throttling), the accuracy with its confidence interval measured through the deployed engine, and argmax parity against the reference model. Rows are stored as JSON Lines. - FIELDS: the schema as data (type, required, rules); SCHEMA_VERSION = 1. - validate(row): every problem, by field name. - status(row): whether the measurement is VALID and why not — parity below its floor, throttling, latency spread above 5 %, accuracy not measured through the deployed engine, latency not measured on the device, or any of these not recorded. It is a validity status, never a verdict against a latency or accuracy budget. - key(row): the identity a row is unique on; a software update on the device gives a new key. - read_rows / write_row: JSON Lines, append-only, refusing malformed and duplicate rows. to_parquet for publishing (optional `parquet` extra). - throughput(row) is derived, never stored. Reached as `fastermodels.measurement`; the generic names stay off the package top level. --- fastermodels/__init__.py | 1 + fastermodels/_modidx.py | 23 ++ fastermodels/measurement.py | 248 +++++++++++++++ nbs/00_model.ipynb | 1 + nbs/01_eval.ipynb | 1 + nbs/02_card.ipynb | 1 + nbs/03_gate.ipynb | 1 + nbs/04_measurement.ipynb | 516 +++++++++++++++++++++++++++++++ nbs/_quarto.yml | 1 + nbs/tests/test_measurement.ipynb | 373 ++++++++++++++++++++++ pyproject.toml | 3 +- 11 files changed, 1168 insertions(+), 1 deletion(-) create mode 100644 fastermodels/measurement.py create mode 100644 nbs/04_measurement.ipynb create mode 100644 nbs/tests/test_measurement.ipynb diff --git a/fastermodels/__init__.py b/fastermodels/__init__.py index dc88e53..03e93e2 100644 --- a/fastermodels/__init__.py +++ b/fastermodels/__init__.py @@ -5,3 +5,4 @@ params, macs, peak_activation_bytes) from fastermodels.card import render_card, check_card, FORBIDDEN from fastermodels.gate import run_gate, gate_passed, GateRow +from fastermodels import measurement as measurement diff --git a/fastermodels/_modidx.py b/fastermodels/_modidx.py index 4062ac7..51d5b40 100644 --- a/fastermodels/_modidx.py +++ b/fastermodels/_modidx.py @@ -36,6 +36,29 @@ 'fastermodels.gate._reload': ('gate.html#_reload', 'fastermodels/gate.py'), 'fastermodels.gate.gate_passed': ('gate.html#gate_passed', 'fastermodels/gate.py'), 'fastermodels.gate.run_gate': ('gate.html#run_gate', 'fastermodels/gate.py')}, + 'fastermodels.measurement': { 'fastermodels.measurement._canonical': ( 'measurement.html#_canonical', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._cross_problems': ( 'measurement.html#_cross_problems', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._field_problems': ( 'measurement.html#_field_problems', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._is_date': ('measurement.html#_is_date', 'fastermodels/measurement.py'), + 'fastermodels.measurement._is_json': ('measurement.html#_is_json', 'fastermodels/measurement.py'), + 'fastermodels.measurement._item': ('measurement.html#_item', 'fastermodels/measurement.py'), + 'fastermodels.measurement._json': ('measurement.html#_json', 'fastermodels/measurement.py'), + 'fastermodels.measurement._parse': ('measurement.html#_parse', 'fastermodels/measurement.py'), + 'fastermodels.measurement._typed': ('measurement.html#_typed', 'fastermodels/measurement.py'), + 'fastermodels.measurement.key': ('measurement.html#key', 'fastermodels/measurement.py'), + 'fastermodels.measurement.read_rows': ( 'measurement.html#read_rows', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.status': ('measurement.html#status', 'fastermodels/measurement.py'), + 'fastermodels.measurement.throughput': ( 'measurement.html#throughput', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.to_parquet': ( 'measurement.html#to_parquet', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.validate': ('measurement.html#validate', 'fastermodels/measurement.py'), + 'fastermodels.measurement.write_row': ( 'measurement.html#write_row', + 'fastermodels/measurement.py')}, 'fastermodels.model': { 'fastermodels.model.FasterModel': ('model.html#fastermodel', 'fastermodels/model.py'), 'fastermodels.model.FasterModel.__init__': ('model.html#fastermodel.__init__', 'fastermodels/model.py'), 'fastermodels.model.FasterModel._from_pretrained': ( 'model.html#fastermodel._from_pretrained', diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py new file mode 100644 index 0000000..ff02ba9 --- /dev/null +++ b/fastermodels/measurement.py @@ -0,0 +1,248 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_measurement.ipynb. + +# %% ../nbs/04_measurement.ipynb #imports +from __future__ import annotations + +import json +import math +import numbers +import re +from datetime import datetime +from pathlib import Path + +# %% auto #0 +__all__ = ['SCHEMA_VERSION', 'FIELDS', 'validate', 'status', 'key', 'throughput', 'read_rows', 'write_row', 'to_parquet'] + +# %% ../nbs/04_measurement.ipynb #schema +SCHEMA_VERSION = 1 +_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|\+00:00)' +_PCT = dict(ge=0, le=100) + +# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None +FIELDS = { + # identity and provenance + 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)), + 'run_id': dict(type=str, required=True), + 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True), + 'series': dict(type=str, required=False), + 'harness_commit': dict(type=str, required=True), + 'fasterai_version': dict(type=str, required=False), + 'report_url': dict(type=str, required=False), + 'hf_repo': dict(type=str, required=False), + # model + 'model_id': dict(type=str, required=True), + 'model_revision': dict(type=str, required=True), + 'task': dict(type=str, required=True, choices=('classification', 'detection')), + 'architecture': dict(type=str, required=True), + 'base_weights': dict(type=str, required=True), + 'weights_license': dict(type=str, required=True), + 'variant': dict(type=str, required=True), + 'recipe': dict(type=dict, required=True, json=True), + 'input_resolution': dict(type=int, required=True, gt=0), + 'macs': dict(type=int, required=True, gt=0), + 'params': dict(type=int, required=True, gt=0), + 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned + 'engine_hash': dict(type=str, required=True, pattern=r'[0-9a-f]{64}'), # sha256 of the engine file that ran + 'size_mb': dict(type=float, required=True, gt=0), + # device and runtime + 'device': dict(type=str, required=True), + 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')), + 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')), + 'runtime_version': dict(type=str, required=True), + 'platform_version': dict(type=str, required=False), + 'precision': dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')), + 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')), + 'power_mode': dict(type=str, required=False), + 'clocks_locked': dict(type=bool, required=False), + 'batch_size': dict(type=int, required=True, ge=1), + # conditions + 'temp_start_c': dict(type=float, required=False), + 'temp_end_c': dict(type=float, required=False), + 'gpu_freq_mhz': dict(type=float, required=False, gt=0), + 'throttled': dict(type=bool, required=False), + 'warmup_s': dict(type=float, required=False, ge=0), + 'duration_s': dict(type=float, required=False, ge=0), + 'repeats': dict(type=int, required=True, ge=1), + 'latency_spread_pct': dict(type=float, required=False, ge=0), + # latency + 'latency_ms_p50': dict(type=float, required=True, gt=0), + 'latency_ms_p90': dict(type=float, required=False, gt=0), + 'latency_ms_p99': dict(type=float, required=False, gt=0), + 'latency_scope': dict(type=str, required=True), + 'host_latency_ms': dict(type=float, required=False, gt=0), + 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')), + # quality + 'metric': dict(type=str, required=True), + 'eval_set': dict(type=str, required=True), + 'eval_set_hash': dict(type=str, required=True), + 'dataset_license': dict(type=str, required=True), + 'eval_n': dict(type=int, required=True, gt=0), + 'accuracy': dict(type=float, required=True, **_PCT), + 'ci_low': dict(type=float, required=True, **_PCT), + 'ci_high': dict(type=float, required=True, **_PCT), + 'ci_method': dict(type=str, required=True, choices=('wilson', 'bootstrap')), + 'measured_through': dict(type=str, required=True, choices=('deployed_engine', 'dev_model')), + 'argmax_parity': dict(type=float, required=False, **_PCT), + 'parity_floor': dict(type=float, required=True, **_PCT), + 'baseline_run_id': dict(type=str, required=False), +} + +def _item(o): + "A numpy scalar as the Python number JSON knows" + if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item() + raise TypeError(f'{type(o).__name__} is not JSON serializable') + + +def _json(obj, **kw): return json.dumps(obj, default=_item, allow_nan=False, **kw) +def _canonical(d): return _json(d, sort_keys=True, separators=(',', ':')) + + +def _is_json(v): + try: return _json(v) is not None + except (TypeError, ValueError): return False + + +def _is_date(v): + try: return datetime.fromisoformat(v.replace('Z', '+00:00')) is not None # Python 3.10 does not read 'Z' + except ValueError: return False + + +_RULES = { + 'choices': (lambda v, c: v in c, 'not one of {}'), + 'gt': (lambda v, b: v > b, 'not > {}'), + 'ge': (lambda v, b: v >= b, 'not >= {}'), + 'le': (lambda v, b: v <= b, 'not <= {}'), + 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'), + 'date': (lambda v, _: _is_date(v), 'is not a date'), + 'json': (lambda v, _: _is_json(v), 'is not JSON'), +} +_ORDERED = (('ci_low', 'accuracy', 'ci_high'), ('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99')) + + +def _typed(v, t): + "`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite" + if isinstance(v, bool): return t is bool + if t is int: return isinstance(v, numbers.Integral) + if t is float: return isinstance(v, numbers.Real) and math.isfinite(v) + return isinstance(v, t) + + +def _field_problems(name, v): + "The problems of one field value, against its spec" + spec = FIELDS[name] + if v is None: return [f'{name}: missing'] if spec['required'] else [] + if not _typed(v, spec['type']): return [f'{name}: expected {spec["type"].__name__}, got {v!r}'] + return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])] + + +def _cross_problems(row): + "Problems between fields; each field involved is already well typed" + p, mode = row['precision'], row.get('int8_mode') + out = [] + if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8') + if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}') + for fields in _ORDERED: + vals = [(f, row[f]) for f in fields if row.get(f) is not None] + if [v for _, v in vals] != sorted(v for _, v in vals): + out.append(f"{', '.join(fields)}: not in order, " + ', '.join(f'{f}={v}' for f, v in vals)) + return out + + +def validate( + row: dict, # one measurement row +) -> list[str]: + "Every structural problem of `row`, each prefixed by its field name; an empty list means well formed" + if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}'] + out = [f'{k}: unknown field' for k in row if k not in FIELDS] + for name in FIELDS: out += _field_problems(name, row.get(name)) + return out or _cross_problems(row) + +# %% ../nbs/04_measurement.ipynb #status +# field, when its value makes the measurement invalid, and why; a field that is None is not recorded +_VALIDITY = ( + ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'), + ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'), + ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'), + ('throttled', lambda v, r, m: v, 'the device throttled during the session'), + ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'), +) + + +def status( + row: dict, # one measurement row + max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent +) -> tuple[str, list[str]]: + "('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget" + reasons = validate(row) + if reasons: return 'INVALID', reasons + for f, bad, why in _VALIDITY: + v = row.get(f) + if v is None: reasons.append(f'{f}: not recorded') + elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct)) + return ('INVALID', reasons) if reasons else ('VALID', []) + +# %% ../nbs/04_measurement.ipynb #key +def key( + row: dict, # one measurement row +) -> tuple: + "The identity a row is unique on; a software update on the device gives a new key" + return (row['model_id'], row['model_revision'], row['variant'], _canonical(row['recipe']), + row['input_resolution'], row['device'], row['runtime'], row['runtime_version'], + row.get('platform_version'), row['precision'], row.get('int8_mode'), row['batch_size'], + row['measured_at']) + + +def throughput( + row: dict, # one measurement row +) -> float: + "Images per second at the timed batch, derived from the median latency and never stored" + return row['batch_size'] * 1000 / row['latency_ms_p50'] + +# %% ../nbs/04_measurement.ipynb #io +def _parse(text, path): + "The rows of a JSON Lines text; a malformed line raises with its number" + rows = [] + for i, line in enumerate(text.splitlines(), 1): + if not line.strip(): continue + try: r = json.loads(line) + except json.JSONDecodeError as e: raise ValueError(f'{path}:{i}: not JSON ({e.msg})') from None + if not isinstance(r, dict): raise ValueError(f'{path}:{i}: not a JSON object') + rows.append(r) + return rows + + +def read_rows( + path: str | Path, # a JSON Lines file of measurement rows +) -> list[dict]: + "The rows of `path`, as stored; `status` judges them" + return _parse(Path(path).read_text(), path) + + +def write_row( + path: str | Path, # a JSON Lines file, created if absent + row: dict, # the measurement row to append +): + "Append `row` to `path`; refuses a malformed row and a row whose key is already in the file" + problems = validate(row) + if problems: raise ValueError('row refused: ' + '; '.join(problems)) + p, k = Path(path), key(row) + text = p.read_text() if p.exists() else '' + if k in {key(r) for r in _parse(text, path)}: raise ValueError(f'row refused: key {k} already in {path}') + # ponytail: no file lock, one writer at a time + with p.open('a') as f: f.write(('\n' if text and not text.endswith('\n') else '') + _json(row) + '\n') + +# %% ../nbs/04_measurement.ipynb #parquet +def to_parquet( + rows: list[dict], # well-formed measurement rows, e.g. from `read_rows` + path: str | Path, # the Parquet file to write +): + "Write `rows` as one Parquet table, one column per field in `FIELDS` order, dict fields as canonical JSON" + try: + import pandas as pd + import pyarrow # noqa: F401 + except ImportError: raise ImportError("to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'") from None + for i, r in enumerate(rows): + problems = validate(r) + if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems)) + table = [{k: _canonical(r[k]) if FIELDS[k]['type'] is dict else r.get(k) for k in FIELDS} for r in rows] + pd.DataFrame(table, columns=list(FIELDS)).to_parquet(path, engine='pyarrow', index=False) diff --git a/nbs/00_model.ipynb b/nbs/00_model.ipynb index bcea8bf..f900e05 100644 --- a/nbs/00_model.ipynb +++ b/nbs/00_model.ipynb @@ -374,6 +374,7 @@ "- [Eval](01_eval.html) - the per-image harness the published numbers come from\n", "- [Card](02_card.html) - what the artifact says about itself\n", "- [Gate](03_gate.html) - the conditions an artifact meets before publication\n", + "- [Measurement](04_measurement.html) - one on-device measurement row of a published model\n", "\n", "Tests live in `nbs/tests/test_model.ipynb`." ] diff --git a/nbs/01_eval.ipynb b/nbs/01_eval.ipynb index 3011d53..30bcdb9 100644 --- a/nbs/01_eval.ipynb +++ b/nbs/01_eval.ipynb @@ -436,6 +436,7 @@ "- [Model](00_model.html) - the model these numbers are measured on\n", "- [Card](02_card.html) - where `k`, `n`, the interval and the delta are written down\n", "- [Gate](03_gate.html) - the condition that reads `lo` against the floor\n", + "- [Measurement](04_measurement.html) - where an on-device accuracy records its interval and `ci_method`\n", "\n", "Tests live in `nbs/tests/test_eval.ipynb`." ] diff --git a/nbs/02_card.ipynb b/nbs/02_card.ipynb index 0e17c64..436b517 100644 --- a/nbs/02_card.ipynb +++ b/nbs/02_card.ipynb @@ -294,6 +294,7 @@ "- [Eval](01_eval.html) - where `k`, `n`, the delta and the agreement come from\n", "- [Gate](03_gate.html) - condition 7 refuses to publish a card `check_card` flags\n", "- [Model](00_model.html) - the artifact the card describes\n", + "- [Measurement](04_measurement.html) - the on-device measurement rows a latency comes from\n", "\n", "Tests live in `nbs/tests/test_card.ipynb`." ] diff --git a/nbs/03_gate.ipynb b/nbs/03_gate.ipynb index c796619..470821f 100644 --- a/nbs/03_gate.ipynb +++ b/nbs/03_gate.ipynb @@ -325,6 +325,7 @@ "- [Model](00_model.html) - `state_hash`, the digest condition 1 compares\n", "- [Card](02_card.html) - `check_card`, which condition 7 runs\n", "- [Eval](01_eval.html) - the paired interval condition 3 reads\n", + "- [Measurement](04_measurement.html) - the validity checks of one on-device measurement row\n", "\n", "Tests live in `nbs/tests/test_gate.ipynb`." ] diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb new file mode 100644 index 0000000..24f0f56 --- /dev/null +++ b/nbs/04_measurement.ipynb @@ -0,0 +1,516 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "frontmatter", + "metadata": {}, + "source": [ + "---\n", + "description: One on-device measurement row, and the checks that decide whether it is well formed and valid\n", + "output-file: measurement.html\n", + "title: Measurement\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "default-exp", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp measurement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "from __future__ import annotations\n", + "\n", + "import json\n", + "import math\n", + "import numbers\n", + "import re\n", + "from datetime import datetime\n", + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "showdoc-setup", + "metadata": {}, + "outputs": [], + "source": [ + "#| include: false\n", + "from nbdev.showdoc import *" + ] + }, + { + "cell_type": "markdown", + "id": "overview", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "A measurement row is one session of one model variant on one device and software stack, at one precision\n", + "and one batch size, on one date. Rows are stored as JSON Lines, one object per line, appended and never\n", + "rewritten; `to_parquet` writes a set of rows as one table for publishing. `FIELDS` is the schema: each\n", + "field's type, whether it is required, and the rules its value obeys. A field that is not required may be\n", + "absent or `None`, which means *not recorded*.\n", + "\n", + "| Group | Fields |\n", + "|---|---|\n", + "| identity and provenance | `schema_version`, `run_id`, `measured_at` (ISO-8601, UTC), `series`, `harness_commit`, `fasterai_version`, `report_url`, `hf_repo` |\n", + "| model | `model_id`, `model_revision`, `task`, `architecture`, `base_weights`, `weights_license`, `variant`, `recipe`, `input_resolution`, `macs`, `params`, `model_hash`, `engine_hash`, `size_mb` |\n", + "| device and runtime | `device`, `device_host`, `runtime`, `runtime_version`, `platform_version`, `precision` (what ran), `int8_mode` (iff `int8`), `power_mode`, `clocks_locked`, `batch_size` |\n", + "| conditions | `temp_start_c`, `temp_end_c`, `gpu_freq_mhz`, `throttled`, `warmup_s`, `duration_s`, `repeats`, `latency_spread_pct` |\n", + "| latency | `latency_ms_p50`, `latency_ms_p90`, `latency_ms_p99`, `latency_scope`, `host_latency_ms`, `latency_source` |\n", + "| quality | `metric`, `eval_set`, `eval_set_hash`, `dataset_license`, `eval_n`, `accuracy`, `ci_low`, `ci_high`, `ci_method`, `measured_through`, `argmax_parity`, `parity_floor`, `baseline_run_id` |\n", + "\n", + "Throughput is derived, never stored: `throughput(row)` is `batch_size * 1000 / latency_ms_p50`.\n", + "\n", + "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of the key.\n", + "- `latency_scope` says what the latency covers, e.g. engine execution only, or host transfers included; two rows are only comparable when it matches.\n", + "- `engine_hash` is pinned to the sha256 of the engine file that ran; `model_hash` is whatever fingerprint the exporter records, so it is only a string.\n", + "\n", + "Two questions are kept apart: `validate` asks whether a row is **well formed**, `status` whether the\n", + "measurement it records is **valid**. Neither compares a row to a latency or accuracy budget: a row carries no\n", + "verdict column, and whoever reads it recomputes their own. A well-formed row is `INVALID` when:\n", + "\n", + "- `measured_through` is not `deployed_engine`: the accuracy was not measured through the engine that was timed;\n", + "- `latency_source` is not `measured`: the latency was not measured on the device;\n", + "- `argmax_parity` is below `parity_floor`;\n", + "- `throttled` is true;\n", + "- `latency_spread_pct` is above `max_spread_pct` (5 by default);\n", + "- any of the last three is not recorded." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "schema", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "SCHEMA_VERSION = 1\n", + "_UTC = r'\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|\\+00:00)'\n", + "_PCT = dict(ge=0, le=100)\n", + "\n", + "# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None\n", + "FIELDS = {\n", + " # identity and provenance\n", + " 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)),\n", + " 'run_id': dict(type=str, required=True),\n", + " 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True),\n", + " 'series': dict(type=str, required=False),\n", + " 'harness_commit': dict(type=str, required=True),\n", + " 'fasterai_version': dict(type=str, required=False),\n", + " 'report_url': dict(type=str, required=False),\n", + " 'hf_repo': dict(type=str, required=False),\n", + " # model\n", + " 'model_id': dict(type=str, required=True),\n", + " 'model_revision': dict(type=str, required=True),\n", + " 'task': dict(type=str, required=True, choices=('classification', 'detection')),\n", + " 'architecture': dict(type=str, required=True),\n", + " 'base_weights': dict(type=str, required=True),\n", + " 'weights_license': dict(type=str, required=True),\n", + " 'variant': dict(type=str, required=True),\n", + " 'recipe': dict(type=dict, required=True, json=True),\n", + " 'input_resolution': dict(type=int, required=True, gt=0),\n", + " 'macs': dict(type=int, required=True, gt=0),\n", + " 'params': dict(type=int, required=True, gt=0),\n", + " 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned\n", + " 'engine_hash': dict(type=str, required=True, pattern=r'[0-9a-f]{64}'), # sha256 of the engine file that ran\n", + " 'size_mb': dict(type=float, required=True, gt=0),\n", + " # device and runtime\n", + " 'device': dict(type=str, required=True),\n", + " 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')),\n", + " 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')),\n", + " 'runtime_version': dict(type=str, required=True),\n", + " 'platform_version': dict(type=str, required=False),\n", + " 'precision': dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')),\n", + " 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')),\n", + " 'power_mode': dict(type=str, required=False),\n", + " 'clocks_locked': dict(type=bool, required=False),\n", + " 'batch_size': dict(type=int, required=True, ge=1),\n", + " # conditions\n", + " 'temp_start_c': dict(type=float, required=False),\n", + " 'temp_end_c': dict(type=float, required=False),\n", + " 'gpu_freq_mhz': dict(type=float, required=False, gt=0),\n", + " 'throttled': dict(type=bool, required=False),\n", + " 'warmup_s': dict(type=float, required=False, ge=0),\n", + " 'duration_s': dict(type=float, required=False, ge=0),\n", + " 'repeats': dict(type=int, required=True, ge=1),\n", + " 'latency_spread_pct': dict(type=float, required=False, ge=0),\n", + " # latency\n", + " 'latency_ms_p50': dict(type=float, required=True, gt=0),\n", + " 'latency_ms_p90': dict(type=float, required=False, gt=0),\n", + " 'latency_ms_p99': dict(type=float, required=False, gt=0),\n", + " 'latency_scope': dict(type=str, required=True),\n", + " 'host_latency_ms': dict(type=float, required=False, gt=0),\n", + " 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')),\n", + " # quality\n", + " 'metric': dict(type=str, required=True),\n", + " 'eval_set': dict(type=str, required=True),\n", + " 'eval_set_hash': dict(type=str, required=True),\n", + " 'dataset_license': dict(type=str, required=True),\n", + " 'eval_n': dict(type=int, required=True, gt=0),\n", + " 'accuracy': dict(type=float, required=True, **_PCT),\n", + " 'ci_low': dict(type=float, required=True, **_PCT),\n", + " 'ci_high': dict(type=float, required=True, **_PCT),\n", + " 'ci_method': dict(type=str, required=True, choices=('wilson', 'bootstrap')),\n", + " 'measured_through': dict(type=str, required=True, choices=('deployed_engine', 'dev_model')),\n", + " 'argmax_parity': dict(type=float, required=False, **_PCT),\n", + " 'parity_floor': dict(type=float, required=True, **_PCT),\n", + " 'baseline_run_id': dict(type=str, required=False),\n", + "}\n", + "\n", + "def _item(o):\n", + " \"A numpy scalar as the Python number JSON knows\"\n", + " if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item()\n", + " raise TypeError(f'{type(o).__name__} is not JSON serializable')\n", + "\n", + "\n", + "def _json(obj, **kw): return json.dumps(obj, default=_item, allow_nan=False, **kw)\n", + "def _canonical(d): return _json(d, sort_keys=True, separators=(',', ':'))\n", + "\n", + "\n", + "def _is_json(v):\n", + " try: return _json(v) is not None\n", + " except (TypeError, ValueError): return False\n", + "\n", + "\n", + "def _is_date(v):\n", + " try: return datetime.fromisoformat(v.replace('Z', '+00:00')) is not None # Python 3.10 does not read 'Z'\n", + " except ValueError: return False\n", + "\n", + "\n", + "_RULES = {\n", + " 'choices': (lambda v, c: v in c, 'not one of {}'),\n", + " 'gt': (lambda v, b: v > b, 'not > {}'),\n", + " 'ge': (lambda v, b: v >= b, 'not >= {}'),\n", + " 'le': (lambda v, b: v <= b, 'not <= {}'),\n", + " 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'),\n", + " 'date': (lambda v, _: _is_date(v), 'is not a date'),\n", + " 'json': (lambda v, _: _is_json(v), 'is not JSON'),\n", + "}\n", + "_ORDERED = (('ci_low', 'accuracy', 'ci_high'), ('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'))\n", + "\n", + "\n", + "def _typed(v, t):\n", + " \"`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite\"\n", + " if isinstance(v, bool): return t is bool\n", + " if t is int: return isinstance(v, numbers.Integral)\n", + " if t is float: return isinstance(v, numbers.Real) and math.isfinite(v)\n", + " return isinstance(v, t)\n", + "\n", + "\n", + "def _field_problems(name, v):\n", + " \"The problems of one field value, against its spec\"\n", + " spec = FIELDS[name]\n", + " if v is None: return [f'{name}: missing'] if spec['required'] else []\n", + " if not _typed(v, spec['type']): return [f'{name}: expected {spec[\"type\"].__name__}, got {v!r}']\n", + " return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])]\n", + "\n", + "\n", + "def _cross_problems(row):\n", + " \"Problems between fields; each field involved is already well typed\"\n", + " p, mode = row['precision'], row.get('int8_mode')\n", + " out = []\n", + " if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8')\n", + " if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}')\n", + " for fields in _ORDERED:\n", + " vals = [(f, row[f]) for f in fields if row.get(f) is not None]\n", + " if [v for _, v in vals] != sorted(v for _, v in vals):\n", + " out.append(f\"{', '.join(fields)}: not in order, \" + ', '.join(f'{f}={v}' for f, v in vals))\n", + " return out\n", + "\n", + "\n", + "def validate(\n", + " row: dict, # one measurement row\n", + ") -> list[str]:\n", + " \"Every structural problem of `row`, each prefixed by its field name; an empty list means well formed\"\n", + " if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}']\n", + " out = [f'{k}: unknown field' for k in row if k not in FIELDS]\n", + " for name in FIELDS: out += _field_problems(name, row.get(name))\n", + " return out or _cross_problems(row)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-validate", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(validate)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "status", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "# field, when its value makes the measurement invalid, and why; a field that is None is not recorded\n", + "_VALIDITY = (\n", + " ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'),\n", + " ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'),\n", + " ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'),\n", + " ('throttled', lambda v, r, m: v, 'the device throttled during the session'),\n", + " ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'),\n", + ")\n", + "\n", + "\n", + "def status(\n", + " row: dict, # one measurement row\n", + " max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent\n", + ") -> tuple[str, list[str]]:\n", + " \"('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget\"\n", + " reasons = validate(row)\n", + " if reasons: return 'INVALID', reasons\n", + " for f, bad, why in _VALIDITY:\n", + " v = row.get(f)\n", + " if v is None: reasons.append(f'{f}: not recorded')\n", + " elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct))\n", + " return ('INVALID', reasons) if reasons else ('VALID', [])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-status", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(status)" + ] + }, + { + "cell_type": "markdown", + "id": "status-note", + "metadata": {}, + "source": [ + "An unrecorded condition is not a clean one: a row that does not say whether the device throttled, what\n", + "its latency spread was, or how its outputs agreed with the reference is `INVALID`, with the reason\n", + "`not recorded`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "key", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def key(\n", + " row: dict, # one measurement row\n", + ") -> tuple:\n", + " \"The identity a row is unique on; a software update on the device gives a new key\"\n", + " return (row['model_id'], row['model_revision'], row['variant'], _canonical(row['recipe']),\n", + " row['input_resolution'], row['device'], row['runtime'], row['runtime_version'],\n", + " row.get('platform_version'), row['precision'], row.get('int8_mode'), row['batch_size'],\n", + " row['measured_at'])\n", + "\n", + "\n", + "def throughput(\n", + " row: dict, # one measurement row\n", + ") -> float:\n", + " \"Images per second at the timed batch, derived from the median latency and never stored\"\n", + " return row['batch_size'] * 1000 / row['latency_ms_p50']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-key", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-throughput", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(throughput)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "io", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _parse(text, path):\n", + " \"The rows of a JSON Lines text; a malformed line raises with its number\"\n", + " rows = []\n", + " for i, line in enumerate(text.splitlines(), 1):\n", + " if not line.strip(): continue\n", + " try: r = json.loads(line)\n", + " except json.JSONDecodeError as e: raise ValueError(f'{path}:{i}: not JSON ({e.msg})') from None\n", + " if not isinstance(r, dict): raise ValueError(f'{path}:{i}: not a JSON object')\n", + " rows.append(r)\n", + " return rows\n", + "\n", + "\n", + "def read_rows(\n", + " path: str | Path, # a JSON Lines file of measurement rows\n", + ") -> list[dict]:\n", + " \"The rows of `path`, as stored; `status` judges them\"\n", + " return _parse(Path(path).read_text(), path)\n", + "\n", + "\n", + "def write_row(\n", + " path: str | Path, # a JSON Lines file, created if absent\n", + " row: dict, # the measurement row to append\n", + "):\n", + " \"Append `row` to `path`; refuses a malformed row and a row whose key is already in the file\"\n", + " problems = validate(row)\n", + " if problems: raise ValueError('row refused: ' + '; '.join(problems))\n", + " p, k = Path(path), key(row)\n", + " text = p.read_text() if p.exists() else ''\n", + " if k in {key(r) for r in _parse(text, path)}: raise ValueError(f'row refused: key {k} already in {path}')\n", + " # ponytail: no file lock, one writer at a time\n", + " with p.open('a') as f: f.write(('\\n' if text and not text.endswith('\\n') else '') + _json(row) + '\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-read-rows", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(read_rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-write-row", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(write_row)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "parquet", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def to_parquet(\n", + " rows: list[dict], # well-formed measurement rows, e.g. from `read_rows`\n", + " path: str | Path, # the Parquet file to write\n", + "):\n", + " \"Write `rows` as one Parquet table, one column per field in `FIELDS` order, dict fields as canonical JSON\"\n", + " try:\n", + " import pandas as pd\n", + " import pyarrow # noqa: F401\n", + " except ImportError: raise ImportError(\"to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'\") from None\n", + " for i, r in enumerate(rows):\n", + " problems = validate(r)\n", + " if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems))\n", + " table = [{k: _canonical(r[k]) if FIELDS[k]['type'] is dict else r.get(k) for k in FIELDS} for r in rows]\n", + " pd.DataFrame(table, columns=list(FIELDS)).to_parquet(path, engine='pyarrow', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-to-parquet", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(to_parquet)" + ] + }, + { + "cell_type": "markdown", + "id": "parquet-note", + "metadata": {}, + "source": [ + "`to_parquet` needs the optional `parquet` extra: `pip install 'fastermodels[parquet]'`." + ] + }, + { + "cell_type": "markdown", + "id": "usage", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Usage\n", + "\n", + "```python\n", + "from fastermodels import measurement\n", + "\n", + "row = dict(schema_version=1, run_id='2026-09-28-a', measured_at='2026-09-28T10:00:00Z',\n", + " harness_commit='deadbee', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", + " task='classification', architecture='resnet50', base_weights='timm/resnet50.a1_in1k',\n", + " weights_license='apache-2.0', variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512},\n", + " input_resolution=224, macs=2_900_000_000, params=17_000_000, engine_hash='0' * 64, size_mb=18.2,\n", + " device='jetson-nano-4gb', device_host='own', runtime='tensorrt', runtime_version='8.2.1',\n", + " platform_version='jetpack-4.6.1', precision='int8', int8_mode='explicit', batch_size=1,\n", + " throttled=False, repeats=5, latency_spread_pct=1.2, latency_ms_p50=2.1,\n", + " latency_scope='engine execute, host transfers excluded', latency_source='measured',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:...', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0)\n", + "\n", + "measurement.validate(row) # [] : well formed\n", + "measurement.status(row) # ('VALID', [])\n", + "measurement.throughput(row) # 476.19... images per second, derived\n", + "measurement.write_row('rows.jsonl', row) # appends one line; a second call with the same row is refused\n", + "measurement.to_parquet(measurement.read_rows('rows.jsonl'), 'rows.parquet')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "see-also", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## See Also\n", + "\n", + "- [Model](00_model.html) - the published model a row measures\n", + "- [Eval](01_eval.html) - `wilson`, the interval `ci_method='wilson'` refers to, and `agreement`\n", + "- [Card](02_card.html) - where a published latency is written down\n", + "- [Gate](03_gate.html) - the conditions an artifact meets before it is published\n", + "\n", + "Tests live in `nbs/tests/test_measurement.ipynb`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/_quarto.yml b/nbs/_quarto.yml index 7b58c57..b23b528 100644 --- a/nbs/_quarto.yml +++ b/nbs/_quarto.yml @@ -34,5 +34,6 @@ website: - 01_eval.ipynb - 02_card.ipynb - 03_gate.ipynb + - 04_measurement.ipynb metadata-files: [nbdev.yml] diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb new file mode 100644 index 0000000..94682ce --- /dev/null +++ b/nbs/tests/test_measurement.ipynb @@ -0,0 +1,373 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "title", + "metadata": {}, + "source": [ + "# Tests: `fastermodels.measurement` (source `nbs/04_measurement.ipynb`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "from fastcore.test import *\n", + "import json, tempfile\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "\n", + "import fastermodels\n", + "from fastermodels import measurement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fixture", + "metadata": {}, + "outputs": [], + "source": [ + "_ROW = dict(schema_version=1, run_id='r1', measured_at='2026-09-28T10:00:00Z', series='2026-09-28',\n", + " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None, hf_repo=None,\n", + " model_id='org/resnet50-p30', model_revision='a1b2c3d', task='classification',\n", + " architecture='resnet50', base_weights='timm/resnet50.a1_in1k', weights_license='apache-2.0',\n", + " variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512}, input_resolution=224,\n", + " macs=2_900_000_000, params=17_000_000, model_hash=None, engine_hash='ab' * 32, size_mb=18.2,\n", + " device='jetson-nano-4gb', device_host='own', runtime='tensorrt', runtime_version='8.2.1',\n", + " platform_version='jetpack-4.6.1', precision='int8', int8_mode='explicit', power_mode='MAXN',\n", + " clocks_locked=True, batch_size=4,\n", + " temp_start_c=41.0, temp_end_c=47.5, gpu_freq_mhz=921.6, throttled=False, warmup_s=10.0,\n", + " duration_s=60.0, repeats=5, latency_spread_pct=1.2,\n", + " latency_ms_p50=8.0, latency_ms_p90=8.4, latency_ms_p99=9.1, latency_scope='engine execute',\n", + " host_latency_ms=8.5, latency_source='measured',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:0f', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0, baseline_run_id='r0')\n", + "\n", + "\n", + "def problems(**change):\n", + " \"measurement.validate() on the valid row with some fields changed; a value of `...` removes the field\"\n", + " r = {**_ROW, **change}\n", + " return measurement.validate({k: v for k, v in r.items() if v is not ...})\n", + "\n", + "\n", + "def names(ps): return {p.split(':')[0] for p in ps}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "valid", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(set(_ROW), set(measurement.FIELDS))\n", + "test_eq(measurement.validate(_ROW), [])\n", + "test_eq(measurement.SCHEMA_VERSION, 1)\n", + "# optional fields may be absent: absent means not recorded\n", + "test_eq(problems(series=..., fasterai_version=..., platform_version=..., temp_start_c=..., latency_ms_p90=...,\n", + " latency_ms_p99=..., baseline_run_id=..., model_hash=...), [])\n", + "# JSON writes 8 as 8: an int is accepted where a float is expected\n", + "test_eq(problems(latency_ms_p50=8), [])\n", + "# numpy scalars, as a harness produces them, are numbers; a numpy or Python bool is not\n", + "test_eq(problems(batch_size=np.int64(4), eval_n=np.int32(50_000), macs=np.uint64(2_900_000_000)), [])\n", + "test_eq(problems(latency_ms_p50=np.float64(8.0), accuracy=np.float32(78.9)), [])\n", + "test_eq(names(problems(batch_size=np.bool_(True))), {'batch_size'})\n", + "test_eq(names(problems(latency_ms_p50=np.float64('nan'))), {'latency_ms_p50'})\n", + "# the module is reached by its name, its generic function names are not at the package top level\n", + "test_eq(fastermodels.measurement is measurement, True)\n", + "test_eq(hasattr(fastermodels, 'status') or hasattr(fastermodels, 'key'), False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fields", + "metadata": {}, + "outputs": [], + "source": [ + "# missing required, by name\n", + "test_eq(problems(run_id=...), ['run_id: missing'])\n", + "test_eq(problems(latency_ms_p50=None), ['latency_ms_p50: missing'])\n", + "test_eq(problems(weights_license=...), ['weights_license: missing'])\n", + "test_eq(names(problems(model_id=..., device=...)), {'model_id', 'device'})\n", + "\n", + "# wrong type\n", + "test_eq(problems(batch_size='4'), [\"batch_size: expected int, got '4'\"])\n", + "test_eq(names(problems(recipe=[0.3])), {'recipe'})\n", + "test_eq(names(problems(variant={'prune': 0.3})), {'variant'})\n", + "test_eq(names(problems(runtime_version=8.2)), {'runtime_version'})\n", + "test_eq(names(problems(throttled='no')), {'throttled'})\n", + "test_eq(names(problems(batch_size=4.0)), {'batch_size'})\n", + "test_eq(names(problems(latency_ms_p50=float('nan'))), {'latency_ms_p50'})\n", + "test_eq(names(problems(latency_ms_p50=float('inf'))), {'latency_ms_p50'})\n", + "\n", + "# a bool is an int in Python, and neither a count nor a measure here\n", + "test_eq(problems(batch_size=True), ['batch_size: expected int, got True'])\n", + "test_eq(names(problems(schema_version=True)), {'schema_version'})\n", + "test_eq(names(problems(latency_ms_p50=True)), {'latency_ms_p50'})\n", + "test_eq(names(problems(macs=False)), {'macs'})\n", + "test_eq(names(problems(input_resolution=True)), {'input_resolution'})\n", + "\n", + "# out of range\n", + "test_eq(names(problems(batch_size=0)), {'batch_size'})\n", + "test_eq(names(problems(repeats=0)), {'repeats'})\n", + "test_eq(names(problems(input_resolution=0)), {'input_resolution'})\n", + "test_eq(names(problems(latency_ms_p50=0.0)), {'latency_ms_p50'})\n", + "test_eq(names(problems(latency_ms_p99=0.0)), {'latency_ms_p99'})\n", + "test_eq(names(problems(size_mb=-1.0)), {'size_mb'})\n", + "test_eq(names(problems(latency_spread_pct=-0.1)), {'latency_spread_pct'})\n", + "test_eq(names(problems(warmup_s=-1.0)), {'warmup_s'})\n", + "test_eq(names(problems(duration_s=-1.0)), {'duration_s'})\n", + "test_eq(problems(warmup_s=0), [])\n", + "test_eq(names(problems(host_latency_ms=0)), {'host_latency_ms'})\n", + "test_eq(names(problems(eval_n=0)), {'eval_n'})\n", + "test_eq(names(problems(argmax_parity=100.5)), {'argmax_parity'})\n", + "test_eq(names(problems(accuracy=150.0, ci_high=160.0)), {'accuracy', 'ci_high'})\n", + "\n", + "# closed vocabularies\n", + "test_eq(names(problems(precision='bf16', int8_mode=None)), {'precision'})\n", + "test_eq(names(problems(task='segmentation')), {'task'})\n", + "test_eq(names(problems(ci_method='normal')), {'ci_method'})\n", + "test_eq(names(problems(runtime='TensorRT 8.2.1')), {'runtime'})\n", + "test_eq(names(problems(device_host='borrowed')), {'device_host'})\n", + "test_eq(names(problems(latency_source='guessed')), {'latency_source'})\n", + "test_eq(names(problems(measured_through='notebook')), {'measured_through'})\n", + "\n", + "# schema version mismatch\n", + "test_eq(problems(schema_version=2), ['schema_version: 2 not one of (1,)'])\n", + "\n", + "# unknown fields: a derived throughput, a verdict, and a name from before the merge\n", + "test_eq(problems(throughput_img_s=500.0), ['throughput_img_s: unknown field'])\n", + "test_eq(problems(passed=True), ['passed: unknown field'])\n", + "test_eq(problems(latency_ms=8.0), ['latency_ms: unknown field'])\n", + "\n", + "# not a row at all\n", + "test_eq(measurement.validate([_ROW]), ['row: expected a dict, got list'])\n", + "test_eq(measurement.validate(None), ['row: expected a dict, got NoneType'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cross", + "metadata": {}, + "outputs": [], + "source": [ + "# int8_mode is required iff the precision is int8\n", + "test_eq(problems(int8_mode=None), ['int8_mode: missing, required when precision is int8'])\n", + "test_eq(problems(int8_mode=...), ['int8_mode: missing, required when precision is int8'])\n", + "test_eq(problems(precision='fp16'), [\"int8_mode: 'explicit' given for precision 'fp16'\"])\n", + "test_eq(problems(precision='mixed'), [\"int8_mode: 'explicit' given for precision 'mixed'\"])\n", + "test_eq(problems(precision='fp16', int8_mode=None), [])\n", + "test_eq(problems(precision='mixed', int8_mode=None), [])\n", + "test_eq(names(problems(int8_mode='dynamic')), {'int8_mode'})\n", + "\n", + "# the interval brackets the accuracy\n", + "test_eq(names(problems(accuracy=80.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(names(problems(ci_low=79.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(problems(ci_low=78.9, ci_high=78.9), [])\n", + "\n", + "# latency percentiles are ordered, among those recorded\n", + "test_eq(names(problems(latency_ms_p90=7.9)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(names(problems(latency_ms_p99=8.3)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(names(problems(latency_ms_p90=..., latency_ms_p99=7.0)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(problems(latency_ms_p90=8.0, latency_ms_p99=8.0), [])\n", + "test_eq(problems(latency_ms_p90=None), [])\n", + "\n", + "# the engine digest is 64 lowercase hex characters\n", + "test_eq(names(problems(engine_hash='ab' * 31)), {'engine_hash'})\n", + "test_eq(names(problems(engine_hash='AB' * 32)), {'engine_hash'})\n", + "test_eq(names(problems(engine_hash='sha256:' + 'ab' * 32)), {'engine_hash'})\n", + "\n", + "# the date is ISO-8601 in UTC, and a real date\n", + "test_eq(problems(measured_at='2026-09-28T10:00:00.123+00:00'), [])\n", + "test_eq(names(problems(measured_at='2026-09-28')), {'measured_at'})\n", + "test_eq(names(problems(measured_at='2026-09-28T10:00:00+02:00')), {'measured_at'})\n", + "test_eq(names(problems(measured_at='2026-13-28T10:00:00Z')), {'measured_at'})\n", + "\n", + "# recipe keys are free, but the recipe is stored as JSON\n", + "test_eq(problems(recipe={}), [])\n", + "test_eq(names(problems(recipe={'x': object()})), {'recipe'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "status", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(measurement.status(_ROW), ('VALID', []))\n", + "\n", + "\n", + "def reasons(**change):\n", + " \"measurement.status() reasons for the valid row with some fields changed\"\n", + " v, rs = measurement.status({k: v for k, v in {**_ROW, **change}.items() if v is not ...})\n", + " test_eq(v, 'INVALID')\n", + " return rs\n", + "\n", + "\n", + "test_eq(reasons(measured_through='dev_model'), ['measured_through: accuracy not measured through the deployed engine'])\n", + "test_eq(reasons(latency_source='estimated'), ['latency_source: latency not measured on the device'])\n", + "test_eq(reasons(argmax_parity=98.9), ['argmax_parity: 98.9 below parity_floor 99.0'])\n", + "test_eq(reasons(argmax_parity=None), ['argmax_parity: not recorded'])\n", + "test_eq(reasons(argmax_parity=...), ['argmax_parity: not recorded'])\n", + "test_eq(measurement.status({**_ROW, 'argmax_parity': 99.0}), ('VALID', [])) # at the floor holds\n", + "test_eq(reasons(throttled=True), ['throttled: the device throttled during the session'])\n", + "test_eq(reasons(throttled=None), ['throttled: not recorded'])\n", + "test_eq(reasons(latency_spread_pct=5.1), ['latency_spread_pct: 5.1 above 5.0'])\n", + "test_eq(reasons(latency_spread_pct=None), ['latency_spread_pct: not recorded'])\n", + "test_eq(measurement.status({**_ROW, 'latency_spread_pct': 5.0}), ('VALID', []))\n", + "test_eq(measurement.status({**_ROW, 'latency_spread_pct': 5.1}, max_spread_pct=6.0), ('VALID', []))\n", + "\n", + "# the informational conditions do not decide validity\n", + "test_eq(measurement.status({**_ROW, 'temp_start_c': None, 'temp_end_c': None, 'gpu_freq_mhz': None,\n", + " 'clocks_locked': None, 'warmup_s': None}), ('VALID', []))\n", + "\n", + "# every reason is reported, not only the first\n", + "test_eq(names(reasons(throttled=None, latency_spread_pct=None, argmax_parity=None, latency_source='estimated',\n", + " measured_through='dev_model')),\n", + " {'throttled', 'latency_spread_pct', 'argmax_parity', 'latency_source', 'measured_through'})\n", + "\n", + "# a malformed row is invalid for its structural problems\n", + "test_eq(reasons(batch_size=True), ['batch_size: expected int, got True'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "key-throughput", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(measurement.key(_ROW), measurement.key(dict(_ROW)))\n", + "# the recipe is canonical: key order does not make a new row\n", + "test_eq(measurement.key({**_ROW, 'recipe': {'calib_n': 512, 'quant': 'int8', 'prune': 0.30}}), measurement.key(_ROW))\n", + "# a software update on the device is a new key\n", + "test_ne(measurement.key({**_ROW, 'runtime_version': '8.5.2'}), measurement.key(_ROW))\n", + "test_ne(measurement.key({**_ROW, 'platform_version': 'jetpack-5.1.3'}), measurement.key(_ROW))\n", + "for _f, _v in [('batch_size', 1), ('recipe', {'prune': 0.40}), ('variant', 'p40-int8'), ('input_resolution', 160),\n", + " ('device', 'raspberry-pi-5'), ('runtime', 'onnxruntime'), ('measured_at', '2026-09-29T10:00:00Z'),\n", + " ('model_revision', 'e4f5a6b')]:\n", + " test_ne(measurement.key({**_ROW, _f: _v}), measurement.key(_ROW))\n", + "# fields outside the identity do not\n", + "test_eq(measurement.key({**_ROW, 'latency_ms_p50': 9.0, 'run_id': 'r2', 'series': None}), measurement.key(_ROW))\n", + "\n", + "# throughput is derived: batch_size * 1000 / latency_ms_p50\n", + "test_eq(measurement.throughput(_ROW), 500.0)\n", + "test_close(measurement.throughput({**_ROW, 'batch_size': 1, 'latency_ms_p50': 2.1}), 476.19, eps=0.01)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "io", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as _d:\n", + " _p = Path(_d, 'rows.jsonl')\n", + "\n", + " # a valid row is appended, and reads back equal\n", + " measurement.write_row(_p, _ROW)\n", + " test_eq(measurement.read_rows(_p), [_ROW])\n", + " _r2 = {**_ROW, 'runtime_version': '8.5.2', 'run_id': 'r2'}\n", + " measurement.write_row(str(_p), _r2)\n", + " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", + " test_eq(len(_p.read_text().splitlines()), 2)\n", + "\n", + " # an invalid row is refused, naming its problems, and nothing is written\n", + " with ExceptionExpected(ValueError, 'batch_size'): measurement.write_row(_p, {**_ROW, 'batch_size': 0, 'run_id': 'r3'})\n", + " with ExceptionExpected(ValueError, 'throughput_img_s'): measurement.write_row(_p, {**_ROW, 'throughput_img_s': 1.0})\n", + " # a duplicate key is refused, even with other fields changed\n", + " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_ROW, 'run_id': 'again', 'latency_ms_p50': 7.0})\n", + " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", + "\n", + " # numpy scalars are written as plain JSON numbers\n", + " _np = {**_ROW, 'run_id': 'r4', 'batch_size': np.int64(2), 'latency_ms_p50': np.float64(4.0),\n", + " 'recipe': {'prune': np.float64(0.3), 'calib_n': np.int64(512)}}\n", + " measurement.write_row(_p, _np)\n", + " test_eq(json.loads(_p.read_text().splitlines()[-1])['batch_size'], 2)\n", + " test_eq(measurement.read_rows(_p)[-1]['recipe'], {'prune': 0.3, 'calib_n': 512})\n", + " _p.write_text(''.join(l + '\\n' for l in _p.read_text().splitlines()[:-1]))\n", + " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", + "\n", + " # a row written by hand without a final newline does not swallow the next one\n", + " _p.write_text(_p.read_text().rstrip('\\n'))\n", + " _r3 = {**_ROW, 'batch_size': 1}\n", + " measurement.write_row(_p, _r3)\n", + " test_eq(measurement.read_rows(_p), [_ROW, _r2, _r3])\n", + "\n", + " # a malformed line is reported with its line number; blank lines are skipped\n", + " _bad = Path(_d, 'bad.jsonl')\n", + " _bad.write_text(json.dumps(_ROW) + '\\n\\n{\"run_id\": \\n')\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.read_rows(_bad)\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.write_row(_bad, _r2)\n", + " _bad.write_text('[1, 2]\\n')\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:1: not a JSON object'): measurement.read_rows(_bad)\n", + "\n", + " # reading a file that is not there is an error, not an empty table\n", + " with ExceptionExpected(FileNotFoundError): measurement.read_rows(Path(_d, 'none.jsonl'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "parquet", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util, sys\n", + "\n", + "# without the extra installed, a one-line error names it (pandas blocked before any import of it)\n", + "_saved = sys.modules.get('pandas')\n", + "sys.modules['pandas'] = None\n", + "try:\n", + " with ExceptionExpected(ImportError, r\"fastermodels\\[parquet\\]\"): measurement.to_parquet([_ROW], 'x.parquet')\n", + "finally:\n", + " if _saved is None: sys.modules.pop('pandas', None)\n", + " else: sys.modules['pandas'] = _saved\n", + "\n", + "if importlib.util.find_spec('pyarrow') and importlib.util.find_spec('pandas'):\n", + " import pandas as pd\n", + " with tempfile.TemporaryDirectory() as _d:\n", + " _pq = Path(_d, 'rows.parquet')\n", + " _rows = [_ROW, {**_ROW, 'run_id': 'r2', 'precision': 'fp16', 'int8_mode': None,\n", + " 'recipe': {'quant': 'none', 'prune': 0.30}}]\n", + " _sparse = {k: v for k, v in _ROW.items() if k not in ('series', 'latency_ms_p99')}\n", + " measurement.to_parquet(_rows + [{**_sparse, 'run_id': 'r3'}], _pq)\n", + " _df = pd.read_parquet(_pq)\n", + " # one column per field, in schema order, one line per row\n", + " test_eq(list(_df.columns), list(measurement.FIELDS))\n", + " test_eq(len(_df), 3)\n", + " # the recipe is canonical JSON: key order is not a difference\n", + " test_eq(_df['recipe'][0], '{\"calib_n\":512,\"prune\":0.3,\"quant\":\"int8\"}')\n", + " test_eq(json.loads(_df['recipe'][1]), {'prune': 0.30, 'quant': 'none'})\n", + " test_eq(_df['latency_ms_p50'].tolist(), [8.0, 8.0, 8.0])\n", + " test_eq(_df['int8_mode'].tolist(), ['explicit', None, 'explicit'])\n", + " # an absent field is a null, not a hole\n", + " test_eq(_df['series'].isna().tolist(), [False, False, True])\n", + " test_eq(_df['latency_ms_p99'].isna().tolist(), [False, False, True])\n", + " # a malformed row is refused with its index, and nothing is written\n", + " with ExceptionExpected(ValueError, 'row 1 refused'): measurement.to_parquet([_ROW, {**_ROW, 'batch_size': 0}], Path(_d, 'no.parquet'))\n", + " test_eq(Path(_d, 'no.parquet').exists(), False)\n", + "else: print('pandas or pyarrow not installed: to_parquet round trip skipped')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 1c37268..5de415c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = ['torch', 'huggingface_hub>=0.25', 'safetensors', 'numpy'] [project.optional-dependencies] test = ['torchvision', 'torch-pruning', 'onnx', 'fastcore'] -dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore'] +dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore', 'pandas', 'pyarrow'] +parquet = ['pandas', 'pyarrow'] [project.urls] Repository = "https://github.com/FasterAI-Labs/fastermodels" From d26ccfc0b1a90cf9326d933b9b0d9213b3527c5f Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Tue, 29 Sep 2026 11:46:26 +0200 Subject: [PATCH 2/3] Split measurement rows into latency rows and quality rows How fast an engine runs depends on its graph (architecture, widths, numeric precision) and on the device and software stack, not on the values of its weights; how accurate it is depends on the weights and the evaluation set. A single row forced every speed measurement to carry an accuracy. Each row now has a `kind`: - latency: one engine timed on one device. It needs no accuracy, so an engine built from random weights (`weights='random'`) is measured like any other, which is what a latency predictor trains on. Graph fields (`graph_hash`, widths through macs/params, recipe), the engine's layer counts and the precisions it actually ran, device, conditions and latency percentiles. - quality: one artifact's accuracy on one evaluation set, through the engine it names (`engine_hash`, None only for the development model). The identity and provenance fields are defined once and shared. `FIELDS[kind]` is the schema; validate, status and key dispatch on the kind, a missing or unknown kind is one clear problem, and key uniqueness in a file is per kind. `join` pairs the VALID rows of both kinds that share an engine_hash, keeping identity per side (latency_run_id, quality_run_id, ...) and refusing two sides that disagree on the same engine. `to_parquet` writes one kind per file. `graph_hash(onnx_path)` hashes the ONNX graph with every initializer's values removed and its name, shape and type kept: two models of the same shape share it, another width or a renamed weight does not. It needs the new optional `onnx` extra. --- fastermodels/_modidx.py | 4 + fastermodels/measurement.py | 211 +++++++++---- nbs/04_measurement.ipynb | 397 +++++++++++++++++------- nbs/tests/test_measurement.ipynb | 514 ++++++++++++++++++++----------- pyproject.toml | 1 + 5 files changed, 773 insertions(+), 354 deletions(-) diff --git a/fastermodels/_modidx.py b/fastermodels/_modidx.py index 51d5b40..700cdc5 100644 --- a/fastermodels/_modidx.py +++ b/fastermodels/_modidx.py @@ -46,8 +46,12 @@ 'fastermodels.measurement._is_json': ('measurement.html#_is_json', 'fastermodels/measurement.py'), 'fastermodels.measurement._item': ('measurement.html#_item', 'fastermodels/measurement.py'), 'fastermodels.measurement._json': ('measurement.html#_json', 'fastermodels/measurement.py'), + 'fastermodels.measurement._pair': ('measurement.html#_pair', 'fastermodels/measurement.py'), 'fastermodels.measurement._parse': ('measurement.html#_parse', 'fastermodels/measurement.py'), 'fastermodels.measurement._typed': ('measurement.html#_typed', 'fastermodels/measurement.py'), + 'fastermodels.measurement.graph_hash': ( 'measurement.html#graph_hash', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.join': ('measurement.html#join', 'fastermodels/measurement.py'), 'fastermodels.measurement.key': ('measurement.html#key', 'fastermodels/measurement.py'), 'fastermodels.measurement.read_rows': ( 'measurement.html#read_rows', 'fastermodels/measurement.py'), diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py index ff02ba9..251c675 100644 --- a/fastermodels/measurement.py +++ b/fastermodels/measurement.py @@ -3,6 +3,7 @@ # %% ../nbs/04_measurement.ipynb #imports from __future__ import annotations +import hashlib import json import math import numbers @@ -11,47 +12,54 @@ from pathlib import Path # %% auto #0 -__all__ = ['SCHEMA_VERSION', 'FIELDS', 'validate', 'status', 'key', 'throughput', 'read_rows', 'write_row', 'to_parquet'] +__all__ = ['SCHEMA_VERSION', 'FIELDS', 'validate', 'status', 'key', 'throughput', 'join', 'graph_hash', 'read_rows', 'write_row', + 'to_parquet'] # %% ../nbs/04_measurement.ipynb #schema SCHEMA_VERSION = 1 _UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|\+00:00)' +_HEX = r'[0-9a-f]{64}' _PCT = dict(ge=0, le=100) +_PRECISION = dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')) # name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None -FIELDS = { - # identity and provenance +_SHARED = { 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)), + 'kind': dict(type=str, required=True, choices=('latency', 'quality')), 'run_id': dict(type=str, required=True), 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True), 'series': dict(type=str, required=False), 'harness_commit': dict(type=str, required=True), 'fasterai_version': dict(type=str, required=False), 'report_url': dict(type=str, required=False), - 'hf_repo': dict(type=str, required=False), - # model - 'model_id': dict(type=str, required=True), - 'model_revision': dict(type=str, required=True), - 'task': dict(type=str, required=True, choices=('classification', 'detection')), +} + +_LATENCY = { + # graph 'architecture': dict(type=str, required=True), - 'base_weights': dict(type=str, required=True), - 'weights_license': dict(type=str, required=True), - 'variant': dict(type=str, required=True), - 'recipe': dict(type=dict, required=True, json=True), + 'graph_hash': dict(type=str, required=True, pattern=_HEX), # `graph_hash`: the ONNX graph without its weight values 'input_resolution': dict(type=int, required=True, gt=0), 'macs': dict(type=int, required=True, gt=0), 'params': dict(type=int, required=True, gt=0), - 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned - 'engine_hash': dict(type=str, required=True, pattern=r'[0-9a-f]{64}'), # sha256 of the engine file that ran + 'variant': dict(type=str, required=True), + 'recipe': dict(type=dict, required=True, json=True), + 'weights': dict(type=str, required=True, choices=('trained', 'random')), + 'model_id': dict(type=str, required=False), + 'model_revision': dict(type=str, required=False), + # artifact + 'precision': _PRECISION, + 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')), + 'engine_hash': dict(type=str, required=True, pattern=_HEX), # sha256 of the engine file that ran 'size_mb': dict(type=float, required=True, gt=0), + 'engine_layers': dict(type=int, required=False, gt=0), + 'reformat_layers': dict(type=int, required=False, ge=0), + 'layer_precisions': dict(type=dict, required=False, counts=True), # device and runtime 'device': dict(type=str, required=True), 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')), 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')), 'runtime_version': dict(type=str, required=True), 'platform_version': dict(type=str, required=False), - 'precision': dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')), - 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')), 'power_mode': dict(type=str, required=False), 'clocks_locked': dict(type=bool, required=False), 'batch_size': dict(type=int, required=True, ge=1), @@ -71,7 +79,16 @@ 'latency_scope': dict(type=str, required=True), 'host_latency_ms': dict(type=float, required=False, gt=0), 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')), - # quality +} + +_QUALITY = { + 'model_id': dict(type=str, required=True), + 'model_revision': dict(type=str, required=True), + 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned + 'weights_license': dict(type=str, required=True), + 'engine_hash': dict(type=str, required=False, pattern=_HEX), # the engine the accuracy was measured through + 'precision': _PRECISION, + 'device': dict(type=str, required=False), 'metric': dict(type=str, required=True), 'eval_set': dict(type=str, required=True), 'eval_set_hash': dict(type=str, required=True), @@ -87,6 +104,9 @@ 'baseline_run_id': dict(type=str, required=False), } +FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}} + + def _item(o): "A numpy scalar as the Python number JSON knows" if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item() @@ -107,6 +127,14 @@ def _is_date(v): except ValueError: return False +def _typed(v, t): + "`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite" + if isinstance(v, bool): return t is bool + if t is int: return isinstance(v, numbers.Integral) + if t is float: return isinstance(v, numbers.Real) and math.isfinite(v) + return isinstance(v, t) + + _RULES = { 'choices': (lambda v, c: v in c, 'not one of {}'), 'gt': (lambda v, b: v > b, 'not > {}'), @@ -115,21 +143,15 @@ def _is_date(v): 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'), 'date': (lambda v, _: _is_date(v), 'is not a date'), 'json': (lambda v, _: _is_json(v), 'is not JSON'), + 'counts': (lambda v, _: all(isinstance(k, str) and _typed(n, int) and n >= 0 for k, n in v.items()), + 'is not a {{name: count}} dict'), } -_ORDERED = (('ci_low', 'accuracy', 'ci_high'), ('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99')) - - -def _typed(v, t): - "`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite" - if isinstance(v, bool): return t is bool - if t is int: return isinstance(v, numbers.Integral) - if t is float: return isinstance(v, numbers.Real) and math.isfinite(v) - return isinstance(v, t) +_ORDERED = {'latency': (('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'), ('reformat_layers', 'engine_layers')), + 'quality': (('ci_low', 'accuracy', 'ci_high'),)} -def _field_problems(name, v): +def _field_problems(name, spec, v): "The problems of one field value, against its spec" - spec = FIELDS[name] if v is None: return [f'{name}: missing'] if spec['required'] else [] if not _typed(v, spec['type']): return [f'{name}: expected {spec["type"].__name__}, got {v!r}'] return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])] @@ -137,11 +159,16 @@ def _field_problems(name, v): def _cross_problems(row): "Problems between fields; each field involved is already well typed" - p, mode = row['precision'], row.get('int8_mode') - out = [] - if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8') - if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}') - for fields in _ORDERED: + out, kind = [], row['kind'] + if kind == 'latency': + p, mode = row['precision'], row.get('int8_mode') + if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8') + if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}') + if row['weights'] == 'random' and row.get('model_id') is not None: + out.append(f"model_id: {row['model_id']!r} given for random weights") + elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None: + out.append('engine_hash: missing, required when measured through the deployed engine') + for fields in _ORDERED[kind]: vals = [(f, row[f]) for f in fields if row.get(f) is not None] if [v for _, v in vals] != sorted(v for _, v in vals): out.append(f"{', '.join(fields)}: not in order, " + ', '.join(f'{f}={v}' for f, v in vals)) @@ -149,55 +176,116 @@ def _cross_problems(row): def validate( - row: dict, # one measurement row + row: dict, # one measurement row, of either kind ) -> list[str]: "Every structural problem of `row`, each prefixed by its field name; an empty list means well formed" if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}'] - out = [f'{k}: unknown field' for k in row if k not in FIELDS] - for name in FIELDS: out += _field_problems(name, row.get(name)) + kind = row.get('kind') + if kind is None: return ['kind: missing'] + if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}'] + fields = FIELDS[kind] + out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields] + for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name)) return out or _cross_problems(row) # %% ../nbs/04_measurement.ipynb #status -# field, when its value makes the measurement invalid, and why; a field that is None is not recorded -_VALIDITY = ( - ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'), - ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'), - ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'), - ('throttled', lambda v, r, m: v, 'the device throttled during the session'), - ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'), -) +# per kind: field, when its value makes the measurement invalid, and why; a field that is None is not recorded +_VALIDITY = { + 'latency': ( + ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'), + ('throttled', lambda v, r, m: v, 'the device throttled during the session'), + ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'), + ), + 'quality': ( + ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'), + ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'), + ), +} def status( - row: dict, # one measurement row + row: dict, # one measurement row, of either kind max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent ) -> tuple[str, list[str]]: "('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget" reasons = validate(row) if reasons: return 'INVALID', reasons - for f, bad, why in _VALIDITY: + for f, bad, why in _VALIDITY[row['kind']]: v = row.get(f) if v is None: reasons.append(f'{f}: not recorded') elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct)) return ('INVALID', reasons) if reasons else ('VALID', []) # %% ../nbs/04_measurement.ipynb #key +_KEYS = { + 'latency': ('graph_hash', 'weights', 'precision', 'int8_mode', 'batch_size', 'device', 'runtime', + 'runtime_version', 'platform_version', 'measured_at'), + 'quality': ('model_id', 'model_revision', 'engine_hash', 'eval_set_hash', 'metric', 'measured_at'), +} + + def key( - row: dict, # one measurement row + row: dict, # one well-formed measurement row ) -> tuple: - "The identity a row is unique on; a software update on the device gives a new key" - return (row['model_id'], row['model_revision'], row['variant'], _canonical(row['recipe']), - row['input_resolution'], row['device'], row['runtime'], row['runtime_version'], - row.get('platform_version'), row['precision'], row.get('int8_mode'), row['batch_size'], - row['measured_at']) + "The identity a row is unique on, its kind first; a software update on the device gives a new latency key" + return (row['kind'],) + tuple(row.get(f) for f in _KEYS[row['kind']]) def throughput( - row: dict, # one measurement row + row: dict, # one latency row ) -> float: "Images per second at the timed batch, derived from the median latency and never stored" + if row.get('kind') != 'latency': raise ValueError(f"throughput needs a latency row, got kind {row.get('kind')!r}") return row['batch_size'] * 1000 / row['latency_ms_p50'] +# %% ../nbs/04_measurement.ipynb #join +_OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed +_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree + + +def _pair(lat, qual): + "One joined dict from a latency row and a quality row of the same engine" + out = {'schema_version': lat['schema_version']} + for side, r in (('latency', lat), ('quality', qual)): + out |= {f'{side}_{f}': r.get(f) for f in _OWN} + out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH} + for f in _BOTH: + a, b = lat.get(f), qual.get(f) + if None not in (a, b) and a != b: + raise ValueError(f"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine") + out[f] = b if a is None else a + return out + + +def join( + latency_rows: list[dict], # latency rows; rows of the other kind and invalid rows are skipped + quality_rows: list[dict], # quality rows; rows of the other kind and invalid rows are skipped +) -> list[dict]: + "One dict per pair of VALID latency and quality rows sharing an `engine_hash`" + by_engine = {} + for q in quality_rows: + if q.get('kind') == 'quality' and status(q)[0] == 'VALID': by_engine.setdefault(q['engine_hash'], []).append(q) + return [_pair(l, q) for l in latency_rows if l.get('kind') == 'latency' and status(l)[0] == 'VALID' + for q in by_engine.get(l['engine_hash'], [])] + +# %% ../nbs/04_measurement.ipynb #graph +def graph_hash( + onnx_path: str | Path, # an ONNX file +) -> str: + "sha256 of the ONNX graph with every initializer's values removed and its name, shape and type kept" + try: import onnx + except ImportError: raise ImportError("graph_hash needs onnx: pip install 'fastermodels[onnx]'") from None + m = onnx.load(str(onnx_path), load_external_data=False) + # ponytail: top-level initializers only; Constant nodes and subgraph initializers keep their values + for t in m.graph.initializer: + name, dtype, dims = t.name, t.data_type, list(t.dims) + t.Clear() + t.name, t.data_type = name, dtype + t.dims.extend(dims) + h = hashlib.sha256(m.graph.SerializeToString(deterministic=True)) + for o in sorted((o.domain, o.version) for o in m.opset_import): h.update(f'{o[0]}:{o[1]};'.encode()) + return h.hexdigest() + # %% ../nbs/04_measurement.ipynb #io def _parse(text, path): "The rows of a JSON Lines text; a malformed line raises with its number" @@ -212,7 +300,7 @@ def _parse(text, path): def read_rows( - path: str | Path, # a JSON Lines file of measurement rows + path: str | Path, # a JSON Lines file of measurement rows, of either kind ) -> list[dict]: "The rows of `path`, as stored; `status` judges them" return _parse(Path(path).read_text(), path) @@ -227,22 +315,27 @@ def write_row( if problems: raise ValueError('row refused: ' + '; '.join(problems)) p, k = Path(path), key(row) text = p.read_text() if p.exists() else '' - if k in {key(r) for r in _parse(text, path)}: raise ValueError(f'row refused: key {k} already in {path}') + if k in {key(r) for r in _parse(text, path) if r.get('kind') == row['kind']}: + raise ValueError(f'row refused: key {k} already in {path}') # ponytail: no file lock, one writer at a time with p.open('a') as f: f.write(('\n' if text and not text.endswith('\n') else '') + _json(row) + '\n') # %% ../nbs/04_measurement.ipynb #parquet def to_parquet( - rows: list[dict], # well-formed measurement rows, e.g. from `read_rows` + rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows` path: str | Path, # the Parquet file to write ): - "Write `rows` as one Parquet table, one column per field in `FIELDS` order, dict fields as canonical JSON" + "Write `rows` as one Parquet table, one column per field of their kind, dict fields as canonical JSON" try: import pandas as pd import pyarrow # noqa: F401 except ImportError: raise ImportError("to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'") from None + kinds = sorted({str(r.get('kind')) for r in rows}) + if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or "no rows"}') for i, r in enumerate(rows): problems = validate(r) if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems)) - table = [{k: _canonical(r[k]) if FIELDS[k]['type'] is dict else r.get(k) for k in FIELDS} for r in rows] - pd.DataFrame(table, columns=list(FIELDS)).to_parquet(path, engine='pyarrow', index=False) + fields = FIELDS[kinds[0]] + table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields} + for r in rows] + pd.DataFrame(table, columns=list(fields)).to_parquet(path, engine='pyarrow', index=False) diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb index 24f0f56..8af309d 100644 --- a/nbs/04_measurement.ipynb +++ b/nbs/04_measurement.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "---\n", - "description: One on-device measurement row, and the checks that decide whether it is well formed and valid\n", + "description: On-device latency rows and accuracy rows, and the checks that decide whether each is well formed and valid\n", "output-file: measurement.html\n", "title: Measurement\n", "---" @@ -32,6 +32,7 @@ "#| export\n", "from __future__ import annotations\n", "\n", + "import hashlib\n", "import json\n", "import math\n", "import numbers\n", @@ -58,37 +59,53 @@ "source": [ "## Overview\n", "\n", - "A measurement row is one session of one model variant on one device and software stack, at one precision\n", - "and one batch size, on one date. Rows are stored as JSON Lines, one object per line, appended and never\n", - "rewritten; `to_parquet` writes a set of rows as one table for publishing. `FIELDS` is the schema: each\n", - "field's type, whether it is required, and the rules its value obeys. A field that is not required may be\n", - "absent or `None`, which means *not recorded*.\n", - "\n", - "| Group | Fields |\n", - "|---|---|\n", - "| identity and provenance | `schema_version`, `run_id`, `measured_at` (ISO-8601, UTC), `series`, `harness_commit`, `fasterai_version`, `report_url`, `hf_repo` |\n", - "| model | `model_id`, `model_revision`, `task`, `architecture`, `base_weights`, `weights_license`, `variant`, `recipe`, `input_resolution`, `macs`, `params`, `model_hash`, `engine_hash`, `size_mb` |\n", - "| device and runtime | `device`, `device_host`, `runtime`, `runtime_version`, `platform_version`, `precision` (what ran), `int8_mode` (iff `int8`), `power_mode`, `clocks_locked`, `batch_size` |\n", - "| conditions | `temp_start_c`, `temp_end_c`, `gpu_freq_mhz`, `throttled`, `warmup_s`, `duration_s`, `repeats`, `latency_spread_pct` |\n", - "| latency | `latency_ms_p50`, `latency_ms_p90`, `latency_ms_p99`, `latency_scope`, `host_latency_ms`, `latency_source` |\n", - "| quality | `metric`, `eval_set`, `eval_set_hash`, `dataset_license`, `eval_n`, `accuracy`, `ci_low`, `ci_high`, `ci_method`, `measured_through`, `argmax_parity`, `parity_floor`, `baseline_run_id` |\n", - "\n", - "Throughput is derived, never stored: `throughput(row)` is `batch_size * 1000 / latency_ms_p50`.\n", - "\n", - "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of the key.\n", - "- `latency_scope` says what the latency covers, e.g. engine execution only, or host transfers included; two rows are only comparable when it matches.\n", - "- `engine_hash` is pinned to the sha256 of the engine file that ran; `model_hash` is whatever fingerprint the exporter records, so it is only a string.\n", + "Speed and accuracy do not depend on the same things. How fast an engine runs depends on its **graph** — the\n", + "architecture, the widths, the numeric precision — and on the device and software stack; it does not depend on\n", + "the values of the weights. How accurate it is depends on the **weights** and on the evaluation set. So a\n", + "measurement is one of two kinds of row, and neither needs the other:\n", + "\n", + "- a **latency row** times one engine on one device. It needs no accuracy, so a model with random weights\n", + " (`weights='random'`) is measured like any other: this is how a latency predictor gets trained on\n", + " architectures nobody has trained yet.\n", + "- a **quality row** measures one artifact's accuracy on one evaluation set, through the engine that runs it.\n", + "\n", + "Rows are stored as JSON Lines, one object per line, appended and never rewritten; a file may hold both kinds.\n", + "`FIELDS[kind]` is the schema of each kind: every field's type, whether it is required, and the rules its\n", + "value obeys. A field that is not required may be absent or `None`, which means *not recorded*.\n", + "\n", + "| Kind | Group | Fields |\n", + "|---|---|---|\n", + "| both | identity and provenance | `schema_version`, `kind`, `run_id`, `measured_at` (ISO-8601, UTC), `series`, `harness_commit`, `fasterai_version`, `report_url` |\n", + "| latency | graph | `architecture`, `graph_hash`, `input_resolution`, `macs`, `params`, `variant`, `recipe`, `weights` (`trained` or `random`), `model_id`, `model_revision` |\n", + "| latency | artifact | `precision`, `int8_mode` (iff `int8`), `engine_hash`, `size_mb`, `engine_layers`, `reformat_layers`, `layer_precisions` |\n", + "| latency | device and runtime | `device`, `device_host`, `runtime`, `runtime_version`, `platform_version`, `power_mode`, `clocks_locked`, `batch_size` |\n", + "| latency | conditions | `temp_start_c`, `temp_end_c`, `gpu_freq_mhz`, `throttled`, `warmup_s`, `duration_s`, `repeats`, `latency_spread_pct` |\n", + "| latency | latency | `latency_ms_p50`, `latency_ms_p90`, `latency_ms_p99`, `latency_scope`, `host_latency_ms`, `latency_source` |\n", + "| quality | model and artifact | `model_id`, `model_revision`, `model_hash`, `weights_license`, `engine_hash`, `precision`, `device` |\n", + "| quality | accuracy | `metric`, `eval_set`, `eval_set_hash`, `dataset_license`, `eval_n`, `accuracy`, `ci_low`, `ci_high`, `ci_method`, `measured_through`, `argmax_parity`, `parity_floor`, `baseline_run_id` |\n", + "\n", + "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of any key.\n", + "- `graph_hash` is the sha256 of the ONNX graph with the values of every initializer removed, so two models of the same shape share it, whatever their weights. `graph_hash(onnx_path)` computes it.\n", + "- `latency_scope` says what the latency covers, e.g. engine execution only, or host transfers included; two latency rows are only comparable when it matches.\n", + "- `layer_precisions` counts the engine's layers by the precision they actually ran in, e.g. `{'int8': 52, 'fp16': 9}`.\n", + "- `engine_hash` is the sha256 of the engine file that ran; `model_hash` is whatever fingerprint the exporter records, so it is only a string.\n", + "- A quality row names the engine it was measured through; `engine_hash` may be `None` only for an accuracy measured on the development model.\n", + "\n", + "Throughput is derived, never stored: `throughput(row)` is `batch_size * 1000 / latency_ms_p50` on a latency row.\n", "\n", "Two questions are kept apart: `validate` asks whether a row is **well formed**, `status` whether the\n", "measurement it records is **valid**. Neither compares a row to a latency or accuracy budget: a row carries no\n", "verdict column, and whoever reads it recomputes their own. A well-formed row is `INVALID` when:\n", "\n", - "- `measured_through` is not `deployed_engine`: the accuracy was not measured through the engine that was timed;\n", - "- `latency_source` is not `measured`: the latency was not measured on the device;\n", - "- `argmax_parity` is below `parity_floor`;\n", - "- `throttled` is true;\n", - "- `latency_spread_pct` is above `max_spread_pct` (5 by default);\n", - "- any of the last three is not recorded." + "- latency: `latency_source` is not `measured`; `throttled` is true or not recorded; `latency_spread_pct` is\n", + " above `max_spread_pct` (5 by default) or not recorded;\n", + "- quality: `measured_through` is not `deployed_engine`; `argmax_parity` is below `parity_floor` or not recorded.\n", + "\n", + "`join(latency_rows, quality_rows)` pairs the VALID rows of both kinds that share an `engine_hash`: one dict per\n", + "pair, with the fields of both. The identity and provenance fields are kept per side, as `latency_run_id`,\n", + "`quality_run_id`, `latency_measured_at` and so on; `schema_version` is kept once. The fields both kinds carry\n", + "(`engine_hash`, `precision`, `device`, `model_id`, `model_revision`) are one column: a side that did not\n", + "record one takes the other's value, and two sides that disagree on the same engine raise." ] }, { @@ -101,42 +118,48 @@ "#| export\n", "SCHEMA_VERSION = 1\n", "_UTC = r'\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|\\+00:00)'\n", + "_HEX = r'[0-9a-f]{64}'\n", "_PCT = dict(ge=0, le=100)\n", + "_PRECISION = dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed'))\n", "\n", "# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None\n", - "FIELDS = {\n", - " # identity and provenance\n", + "_SHARED = {\n", " 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)),\n", + " 'kind': dict(type=str, required=True, choices=('latency', 'quality')),\n", " 'run_id': dict(type=str, required=True),\n", " 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True),\n", " 'series': dict(type=str, required=False),\n", " 'harness_commit': dict(type=str, required=True),\n", " 'fasterai_version': dict(type=str, required=False),\n", " 'report_url': dict(type=str, required=False),\n", - " 'hf_repo': dict(type=str, required=False),\n", - " # model\n", - " 'model_id': dict(type=str, required=True),\n", - " 'model_revision': dict(type=str, required=True),\n", - " 'task': dict(type=str, required=True, choices=('classification', 'detection')),\n", + "}\n", + "\n", + "_LATENCY = {\n", + " # graph\n", " 'architecture': dict(type=str, required=True),\n", - " 'base_weights': dict(type=str, required=True),\n", - " 'weights_license': dict(type=str, required=True),\n", - " 'variant': dict(type=str, required=True),\n", - " 'recipe': dict(type=dict, required=True, json=True),\n", + " 'graph_hash': dict(type=str, required=True, pattern=_HEX), # `graph_hash`: the ONNX graph without its weight values\n", " 'input_resolution': dict(type=int, required=True, gt=0),\n", " 'macs': dict(type=int, required=True, gt=0),\n", " 'params': dict(type=int, required=True, gt=0),\n", - " 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned\n", - " 'engine_hash': dict(type=str, required=True, pattern=r'[0-9a-f]{64}'), # sha256 of the engine file that ran\n", + " 'variant': dict(type=str, required=True),\n", + " 'recipe': dict(type=dict, required=True, json=True),\n", + " 'weights': dict(type=str, required=True, choices=('trained', 'random')),\n", + " 'model_id': dict(type=str, required=False),\n", + " 'model_revision': dict(type=str, required=False),\n", + " # artifact\n", + " 'precision': _PRECISION,\n", + " 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')),\n", + " 'engine_hash': dict(type=str, required=True, pattern=_HEX), # sha256 of the engine file that ran\n", " 'size_mb': dict(type=float, required=True, gt=0),\n", + " 'engine_layers': dict(type=int, required=False, gt=0),\n", + " 'reformat_layers': dict(type=int, required=False, ge=0),\n", + " 'layer_precisions': dict(type=dict, required=False, counts=True),\n", " # device and runtime\n", " 'device': dict(type=str, required=True),\n", " 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')),\n", " 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')),\n", " 'runtime_version': dict(type=str, required=True),\n", " 'platform_version': dict(type=str, required=False),\n", - " 'precision': dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')),\n", - " 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')),\n", " 'power_mode': dict(type=str, required=False),\n", " 'clocks_locked': dict(type=bool, required=False),\n", " 'batch_size': dict(type=int, required=True, ge=1),\n", @@ -156,7 +179,16 @@ " 'latency_scope': dict(type=str, required=True),\n", " 'host_latency_ms': dict(type=float, required=False, gt=0),\n", " 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')),\n", - " # quality\n", + "}\n", + "\n", + "_QUALITY = {\n", + " 'model_id': dict(type=str, required=True),\n", + " 'model_revision': dict(type=str, required=True),\n", + " 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned\n", + " 'weights_license': dict(type=str, required=True),\n", + " 'engine_hash': dict(type=str, required=False, pattern=_HEX), # the engine the accuracy was measured through\n", + " 'precision': _PRECISION,\n", + " 'device': dict(type=str, required=False),\n", " 'metric': dict(type=str, required=True),\n", " 'eval_set': dict(type=str, required=True),\n", " 'eval_set_hash': dict(type=str, required=True),\n", @@ -172,6 +204,9 @@ " 'baseline_run_id': dict(type=str, required=False),\n", "}\n", "\n", + "FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}}\n", + "\n", + "\n", "def _item(o):\n", " \"A numpy scalar as the Python number JSON knows\"\n", " if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item()\n", @@ -192,6 +227,14 @@ " except ValueError: return False\n", "\n", "\n", + "def _typed(v, t):\n", + " \"`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite\"\n", + " if isinstance(v, bool): return t is bool\n", + " if t is int: return isinstance(v, numbers.Integral)\n", + " if t is float: return isinstance(v, numbers.Real) and math.isfinite(v)\n", + " return isinstance(v, t)\n", + "\n", + "\n", "_RULES = {\n", " 'choices': (lambda v, c: v in c, 'not one of {}'),\n", " 'gt': (lambda v, b: v > b, 'not > {}'),\n", @@ -200,21 +243,15 @@ " 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'),\n", " 'date': (lambda v, _: _is_date(v), 'is not a date'),\n", " 'json': (lambda v, _: _is_json(v), 'is not JSON'),\n", + " 'counts': (lambda v, _: all(isinstance(k, str) and _typed(n, int) and n >= 0 for k, n in v.items()),\n", + " 'is not a {{name: count}} dict'),\n", "}\n", - "_ORDERED = (('ci_low', 'accuracy', 'ci_high'), ('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'))\n", - "\n", - "\n", - "def _typed(v, t):\n", - " \"`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite\"\n", - " if isinstance(v, bool): return t is bool\n", - " if t is int: return isinstance(v, numbers.Integral)\n", - " if t is float: return isinstance(v, numbers.Real) and math.isfinite(v)\n", - " return isinstance(v, t)\n", + "_ORDERED = {'latency': (('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'), ('reformat_layers', 'engine_layers')),\n", + " 'quality': (('ci_low', 'accuracy', 'ci_high'),)}\n", "\n", "\n", - "def _field_problems(name, v):\n", + "def _field_problems(name, spec, v):\n", " \"The problems of one field value, against its spec\"\n", - " spec = FIELDS[name]\n", " if v is None: return [f'{name}: missing'] if spec['required'] else []\n", " if not _typed(v, spec['type']): return [f'{name}: expected {spec[\"type\"].__name__}, got {v!r}']\n", " return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])]\n", @@ -222,11 +259,16 @@ "\n", "def _cross_problems(row):\n", " \"Problems between fields; each field involved is already well typed\"\n", - " p, mode = row['precision'], row.get('int8_mode')\n", - " out = []\n", - " if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8')\n", - " if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}')\n", - " for fields in _ORDERED:\n", + " out, kind = [], row['kind']\n", + " if kind == 'latency':\n", + " p, mode = row['precision'], row.get('int8_mode')\n", + " if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8')\n", + " if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}')\n", + " if row['weights'] == 'random' and row.get('model_id') is not None:\n", + " out.append(f\"model_id: {row['model_id']!r} given for random weights\")\n", + " elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None:\n", + " out.append('engine_hash: missing, required when measured through the deployed engine')\n", + " for fields in _ORDERED[kind]:\n", " vals = [(f, row[f]) for f in fields if row.get(f) is not None]\n", " if [v for _, v in vals] != sorted(v for _, v in vals):\n", " out.append(f\"{', '.join(fields)}: not in order, \" + ', '.join(f'{f}={v}' for f, v in vals))\n", @@ -234,12 +276,16 @@ "\n", "\n", "def validate(\n", - " row: dict, # one measurement row\n", + " row: dict, # one measurement row, of either kind\n", ") -> list[str]:\n", " \"Every structural problem of `row`, each prefixed by its field name; an empty list means well formed\"\n", " if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}']\n", - " out = [f'{k}: unknown field' for k in row if k not in FIELDS]\n", - " for name in FIELDS: out += _field_problems(name, row.get(name))\n", + " kind = row.get('kind')\n", + " if kind is None: return ['kind: missing']\n", + " if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}']\n", + " fields = FIELDS[kind]\n", + " out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields]\n", + " for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name))\n", " return out or _cross_problems(row)" ] }, @@ -261,24 +307,28 @@ "outputs": [], "source": [ "#| export\n", - "# field, when its value makes the measurement invalid, and why; a field that is None is not recorded\n", - "_VALIDITY = (\n", - " ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'),\n", - " ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'),\n", - " ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'),\n", - " ('throttled', lambda v, r, m: v, 'the device throttled during the session'),\n", - " ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'),\n", - ")\n", + "# per kind: field, when its value makes the measurement invalid, and why; a field that is None is not recorded\n", + "_VALIDITY = {\n", + " 'latency': (\n", + " ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'),\n", + " ('throttled', lambda v, r, m: v, 'the device throttled during the session'),\n", + " ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'),\n", + " ),\n", + " 'quality': (\n", + " ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'),\n", + " ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'),\n", + " ),\n", + "}\n", "\n", "\n", "def status(\n", - " row: dict, # one measurement row\n", + " row: dict, # one measurement row, of either kind\n", " max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent\n", ") -> tuple[str, list[str]]:\n", " \"('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget\"\n", " reasons = validate(row)\n", " if reasons: return 'INVALID', reasons\n", - " for f, bad, why in _VALIDITY:\n", + " for f, bad, why in _VALIDITY[row['kind']]:\n", " v = row.get(f)\n", " if v is None: reasons.append(f'{f}: not recorded')\n", " elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct))\n", @@ -295,16 +345,6 @@ "show_doc(status)" ] }, - { - "cell_type": "markdown", - "id": "status-note", - "metadata": {}, - "source": [ - "An unrecorded condition is not a clean one: a row that does not say whether the device throttled, what\n", - "its latency spread was, or how its outputs agreed with the reference is `INVALID`, with the reason\n", - "`not recorded`." - ] - }, { "cell_type": "code", "execution_count": null, @@ -313,20 +353,25 @@ "outputs": [], "source": [ "#| export\n", + "_KEYS = {\n", + " 'latency': ('graph_hash', 'weights', 'precision', 'int8_mode', 'batch_size', 'device', 'runtime',\n", + " 'runtime_version', 'platform_version', 'measured_at'),\n", + " 'quality': ('model_id', 'model_revision', 'engine_hash', 'eval_set_hash', 'metric', 'measured_at'),\n", + "}\n", + "\n", + "\n", "def key(\n", - " row: dict, # one measurement row\n", + " row: dict, # one well-formed measurement row\n", ") -> tuple:\n", - " \"The identity a row is unique on; a software update on the device gives a new key\"\n", - " return (row['model_id'], row['model_revision'], row['variant'], _canonical(row['recipe']),\n", - " row['input_resolution'], row['device'], row['runtime'], row['runtime_version'],\n", - " row.get('platform_version'), row['precision'], row.get('int8_mode'), row['batch_size'],\n", - " row['measured_at'])\n", + " \"The identity a row is unique on, its kind first; a software update on the device gives a new latency key\"\n", + " return (row['kind'],) + tuple(row.get(f) for f in _KEYS[row['kind']])\n", "\n", "\n", "def throughput(\n", - " row: dict, # one measurement row\n", + " row: dict, # one latency row\n", ") -> float:\n", " \"Images per second at the timed batch, derived from the median latency and never stored\"\n", + " if row.get('kind') != 'latency': raise ValueError(f\"throughput needs a latency row, got kind {row.get('kind')!r}\")\n", " return row['batch_size'] * 1000 / row['latency_ms_p50']" ] }, @@ -350,6 +395,110 @@ "show_doc(throughput)" ] }, + { + "cell_type": "markdown", + "id": "key-note", + "metadata": {}, + "source": [ + "A latency row is unique on its graph, its weights (`trained` or `random`), precision, batch size,\n", + "device, software stack and date; a quality row on its model, revision, engine, evaluation set, metric and\n", + "date. Uniqueness is checked per kind." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "join", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed\n", + "_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree\n", + "\n", + "\n", + "def _pair(lat, qual):\n", + " \"One joined dict from a latency row and a quality row of the same engine\"\n", + " out = {'schema_version': lat['schema_version']}\n", + " for side, r in (('latency', lat), ('quality', qual)):\n", + " out |= {f'{side}_{f}': r.get(f) for f in _OWN}\n", + " out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH}\n", + " for f in _BOTH:\n", + " a, b = lat.get(f), qual.get(f)\n", + " if None not in (a, b) and a != b:\n", + " raise ValueError(f\"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine\")\n", + " out[f] = b if a is None else a\n", + " return out\n", + "\n", + "\n", + "def join(\n", + " latency_rows: list[dict], # latency rows; rows of the other kind and invalid rows are skipped\n", + " quality_rows: list[dict], # quality rows; rows of the other kind and invalid rows are skipped\n", + ") -> list[dict]:\n", + " \"One dict per pair of VALID latency and quality rows sharing an `engine_hash`\"\n", + " by_engine = {}\n", + " for q in quality_rows:\n", + " if q.get('kind') == 'quality' and status(q)[0] == 'VALID': by_engine.setdefault(q['engine_hash'], []).append(q)\n", + " return [_pair(l, q) for l in latency_rows if l.get('kind') == 'latency' and status(l)[0] == 'VALID'\n", + " for q in by_engine.get(l['engine_hash'], [])]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-join", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(join)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "graph", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def graph_hash(\n", + " onnx_path: str | Path, # an ONNX file\n", + ") -> str:\n", + " \"sha256 of the ONNX graph with every initializer's values removed and its name, shape and type kept\"\n", + " try: import onnx\n", + " except ImportError: raise ImportError(\"graph_hash needs onnx: pip install 'fastermodels[onnx]'\") from None\n", + " m = onnx.load(str(onnx_path), load_external_data=False)\n", + " # ponytail: top-level initializers only; Constant nodes and subgraph initializers keep their values\n", + " for t in m.graph.initializer:\n", + " name, dtype, dims = t.name, t.data_type, list(t.dims)\n", + " t.Clear()\n", + " t.name, t.data_type = name, dtype\n", + " t.dims.extend(dims)\n", + " h = hashlib.sha256(m.graph.SerializeToString(deterministic=True))\n", + " for o in sorted((o.domain, o.version) for o in m.opset_import): h.update(f'{o[0]}:{o[1]};'.encode())\n", + " return h.hexdigest()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-graph-hash", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(graph_hash)" + ] + }, + { + "cell_type": "markdown", + "id": "graph-note", + "metadata": {}, + "source": [ + "Initializer names are part of the graph: renaming a weight gives a new hash, as it does to the\n", + "engine a runtime builds from it. Opset versions are hashed too. `graph_hash` needs the optional `onnx`\n", + "extra: `pip install 'fastermodels[onnx]'`." + ] + }, { "cell_type": "code", "execution_count": null, @@ -371,7 +520,7 @@ "\n", "\n", "def read_rows(\n", - " path: str | Path, # a JSON Lines file of measurement rows\n", + " path: str | Path, # a JSON Lines file of measurement rows, of either kind\n", ") -> list[dict]:\n", " \"The rows of `path`, as stored; `status` judges them\"\n", " return _parse(Path(path).read_text(), path)\n", @@ -386,7 +535,8 @@ " if problems: raise ValueError('row refused: ' + '; '.join(problems))\n", " p, k = Path(path), key(row)\n", " text = p.read_text() if p.exists() else ''\n", - " if k in {key(r) for r in _parse(text, path)}: raise ValueError(f'row refused: key {k} already in {path}')\n", + " if k in {key(r) for r in _parse(text, path) if r.get('kind') == row['kind']}:\n", + " raise ValueError(f'row refused: key {k} already in {path}')\n", " # ponytail: no file lock, one writer at a time\n", " with p.open('a') as f: f.write(('\\n' if text and not text.endswith('\\n') else '') + _json(row) + '\\n')" ] @@ -420,19 +570,23 @@ "source": [ "#| export\n", "def to_parquet(\n", - " rows: list[dict], # well-formed measurement rows, e.g. from `read_rows`\n", + " rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows`\n", " path: str | Path, # the Parquet file to write\n", "):\n", - " \"Write `rows` as one Parquet table, one column per field in `FIELDS` order, dict fields as canonical JSON\"\n", + " \"Write `rows` as one Parquet table, one column per field of their kind, dict fields as canonical JSON\"\n", " try:\n", " import pandas as pd\n", " import pyarrow # noqa: F401\n", " except ImportError: raise ImportError(\"to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'\") from None\n", + " kinds = sorted({str(r.get('kind')) for r in rows})\n", + " if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or \"no rows\"}')\n", " for i, r in enumerate(rows):\n", " problems = validate(r)\n", " if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems))\n", - " table = [{k: _canonical(r[k]) if FIELDS[k]['type'] is dict else r.get(k) for k in FIELDS} for r in rows]\n", - " pd.DataFrame(table, columns=list(FIELDS)).to_parquet(path, engine='pyarrow', index=False)" + " fields = FIELDS[kinds[0]]\n", + " table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields}\n", + " for r in rows]\n", + " pd.DataFrame(table, columns=list(fields)).to_parquet(path, engine='pyarrow', index=False)" ] }, { @@ -450,7 +604,8 @@ "id": "parquet-note", "metadata": {}, "source": [ - "`to_parquet` needs the optional `parquet` extra: `pip install 'fastermodels[parquet]'`." + "`to_parquet` writes one kind per file and needs the optional `parquet` extra:\n", + "`pip install 'fastermodels[parquet]'`." ] }, { @@ -462,27 +617,39 @@ "\n", "## Usage\n", "\n", + "A latency row, here for an engine built from a model with random weights:\n", + "\n", "```python\n", "from fastermodels import measurement\n", "\n", - "row = dict(schema_version=1, run_id='2026-09-28-a', measured_at='2026-09-28T10:00:00Z',\n", - " harness_commit='deadbee', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", - " task='classification', architecture='resnet50', base_weights='timm/resnet50.a1_in1k',\n", - " weights_license='apache-2.0', variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512},\n", - " input_resolution=224, macs=2_900_000_000, params=17_000_000, engine_hash='0' * 64, size_mb=18.2,\n", - " device='jetson-nano-4gb', device_host='own', runtime='tensorrt', runtime_version='8.2.1',\n", - " platform_version='jetpack-4.6.1', precision='int8', int8_mode='explicit', batch_size=1,\n", - " throttled=False, repeats=5, latency_spread_pct=1.2, latency_ms_p50=2.1,\n", - " latency_scope='engine execute, host transfers excluded', latency_source='measured',\n", - " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:...', dataset_license='custom',\n", - " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", - " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0)\n", - "\n", - "measurement.validate(row) # [] : well formed\n", - "measurement.status(row) # ('VALID', [])\n", - "measurement.throughput(row) # 476.19... images per second, derived\n", - "measurement.write_row('rows.jsonl', row) # appends one line; a second call with the same row is refused\n", - "measurement.to_parquet(measurement.read_rows('rows.jsonl'), 'rows.parquet')\n", + "lat = dict(schema_version=1, kind='latency', run_id='2026-09-28-a', measured_at='2026-09-28T10:00:00Z',\n", + " harness_commit='deadbee', architecture='resnet50', graph_hash=measurement.graph_hash('model.onnx'),\n", + " input_resolution=224, macs=2_900_000_000, params=17_000_000, variant='p30', recipe={'prune': 0.30},\n", + " weights='random', precision='fp16', engine_hash='0' * 64, size_mb=34.1,\n", + " layer_precisions={'fp16': 61}, device='jetson-nano-4gb', device_host='own', runtime='tensorrt',\n", + " runtime_version='8.2.1', platform_version='jetpack-4.6.1', batch_size=1, throttled=False,\n", + " repeats=5, latency_spread_pct=1.2, latency_ms_p50=2.1,\n", + " latency_scope='engine execute, host transfers excluded', latency_source='measured')\n", + "\n", + "measurement.validate(lat) # [] : well formed\n", + "measurement.status(lat) # ('VALID', [])\n", + "measurement.throughput(lat) # 476.19... images per second, derived\n", + "measurement.write_row('rows.jsonl', lat) # appends one line; a second call with the same row is refused\n", + "```\n", + "\n", + "A quality row, and the join of the two kinds on the engine they share:\n", + "\n", + "```python\n", + "qual = dict(schema_version=1, kind='quality', run_id='2026-09-28-b', measured_at='2026-09-28T11:00:00Z',\n", + " harness_commit='deadbee', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", + " weights_license='apache-2.0', engine_hash='1' * 64, precision='fp16', device='jetson-nano-4gb',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:...', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0)\n", + "\n", + "rows = measurement.read_rows('rows.jsonl')\n", + "measurement.join(rows, rows) # each kind is picked from the list it belongs to\n", + "measurement.to_parquet([r for r in rows if r['kind'] == 'latency'], 'latency.parquet')\n", "```" ] }, @@ -495,7 +662,7 @@ "\n", "## See Also\n", "\n", - "- [Model](00_model.html) - the published model a row measures\n", + "- [Model](00_model.html) - the published model a quality row measures\n", "- [Eval](01_eval.html) - `wilson`, the interval `ci_method='wilson'` refers to, and `agreement`\n", "- [Card](02_card.html) - where a published latency is written down\n", "- [Gate](03_gate.html) - the conditions an artifact meets before it is published\n", diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb index 94682ce..4d50987 100644 --- a/nbs/tests/test_measurement.ipynb +++ b/nbs/tests/test_measurement.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "from fastcore.test import *\n", - "import json, tempfile\n", + "import json, re, tempfile\n", "from pathlib import Path\n", "\n", "import numpy as np\n", @@ -32,30 +32,40 @@ "metadata": {}, "outputs": [], "source": [ - "_ROW = dict(schema_version=1, run_id='r1', measured_at='2026-09-28T10:00:00Z', series='2026-09-28',\n", - " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None, hf_repo=None,\n", - " model_id='org/resnet50-p30', model_revision='a1b2c3d', task='classification',\n", - " architecture='resnet50', base_weights='timm/resnet50.a1_in1k', weights_license='apache-2.0',\n", - " variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512}, input_resolution=224,\n", - " macs=2_900_000_000, params=17_000_000, model_hash=None, engine_hash='ab' * 32, size_mb=18.2,\n", + "_ENGINE = 'ab' * 32\n", + "_LAT = dict(schema_version=1, kind='latency', run_id='l1', measured_at='2026-09-28T10:00:00Z', series='2026-09-28',\n", + " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None,\n", + " architecture='resnet50', graph_hash='cd' * 32, input_resolution=224, macs=2_900_000_000,\n", + " params=17_000_000, variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512},\n", + " weights='trained', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", + " precision='int8', int8_mode='explicit', engine_hash=_ENGINE, size_mb=18.2, engine_layers=61,\n", + " reformat_layers=2, layer_precisions={'int8': 52, 'fp16': 9},\n", " device='jetson-nano-4gb', device_host='own', runtime='tensorrt', runtime_version='8.2.1',\n", - " platform_version='jetpack-4.6.1', precision='int8', int8_mode='explicit', power_mode='MAXN',\n", - " clocks_locked=True, batch_size=4,\n", + " platform_version='jetpack-4.6.1', power_mode='MAXN', clocks_locked=True, batch_size=4,\n", " temp_start_c=41.0, temp_end_c=47.5, gpu_freq_mhz=921.6, throttled=False, warmup_s=10.0,\n", " duration_s=60.0, repeats=5, latency_spread_pct=1.2,\n", " latency_ms_p50=8.0, latency_ms_p90=8.4, latency_ms_p99=9.1, latency_scope='engine execute',\n", - " host_latency_ms=8.5, latency_source='measured',\n", - " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:0f', dataset_license='custom',\n", - " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", - " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0, baseline_run_id='r0')\n", + " host_latency_ms=8.5, latency_source='measured')\n", "\n", + "# the same graph with random weights: no model behind it, no accuracy, a first-class latency row\n", + "_RAND = {**{k: v for k, v in _LAT.items() if k not in ('model_id', 'model_revision')},\n", + " 'run_id': 'l2', 'weights': 'random', 'engine_hash': 'ef' * 32}\n", "\n", - "def problems(**change):\n", - " \"measurement.validate() on the valid row with some fields changed; a value of `...` removes the field\"\n", - " r = {**_ROW, **change}\n", - " return measurement.validate({k: v for k, v in r.items() if v is not ...})\n", + "_QUAL = dict(schema_version=1, kind='quality', run_id='q1', measured_at='2026-09-28T11:00:00Z', series='2026-09-28',\n", + " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None,\n", + " model_id='org/resnet50-p30', model_revision='a1b2c3d', model_hash=None, weights_license='apache-2.0',\n", + " engine_hash=_ENGINE, precision='int8', device='jetson-nano-4gb',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:0f', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0, baseline_run_id='q0')\n", "\n", "\n", + "def edit(row, **change):\n", + " \"`row` with some fields changed; a value of `...` removes the field\"\n", + " return {k: v for k, v in {**row, **change}.items() if v is not ...}\n", + "\n", + "\n", + "def problems(row, **change): return measurement.validate(edit(row, **change))\n", "def names(ps): return {p.split(':')[0] for p in ps}" ] }, @@ -66,24 +76,57 @@ "metadata": {}, "outputs": [], "source": [ - "test_eq(set(_ROW), set(measurement.FIELDS))\n", - "test_eq(measurement.validate(_ROW), [])\n", + "test_eq(set(_LAT), set(measurement.FIELDS['latency']))\n", + "test_eq(set(_QUAL), set(measurement.FIELDS['quality']))\n", "test_eq(measurement.SCHEMA_VERSION, 1)\n", + "for _r in (_LAT, _RAND, _QUAL): test_eq(measurement.validate(_r), [])\n", + "\n", + "# the shared fields are defined once, and the schema is data\n", + "_shared = ('schema_version', 'kind', 'run_id', 'measured_at', 'series', 'harness_commit', 'fasterai_version', 'report_url')\n", + "for _f in _shared: test_is(measurement.FIELDS['latency'][_f], measurement.FIELDS['quality'][_f])\n", + "\n", + "# a latency row needs no accuracy, a quality row no latency\n", + "test_eq(problems(_LAT, accuracy=78.9), ['accuracy: unknown field for a latency row'])\n", + "test_eq(problems(_QUAL, latency_ms_p50=8.0), ['latency_ms_p50: unknown field for a quality row'])\n", + "\n", "# optional fields may be absent: absent means not recorded\n", - "test_eq(problems(series=..., fasterai_version=..., platform_version=..., temp_start_c=..., latency_ms_p90=...,\n", - " latency_ms_p99=..., baseline_run_id=..., model_hash=...), [])\n", + "test_eq(problems(_LAT, series=..., fasterai_version=..., platform_version=..., temp_start_c=..., latency_ms_p90=...,\n", + " latency_ms_p99=..., engine_layers=..., reformat_layers=..., layer_precisions=..., model_id=...,\n", + " model_revision=...), [])\n", + "test_eq(problems(_QUAL, model_hash=..., device=..., baseline_run_id=..., argmax_parity=...), [])\n", "# JSON writes 8 as 8: an int is accepted where a float is expected\n", - "test_eq(problems(latency_ms_p50=8), [])\n", + "test_eq(problems(_LAT, latency_ms_p50=8), [])\n", + "\n", "# numpy scalars, as a harness produces them, are numbers; a numpy or Python bool is not\n", - "test_eq(problems(batch_size=np.int64(4), eval_n=np.int32(50_000), macs=np.uint64(2_900_000_000)), [])\n", - "test_eq(problems(latency_ms_p50=np.float64(8.0), accuracy=np.float32(78.9)), [])\n", - "test_eq(names(problems(batch_size=np.bool_(True))), {'batch_size'})\n", - "test_eq(names(problems(latency_ms_p50=np.float64('nan'))), {'latency_ms_p50'})\n", + "test_eq(problems(_LAT, batch_size=np.int64(4), macs=np.uint64(2_900_000_000), latency_ms_p50=np.float64(8.0)), [])\n", + "test_eq(problems(_QUAL, eval_n=np.int32(50_000), accuracy=np.float32(78.9)), [])\n", + "test_eq(names(problems(_LAT, batch_size=np.bool_(True))), {'batch_size'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=np.float64('nan'))), {'latency_ms_p50'})\n", + "\n", "# the module is reached by its name, its generic function names are not at the package top level\n", "test_eq(fastermodels.measurement is measurement, True)\n", "test_eq(hasattr(fastermodels, 'status') or hasattr(fastermodels, 'key'), False)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "kind", + "metadata": {}, + "outputs": [], + "source": [ + "# the kind decides the schema: missing or unknown, one clear problem and nothing else\n", + "test_eq(problems(_LAT, kind=...), ['kind: missing'])\n", + "test_eq(problems(_LAT, kind=None), ['kind: missing'])\n", + "test_eq(problems(_LAT, kind='speed'), [\"kind: 'speed' not one of ('latency', 'quality')\"])\n", + "test_eq(problems(_LAT, kind=['latency']), [\"kind: ['latency'] not one of ('latency', 'quality')\"])\n", + "# a row of one kind is not a row of the other\n", + "test_eq(names(problems(_LAT, kind='quality')) >= {'architecture', 'accuracy', 'weights_license'}, True)\n", + "# not a row at all\n", + "test_eq(measurement.validate([_LAT]), ['row: expected a dict, got list'])\n", + "test_eq(measurement.validate(None), ['row: expected a dict, got NoneType'])" + ] + }, { "cell_type": "code", "execution_count": null, @@ -92,64 +135,60 @@ "outputs": [], "source": [ "# missing required, by name\n", - "test_eq(problems(run_id=...), ['run_id: missing'])\n", - "test_eq(problems(latency_ms_p50=None), ['latency_ms_p50: missing'])\n", - "test_eq(problems(weights_license=...), ['weights_license: missing'])\n", - "test_eq(names(problems(model_id=..., device=...)), {'model_id', 'device'})\n", + "test_eq(problems(_LAT, run_id=...), ['run_id: missing'])\n", + "test_eq(problems(_LAT, latency_ms_p50=None), ['latency_ms_p50: missing'])\n", + "test_eq(problems(_LAT, graph_hash=...), ['graph_hash: missing'])\n", + "test_eq(problems(_LAT, weights=...), ['weights: missing'])\n", + "test_eq(problems(_QUAL, weights_license=...), ['weights_license: missing'])\n", + "test_eq(names(problems(_QUAL, model_id=..., eval_set_hash=...)), {'model_id', 'eval_set_hash'})\n", "\n", "# wrong type\n", - "test_eq(problems(batch_size='4'), [\"batch_size: expected int, got '4'\"])\n", - "test_eq(names(problems(recipe=[0.3])), {'recipe'})\n", - "test_eq(names(problems(variant={'prune': 0.3})), {'variant'})\n", - "test_eq(names(problems(runtime_version=8.2)), {'runtime_version'})\n", - "test_eq(names(problems(throttled='no')), {'throttled'})\n", - "test_eq(names(problems(batch_size=4.0)), {'batch_size'})\n", - "test_eq(names(problems(latency_ms_p50=float('nan'))), {'latency_ms_p50'})\n", - "test_eq(names(problems(latency_ms_p50=float('inf'))), {'latency_ms_p50'})\n", + "test_eq(problems(_LAT, batch_size='4'), [\"batch_size: expected int, got '4'\"])\n", + "test_eq(names(problems(_LAT, recipe=[0.3])), {'recipe'})\n", + "test_eq(names(problems(_LAT, variant={'prune': 0.3})), {'variant'})\n", + "test_eq(names(problems(_LAT, runtime_version=8.2)), {'runtime_version'})\n", + "test_eq(names(problems(_LAT, throttled='no')), {'throttled'})\n", + "test_eq(names(problems(_LAT, batch_size=4.0)), {'batch_size'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=float('inf'))), {'latency_ms_p50'})\n", + "test_eq(names(problems(_LAT, layer_precisions=[52, 9])), {'layer_precisions'})\n", "\n", "# a bool is an int in Python, and neither a count nor a measure here\n", - "test_eq(problems(batch_size=True), ['batch_size: expected int, got True'])\n", - "test_eq(names(problems(schema_version=True)), {'schema_version'})\n", - "test_eq(names(problems(latency_ms_p50=True)), {'latency_ms_p50'})\n", - "test_eq(names(problems(macs=False)), {'macs'})\n", - "test_eq(names(problems(input_resolution=True)), {'input_resolution'})\n", + "test_eq(problems(_LAT, batch_size=True), ['batch_size: expected int, got True'])\n", + "test_eq(names(problems(_LAT, schema_version=True)), {'schema_version'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=True)), {'latency_ms_p50'})\n", + "test_eq(names(problems(_LAT, engine_layers=True)), {'engine_layers'})\n", + "test_eq(names(problems(_QUAL, eval_n=False)), {'eval_n'})\n", "\n", "# out of range\n", - "test_eq(names(problems(batch_size=0)), {'batch_size'})\n", - "test_eq(names(problems(repeats=0)), {'repeats'})\n", - "test_eq(names(problems(input_resolution=0)), {'input_resolution'})\n", - "test_eq(names(problems(latency_ms_p50=0.0)), {'latency_ms_p50'})\n", - "test_eq(names(problems(latency_ms_p99=0.0)), {'latency_ms_p99'})\n", - "test_eq(names(problems(size_mb=-1.0)), {'size_mb'})\n", - "test_eq(names(problems(latency_spread_pct=-0.1)), {'latency_spread_pct'})\n", - "test_eq(names(problems(warmup_s=-1.0)), {'warmup_s'})\n", - "test_eq(names(problems(duration_s=-1.0)), {'duration_s'})\n", - "test_eq(problems(warmup_s=0), [])\n", - "test_eq(names(problems(host_latency_ms=0)), {'host_latency_ms'})\n", - "test_eq(names(problems(eval_n=0)), {'eval_n'})\n", - "test_eq(names(problems(argmax_parity=100.5)), {'argmax_parity'})\n", - "test_eq(names(problems(accuracy=150.0, ci_high=160.0)), {'accuracy', 'ci_high'})\n", + "for _f, _v in [('batch_size', 0), ('repeats', 0), ('input_resolution', 0), ('latency_ms_p50', 0.0),\n", + " ('latency_ms_p99', 0.0), ('size_mb', -1.0), ('latency_spread_pct', -0.1), ('warmup_s', -1.0),\n", + " ('duration_s', -1.0), ('host_latency_ms', 0), ('engine_layers', 0), ('reformat_layers', -1)]:\n", + " test_eq(names(problems(_LAT, **{_f: _v})), {_f})\n", + "test_eq(problems(_LAT, warmup_s=0, reformat_layers=0), [])\n", + "test_eq(names(problems(_QUAL, eval_n=0)), {'eval_n'})\n", + "test_eq(names(problems(_QUAL, argmax_parity=100.5)), {'argmax_parity'})\n", + "test_eq(names(problems(_QUAL, accuracy=150.0, ci_high=160.0)), {'accuracy', 'ci_high'})\n", + "\n", + "# a count per precision\n", + "test_eq(problems(_LAT, layer_precisions={}), [])\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': -1})), {'layer_precisions'})\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': 2.5})), {'layer_precisions'})\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': True})), {'layer_precisions'})\n", + "test_eq(problems(_LAT, layer_precisions={'int8': np.int64(52)}), [])\n", "\n", "# closed vocabularies\n", - "test_eq(names(problems(precision='bf16', int8_mode=None)), {'precision'})\n", - "test_eq(names(problems(task='segmentation')), {'task'})\n", - "test_eq(names(problems(ci_method='normal')), {'ci_method'})\n", - "test_eq(names(problems(runtime='TensorRT 8.2.1')), {'runtime'})\n", - "test_eq(names(problems(device_host='borrowed')), {'device_host'})\n", - "test_eq(names(problems(latency_source='guessed')), {'latency_source'})\n", - "test_eq(names(problems(measured_through='notebook')), {'measured_through'})\n", + "for _r, _f, _v in [(_LAT, 'precision', 'bf16'), (_LAT, 'weights', 'pretrained'), (_LAT, 'runtime', 'TensorRT 8.2.1'),\n", + " (_LAT, 'device_host', 'borrowed'), (_LAT, 'latency_source', 'guessed'),\n", + " (_QUAL, 'precision', 'bf16'), (_QUAL, 'ci_method', 'normal'), (_QUAL, 'measured_through', 'notebook')]:\n", + " test_eq(names(problems(_r, **{_f: _v, **({'int8_mode': None} if _f == 'precision' and _r is _LAT else {})})), {_f})\n", "\n", "# schema version mismatch\n", - "test_eq(problems(schema_version=2), ['schema_version: 2 not one of (1,)'])\n", - "\n", - "# unknown fields: a derived throughput, a verdict, and a name from before the merge\n", - "test_eq(problems(throughput_img_s=500.0), ['throughput_img_s: unknown field'])\n", - "test_eq(problems(passed=True), ['passed: unknown field'])\n", - "test_eq(problems(latency_ms=8.0), ['latency_ms: unknown field'])\n", + "test_eq(problems(_LAT, schema_version=2), ['schema_version: 2 not one of (1,)'])\n", "\n", - "# not a row at all\n", - "test_eq(measurement.validate([_ROW]), ['row: expected a dict, got list'])\n", - "test_eq(measurement.validate(None), ['row: expected a dict, got NoneType'])" + "# unknown fields: a derived throughput, a verdict, and a name from before the split\n", + "test_eq(problems(_LAT, throughput_img_s=500.0), ['throughput_img_s: unknown field for a latency row'])\n", + "test_eq(problems(_QUAL, passed=True), ['passed: unknown field for a quality row'])\n", + "test_eq(problems(_LAT, latency_ms=8.0), ['latency_ms: unknown field for a latency row'])" ] }, { @@ -160,40 +199,43 @@ "outputs": [], "source": [ "# int8_mode is required iff the precision is int8\n", - "test_eq(problems(int8_mode=None), ['int8_mode: missing, required when precision is int8'])\n", - "test_eq(problems(int8_mode=...), ['int8_mode: missing, required when precision is int8'])\n", - "test_eq(problems(precision='fp16'), [\"int8_mode: 'explicit' given for precision 'fp16'\"])\n", - "test_eq(problems(precision='mixed'), [\"int8_mode: 'explicit' given for precision 'mixed'\"])\n", - "test_eq(problems(precision='fp16', int8_mode=None), [])\n", - "test_eq(problems(precision='mixed', int8_mode=None), [])\n", - "test_eq(names(problems(int8_mode='dynamic')), {'int8_mode'})\n", - "\n", - "# the interval brackets the accuracy\n", - "test_eq(names(problems(accuracy=80.0)), {'ci_low, accuracy, ci_high'})\n", - "test_eq(names(problems(ci_low=79.0)), {'ci_low, accuracy, ci_high'})\n", - "test_eq(problems(ci_low=78.9, ci_high=78.9), [])\n", - "\n", - "# latency percentiles are ordered, among those recorded\n", - "test_eq(names(problems(latency_ms_p90=7.9)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", - "test_eq(names(problems(latency_ms_p99=8.3)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", - "test_eq(names(problems(latency_ms_p90=..., latency_ms_p99=7.0)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", - "test_eq(problems(latency_ms_p90=8.0, latency_ms_p99=8.0), [])\n", - "test_eq(problems(latency_ms_p90=None), [])\n", - "\n", - "# the engine digest is 64 lowercase hex characters\n", - "test_eq(names(problems(engine_hash='ab' * 31)), {'engine_hash'})\n", - "test_eq(names(problems(engine_hash='AB' * 32)), {'engine_hash'})\n", - "test_eq(names(problems(engine_hash='sha256:' + 'ab' * 32)), {'engine_hash'})\n", + "test_eq(problems(_LAT, int8_mode=None), ['int8_mode: missing, required when precision is int8'])\n", + "test_eq(problems(_LAT, precision='fp16'), [\"int8_mode: 'explicit' given for precision 'fp16'\"])\n", + "test_eq(problems(_LAT, precision='mixed'), [\"int8_mode: 'explicit' given for precision 'mixed'\"])\n", + "test_eq(problems(_LAT, precision='fp16', int8_mode=None), [])\n", + "test_eq(names(problems(_LAT, int8_mode='dynamic')), {'int8_mode'})\n", + "\n", + "# random weights come from no named model\n", + "test_eq(problems(_RAND, model_id='org/resnet50-p30'), [\"model_id: 'org/resnet50-p30' given for random weights\"])\n", + "test_eq(problems(_LAT, model_id=None, model_revision=None), []) # trained weights need not name their model\n", + "\n", + "# a quality row names the engine it was measured through, unless it was measured on the development model\n", + "test_eq(problems(_QUAL, engine_hash=None), ['engine_hash: missing, required when measured through the deployed engine'])\n", + "test_eq(problems(_QUAL, engine_hash=None, measured_through='dev_model'), [])\n", + "\n", + "# ordered fields, among those recorded\n", + "test_eq(names(problems(_QUAL, accuracy=80.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(names(problems(_QUAL, ci_low=79.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(problems(_QUAL, ci_low=78.9, ci_high=78.9), [])\n", + "test_eq(names(problems(_LAT, latency_ms_p90=7.9)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(names(problems(_LAT, latency_ms_p90=..., latency_ms_p99=7.0)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(problems(_LAT, latency_ms_p90=8.0, latency_ms_p99=8.0), [])\n", + "test_eq(problems(_LAT, latency_ms_p90=None), [])\n", + "test_eq(names(problems(_LAT, reformat_layers=62)), {'reformat_layers, engine_layers'})\n", + "\n", + "# digests are 64 lowercase hex characters\n", + "for _r, _f in [(_LAT, 'engine_hash'), (_LAT, 'graph_hash'), (_QUAL, 'engine_hash')]:\n", + " for _bad in ('ab' * 31, 'AB' * 32, 'sha256:' + 'ab' * 32): test_eq(names(problems(_r, **{_f: _bad})), {_f})\n", "\n", "# the date is ISO-8601 in UTC, and a real date\n", - "test_eq(problems(measured_at='2026-09-28T10:00:00.123+00:00'), [])\n", - "test_eq(names(problems(measured_at='2026-09-28')), {'measured_at'})\n", - "test_eq(names(problems(measured_at='2026-09-28T10:00:00+02:00')), {'measured_at'})\n", - "test_eq(names(problems(measured_at='2026-13-28T10:00:00Z')), {'measured_at'})\n", + "test_eq(problems(_LAT, measured_at='2026-09-28T10:00:00.123+00:00'), [])\n", + "test_eq(names(problems(_LAT, measured_at='2026-09-28')), {'measured_at'})\n", + "test_eq(names(problems(_QUAL, measured_at='2026-09-28T10:00:00+02:00')), {'measured_at'})\n", + "test_eq(names(problems(_LAT, measured_at='2026-13-28T10:00:00Z')), {'measured_at'})\n", "\n", "# recipe keys are free, but the recipe is stored as JSON\n", - "test_eq(problems(recipe={}), [])\n", - "test_eq(names(problems(recipe={'x': object()})), {'recipe'})" + "test_eq(problems(_LAT, recipe={}), [])\n", + "test_eq(names(problems(_LAT, recipe={'x': object()})), {'recipe'})" ] }, { @@ -203,40 +245,42 @@ "metadata": {}, "outputs": [], "source": [ - "test_eq(measurement.status(_ROW), ('VALID', []))\n", + "for _r in (_LAT, _RAND, _QUAL): test_eq(measurement.status(_r), ('VALID', []))\n", "\n", "\n", - "def reasons(**change):\n", - " \"measurement.status() reasons for the valid row with some fields changed\"\n", - " v, rs = measurement.status({k: v for k, v in {**_ROW, **change}.items() if v is not ...})\n", + "def reasons(row, **change):\n", + " \"measurement.status() reasons for `row` with some fields changed\"\n", + " v, rs = measurement.status(edit(row, **change))\n", " test_eq(v, 'INVALID')\n", " return rs\n", "\n", "\n", - "test_eq(reasons(measured_through='dev_model'), ['measured_through: accuracy not measured through the deployed engine'])\n", - "test_eq(reasons(latency_source='estimated'), ['latency_source: latency not measured on the device'])\n", - "test_eq(reasons(argmax_parity=98.9), ['argmax_parity: 98.9 below parity_floor 99.0'])\n", - "test_eq(reasons(argmax_parity=None), ['argmax_parity: not recorded'])\n", - "test_eq(reasons(argmax_parity=...), ['argmax_parity: not recorded'])\n", - "test_eq(measurement.status({**_ROW, 'argmax_parity': 99.0}), ('VALID', [])) # at the floor holds\n", - "test_eq(reasons(throttled=True), ['throttled: the device throttled during the session'])\n", - "test_eq(reasons(throttled=None), ['throttled: not recorded'])\n", - "test_eq(reasons(latency_spread_pct=5.1), ['latency_spread_pct: 5.1 above 5.0'])\n", - "test_eq(reasons(latency_spread_pct=None), ['latency_spread_pct: not recorded'])\n", - "test_eq(measurement.status({**_ROW, 'latency_spread_pct': 5.0}), ('VALID', []))\n", - "test_eq(measurement.status({**_ROW, 'latency_spread_pct': 5.1}, max_spread_pct=6.0), ('VALID', []))\n", - "\n", + "# latency: the device, not the weights\n", + "test_eq(reasons(_LAT, latency_source='estimated'), ['latency_source: latency not measured on the device'])\n", + "test_eq(reasons(_LAT, throttled=True), ['throttled: the device throttled during the session'])\n", + "test_eq(reasons(_LAT, throttled=None), ['throttled: not recorded'])\n", + "test_eq(reasons(_RAND, throttled=...), ['throttled: not recorded'])\n", + "test_eq(reasons(_LAT, latency_spread_pct=5.1), ['latency_spread_pct: 5.1 above 5.0'])\n", + "test_eq(reasons(_LAT, latency_spread_pct=None), ['latency_spread_pct: not recorded'])\n", + "test_eq(measurement.status(edit(_LAT, latency_spread_pct=5.0)), ('VALID', []))\n", + "test_eq(measurement.status(edit(_LAT, latency_spread_pct=5.1), max_spread_pct=6.0), ('VALID', []))\n", "# the informational conditions do not decide validity\n", - "test_eq(measurement.status({**_ROW, 'temp_start_c': None, 'temp_end_c': None, 'gpu_freq_mhz': None,\n", - " 'clocks_locked': None, 'warmup_s': None}), ('VALID', []))\n", - "\n", - "# every reason is reported, not only the first\n", - "test_eq(names(reasons(throttled=None, latency_spread_pct=None, argmax_parity=None, latency_source='estimated',\n", - " measured_through='dev_model')),\n", - " {'throttled', 'latency_spread_pct', 'argmax_parity', 'latency_source', 'measured_through'})\n", + "test_eq(measurement.status(edit(_LAT, temp_start_c=None, temp_end_c=None, gpu_freq_mhz=None, clocks_locked=None,\n", + " warmup_s=None, layer_precisions=None)), ('VALID', []))\n", + "test_eq(names(reasons(_LAT, throttled=None, latency_spread_pct=None, latency_source='estimated')),\n", + " {'throttled', 'latency_spread_pct', 'latency_source'})\n", + "\n", + "# quality: through the engine, with parity\n", + "test_eq(reasons(_QUAL, measured_through='dev_model'), ['measured_through: accuracy not measured through the deployed engine'])\n", + "test_eq(reasons(_QUAL, argmax_parity=98.9), ['argmax_parity: 98.9 below parity_floor 99.0'])\n", + "test_eq(reasons(_QUAL, argmax_parity=None), ['argmax_parity: not recorded'])\n", + "test_eq(measurement.status(edit(_QUAL, argmax_parity=99.0)), ('VALID', [])) # at the floor holds\n", + "test_eq(names(reasons(_QUAL, measured_through='dev_model', engine_hash=None, argmax_parity=...)),\n", + " {'measured_through', 'argmax_parity'})\n", "\n", "# a malformed row is invalid for its structural problems\n", - "test_eq(reasons(batch_size=True), ['batch_size: expected int, got True'])" + "test_eq(reasons(_LAT, batch_size=True), ['batch_size: expected int, got True'])\n", + "test_eq(reasons(_LAT, kind=...), ['kind: missing'])" ] }, { @@ -246,22 +290,128 @@ "metadata": {}, "outputs": [], "source": [ - "test_eq(measurement.key(_ROW), measurement.key(dict(_ROW)))\n", - "# the recipe is canonical: key order does not make a new row\n", - "test_eq(measurement.key({**_ROW, 'recipe': {'calib_n': 512, 'quant': 'int8', 'prune': 0.30}}), measurement.key(_ROW))\n", - "# a software update on the device is a new key\n", - "test_ne(measurement.key({**_ROW, 'runtime_version': '8.5.2'}), measurement.key(_ROW))\n", - "test_ne(measurement.key({**_ROW, 'platform_version': 'jetpack-5.1.3'}), measurement.key(_ROW))\n", - "for _f, _v in [('batch_size', 1), ('recipe', {'prune': 0.40}), ('variant', 'p40-int8'), ('input_resolution', 160),\n", - " ('device', 'raspberry-pi-5'), ('runtime', 'onnxruntime'), ('measured_at', '2026-09-29T10:00:00Z'),\n", - " ('model_revision', 'e4f5a6b')]:\n", - " test_ne(measurement.key({**_ROW, _f: _v}), measurement.key(_ROW))\n", - "# fields outside the identity do not\n", - "test_eq(measurement.key({**_ROW, 'latency_ms_p50': 9.0, 'run_id': 'r2', 'series': None}), measurement.key(_ROW))\n", - "\n", - "# throughput is derived: batch_size * 1000 / latency_ms_p50\n", - "test_eq(measurement.throughput(_ROW), 500.0)\n", - "test_close(measurement.throughput({**_ROW, 'batch_size': 1, 'latency_ms_p50': 2.1}), 476.19, eps=0.01)" + "test_eq(measurement.key(_LAT)[0], 'latency')\n", + "test_eq(measurement.key(_QUAL)[0], 'quality')\n", + "test_eq(measurement.key(_LAT), measurement.key(dict(_LAT)))\n", + "# a latency row is unique on its graph and how it ran; a software update on the device is a new key\n", + "for _f, _v in [('graph_hash', '12' * 32), ('weights', 'random'), ('precision', 'fp16'), ('int8_mode', 'implicit'),\n", + " ('batch_size', 1), ('device', 'raspberry-pi-5'), ('runtime', 'onnxruntime'),\n", + " ('runtime_version', '8.5.2'), ('platform_version', 'jetpack-5.1.3'),\n", + " ('measured_at', '2026-09-29T10:00:00Z')]:\n", + " test_ne(measurement.key({**_LAT, _f: _v}), measurement.key(_LAT))\n", + "# the weights' values are not: another engine of the same graph and weights kind, the same key\n", + "test_eq(measurement.key({**_LAT, 'engine_hash': '12' * 32, 'model_id': None, 'run_id': 'l9', 'latency_ms_p50': 9.0,\n", + " 'series': None, 'variant': 'renamed'}), measurement.key(_LAT))\n", + "# a quality row is unique on its model, engine, evaluation set, metric and date\n", + "for _f, _v in [('model_id', 'org/other'), ('model_revision', 'e4f5a6b'), ('engine_hash', '12' * 32),\n", + " ('eval_set_hash', 'sha256:10'), ('metric', 'top5'), ('measured_at', '2026-09-29T10:00:00Z')]:\n", + " test_ne(measurement.key({**_QUAL, _f: _v}), measurement.key(_QUAL))\n", + "test_eq(measurement.key({**_QUAL, 'accuracy': 70.0, 'run_id': 'q9', 'device': None}), measurement.key(_QUAL))\n", + "\n", + "# throughput is derived from a latency row: batch_size * 1000 / latency_ms_p50\n", + "test_eq(measurement.throughput(_LAT), 500.0)\n", + "test_close(measurement.throughput({**_LAT, 'batch_size': 1, 'latency_ms_p50': 2.1}), 476.19, eps=0.01)\n", + "with ExceptionExpected(ValueError, 'latency row'): measurement.throughput(_QUAL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "join", + "metadata": {}, + "outputs": [], + "source": [ + "_j = measurement.join([_LAT, _RAND], [_QUAL])\n", + "test_eq(len(_j), 1) # the random-weights engine has no accuracy to join, and needs none\n", + "_row = _j[0]\n", + "# identity kept per side, schema version once, the kind gone\n", + "test_eq((_row['latency_run_id'], _row['quality_run_id']), ('l1', 'q1'))\n", + "test_eq((_row['latency_measured_at'], _row['quality_measured_at']), (_LAT['measured_at'], _QUAL['measured_at']))\n", + "test_eq(_row['schema_version'], 1)\n", + "test_eq('kind' in _row or 'run_id' in _row, False)\n", + "# every field of both sides, nothing overwritten\n", + "_both = [f for f in measurement.FIELDS['latency'] if f in measurement.FIELDS['quality']]\n", + "_own = [f for f in _both if f not in ('schema_version', 'kind', 'engine_hash', 'precision', 'device', 'model_id',\n", + " 'model_revision')]\n", + "test_eq(len(_row), len(measurement.FIELDS['latency']) + len(measurement.FIELDS['quality']) - len(_both) + len(_own) - 1)\n", + "test_eq((_row['latency_ms_p50'], _row['accuracy'], _row['engine_hash']), (8.0, 78.9, _ENGINE))\n", + "test_eq(measurement.throughput({**_row, 'kind': 'latency'}), 500.0)\n", + "\n", + "# only VALID rows of the right kind are paired, so the file read back can be given as both arguments\n", + "test_eq(len(measurement.join([_LAT, _RAND, _QUAL], [_LAT, _RAND, _QUAL])), 1)\n", + "test_eq(measurement.join([{**_LAT, 'throttled': True}], [_QUAL]), [])\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'argmax_parity': 90.0}]), [])\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'engine_hash': '12' * 32}]), [])\n", + "# one latency row, two quality rows of the same engine: two pairs\n", + "test_eq(len(measurement.join([_LAT], [_QUAL, {**_QUAL, 'run_id': 'q2', 'eval_set_hash': 'sha256:10'}])), 2)\n", + "\n", + "# a field only one side recorded is taken from it; two sides that disagree on one engine raise\n", + "test_eq(measurement.join([{**_LAT, 'model_id': None, 'model_revision': None}], [_QUAL])[0]['model_id'], 'org/resnet50-p30')\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'device': None}])[0]['device'], 'jetson-nano-4gb')\n", + "with ExceptionExpected(ValueError, 'precision'): measurement.join([_LAT], [{**_QUAL, 'precision': 'fp16'}])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "graph-hash", + "metadata": {}, + "outputs": [], + "source": [ + "import sys, torch\n", + "\n", + "# without onnx installed, a one-line error names the extra\n", + "_saved = sys.modules.get('onnx')\n", + "sys.modules['onnx'] = None\n", + "try:\n", + " with ExceptionExpected(ImportError, r\"fastermodels\\[onnx\\]\"): measurement.graph_hash('x.onnx')\n", + "finally:\n", + " if _saved is None: sys.modules.pop('onnx', None)\n", + " else: sys.modules['onnx'] = _saved\n", + "\n", + "import onnx\n", + "\n", + "\n", + "def export(width, seed, path):\n", + " \"A small conv net with `width` channels and weights drawn from `seed`, exported to ONNX\"\n", + " torch.manual_seed(seed)\n", + " net = torch.nn.Sequential(torch.nn.Conv2d(3, width, 3), torch.nn.ReLU(), torch.nn.Flatten(),\n", + " torch.nn.Linear(width * 6 * 6, 10)).eval()\n", + " torch.onnx.export(net, torch.randn(1, 3, 8, 8), str(path), dynamo=False, opset_version=17,\n", + " input_names=['input'], output_names=['output'])\n", + " return path\n", + "\n", + "\n", + "with tempfile.TemporaryDirectory() as _d:\n", + " _a, _b, _c = (export(w, s, Path(_d, f'{n}.onnx')) for n, w, s in (('a', 8, 0), ('b', 8, 1), ('c', 16, 0)))\n", + " test_ne(Path(_a).read_bytes(), Path(_b).read_bytes()) # the files differ: the weights differ\n", + " _h = measurement.graph_hash(_a)\n", + " test_eq(bool(re.fullmatch(r'[0-9a-f]{64}', _h)), True)\n", + " test_eq(measurement.graph_hash(str(_a)), _h) # deterministic, str or Path\n", + " test_eq(measurement.graph_hash(_b), _h) # same shape, other weights: same graph\n", + " test_ne(measurement.graph_hash(_c), _h) # another width: another graph\n", + " # the file is read, not modified\n", + " _before = Path(_a).read_bytes(); measurement.graph_hash(_a); test_eq(Path(_a).read_bytes(), _before)\n", + "\n", + " # initializer names are part of the graph\n", + " _m = onnx.load(str(_a))\n", + " _old = _m.graph.initializer[0].name\n", + " _m.graph.initializer[0].name = 'renamed'\n", + " for _n in _m.graph.node: _n.input[:] = ['renamed' if i == _old else i for i in _n.input]\n", + " onnx.save(_m, str(Path(_d, 'r.onnx')))\n", + " test_ne(measurement.graph_hash(Path(_d, 'r.onnx')), _h)\n", + "\n", + " # so is the element type: the same graph in fp16 is another graph\n", + " _m = onnx.load(str(_a))\n", + " _m.graph.initializer[0].data_type = onnx.TensorProto.FLOAT16\n", + " onnx.save(_m, str(Path(_d, 'h.onnx')))\n", + " test_ne(measurement.graph_hash(Path(_d, 'h.onnx')), _h)\n", + "\n", + " # weights stored outside the file are not read, and the hash does not depend on where they are stored\n", + " _m = onnx.load(str(_b))\n", + " onnx.save(_m, str(Path(_d, 'ext.onnx')), save_as_external_data=True, all_tensors_to_one_file=True,\n", + " location='ext.data', size_threshold=0)\n", + " test_eq(measurement.graph_hash(Path(_d, 'ext.onnx')), _h)" ] }, { @@ -274,41 +424,41 @@ "with tempfile.TemporaryDirectory() as _d:\n", " _p = Path(_d, 'rows.jsonl')\n", "\n", - " # a valid row is appended, and reads back equal\n", - " measurement.write_row(_p, _ROW)\n", - " test_eq(measurement.read_rows(_p), [_ROW])\n", - " _r2 = {**_ROW, 'runtime_version': '8.5.2', 'run_id': 'r2'}\n", - " measurement.write_row(str(_p), _r2)\n", - " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", - " test_eq(len(_p.read_text().splitlines()), 2)\n", + " # rows of both kinds are appended to one file, and read back equal\n", + " for _r in (_LAT, _RAND, _QUAL): measurement.write_row(_p, _r)\n", + " test_eq(measurement.read_rows(_p), [_LAT, _RAND, _QUAL])\n", + " test_eq(len(_p.read_text().splitlines()), 3)\n", "\n", " # an invalid row is refused, naming its problems, and nothing is written\n", - " with ExceptionExpected(ValueError, 'batch_size'): measurement.write_row(_p, {**_ROW, 'batch_size': 0, 'run_id': 'r3'})\n", - " with ExceptionExpected(ValueError, 'throughput_img_s'): measurement.write_row(_p, {**_ROW, 'throughput_img_s': 1.0})\n", + " with ExceptionExpected(ValueError, 'batch_size'): measurement.write_row(_p, {**_LAT, 'batch_size': 0, 'run_id': 'l3'})\n", + " with ExceptionExpected(ValueError, 'kind: missing'): measurement.write_row(_p, edit(_LAT, kind=...))\n", " # a duplicate key is refused, even with other fields changed\n", - " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_ROW, 'run_id': 'again', 'latency_ms_p50': 7.0})\n", - " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", + " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_LAT, 'run_id': 'again', 'latency_ms_p50': 7.0})\n", + " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_QUAL, 'run_id': 'again', 'accuracy': 79.0})\n", + " test_eq(measurement.read_rows(_p), [_LAT, _RAND, _QUAL])\n", + "\n", + " # uniqueness is per kind: a quality row never collides with a latency row\n", + " _r4 = {**_LAT, 'run_id': 'l4', 'batch_size': 1}\n", + " measurement.write_row(_p, _r4)\n", "\n", " # numpy scalars are written as plain JSON numbers\n", - " _np = {**_ROW, 'run_id': 'r4', 'batch_size': np.int64(2), 'latency_ms_p50': np.float64(4.0),\n", - " 'recipe': {'prune': np.float64(0.3), 'calib_n': np.int64(512)}}\n", + " _np = {**_LAT, 'run_id': 'l5', 'batch_size': np.int64(2), 'latency_ms_p50': np.float64(4.0),\n", + " 'recipe': {'prune': np.float64(0.3), 'calib_n': np.int64(512)}, 'layer_precisions': {'int8': np.int64(61)}}\n", " measurement.write_row(_p, _np)\n", " test_eq(json.loads(_p.read_text().splitlines()[-1])['batch_size'], 2)\n", " test_eq(measurement.read_rows(_p)[-1]['recipe'], {'prune': 0.3, 'calib_n': 512})\n", - " _p.write_text(''.join(l + '\\n' for l in _p.read_text().splitlines()[:-1]))\n", - " test_eq(measurement.read_rows(_p), [_ROW, _r2])\n", "\n", " # a row written by hand without a final newline does not swallow the next one\n", " _p.write_text(_p.read_text().rstrip('\\n'))\n", - " _r3 = {**_ROW, 'batch_size': 1}\n", - " measurement.write_row(_p, _r3)\n", - " test_eq(measurement.read_rows(_p), [_ROW, _r2, _r3])\n", + " _r6 = {**_QUAL, 'run_id': 'q6', 'metric': 'top5'}\n", + " measurement.write_row(_p, _r6)\n", + " test_eq([r['run_id'] for r in measurement.read_rows(_p)], ['l1', 'l2', 'q1', 'l4', 'l5', 'q6'])\n", "\n", " # a malformed line is reported with its line number; blank lines are skipped\n", " _bad = Path(_d, 'bad.jsonl')\n", - " _bad.write_text(json.dumps(_ROW) + '\\n\\n{\"run_id\": \\n')\n", + " _bad.write_text(json.dumps(_LAT) + '\\n\\n{\"run_id\": \\n')\n", " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.read_rows(_bad)\n", - " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.write_row(_bad, _r2)\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.write_row(_bad, _QUAL)\n", " _bad.write_text('[1, 2]\\n')\n", " with ExceptionExpected(ValueError, 'bad.jsonl:1: not a JSON object'): measurement.read_rows(_bad)\n", "\n", @@ -323,13 +473,13 @@ "metadata": {}, "outputs": [], "source": [ - "import importlib.util, sys\n", + "import importlib.util\n", "\n", "# without the extra installed, a one-line error names it (pandas blocked before any import of it)\n", "_saved = sys.modules.get('pandas')\n", "sys.modules['pandas'] = None\n", "try:\n", - " with ExceptionExpected(ImportError, r\"fastermodels\\[parquet\\]\"): measurement.to_parquet([_ROW], 'x.parquet')\n", + " with ExceptionExpected(ImportError, r\"fastermodels\\[parquet\\]\"): measurement.to_parquet([_LAT], 'x.parquet')\n", "finally:\n", " if _saved is None: sys.modules.pop('pandas', None)\n", " else: sys.modules['pandas'] = _saved\n", @@ -337,25 +487,29 @@ "if importlib.util.find_spec('pyarrow') and importlib.util.find_spec('pandas'):\n", " import pandas as pd\n", " with tempfile.TemporaryDirectory() as _d:\n", - " _pq = Path(_d, 'rows.parquet')\n", - " _rows = [_ROW, {**_ROW, 'run_id': 'r2', 'precision': 'fp16', 'int8_mode': None,\n", - " 'recipe': {'quant': 'none', 'prune': 0.30}}]\n", - " _sparse = {k: v for k, v in _ROW.items() if k not in ('series', 'latency_ms_p99')}\n", - " measurement.to_parquet(_rows + [{**_sparse, 'run_id': 'r3'}], _pq)\n", + " _pq = Path(_d, 'latency.parquet')\n", + " _sparse = {k: v for k, v in _LAT.items() if k not in ('series', 'latency_ms_p99', 'layer_precisions')}\n", + " measurement.to_parquet([_LAT, _RAND, {**_sparse, 'run_id': 'l3', 'batch_size': 1}], _pq)\n", " _df = pd.read_parquet(_pq)\n", - " # one column per field, in schema order, one line per row\n", - " test_eq(list(_df.columns), list(measurement.FIELDS))\n", + " # one column per field of the kind, in schema order, one line per row\n", + " test_eq(list(_df.columns), list(measurement.FIELDS['latency']))\n", " test_eq(len(_df), 3)\n", - " # the recipe is canonical JSON: key order is not a difference\n", + " # dict fields are canonical JSON: key order is not a difference\n", " test_eq(_df['recipe'][0], '{\"calib_n\":512,\"prune\":0.3,\"quant\":\"int8\"}')\n", - " test_eq(json.loads(_df['recipe'][1]), {'prune': 0.30, 'quant': 'none'})\n", - " test_eq(_df['latency_ms_p50'].tolist(), [8.0, 8.0, 8.0])\n", - " test_eq(_df['int8_mode'].tolist(), ['explicit', None, 'explicit'])\n", + " test_eq(json.loads(_df['layer_precisions'][0]), {'int8': 52, 'fp16': 9})\n", + " test_eq(_df['weights'].tolist(), ['trained', 'random', 'trained'])\n", + " test_eq(_df['model_id'].tolist(), ['org/resnet50-p30', None, 'org/resnet50-p30'])\n", " # an absent field is a null, not a hole\n", " test_eq(_df['series'].isna().tolist(), [False, False, True])\n", - " test_eq(_df['latency_ms_p99'].isna().tolist(), [False, False, True])\n", - " # a malformed row is refused with its index, and nothing is written\n", - " with ExceptionExpected(ValueError, 'row 1 refused'): measurement.to_parquet([_ROW, {**_ROW, 'batch_size': 0}], Path(_d, 'no.parquet'))\n", + " test_eq(_df['layer_precisions'].isna().tolist(), [False, False, True])\n", + "\n", + " measurement.to_parquet([_QUAL], Path(_d, 'quality.parquet'))\n", + " test_eq(list(pd.read_parquet(Path(_d, 'quality.parquet')).columns), list(measurement.FIELDS['quality']))\n", + "\n", + " # one kind per file; no rows, no file; a malformed row is refused with its index; nothing is written\n", + " with ExceptionExpected(ValueError, 'one kind per file'): measurement.to_parquet([_LAT, _QUAL], Path(_d, 'no.parquet'))\n", + " with ExceptionExpected(ValueError, 'no rows'): measurement.to_parquet([], Path(_d, 'no.parquet'))\n", + " with ExceptionExpected(ValueError, 'row 1 refused'): measurement.to_parquet([_LAT, {**_LAT, 'batch_size': 0}], Path(_d, 'no.parquet'))\n", " test_eq(Path(_d, 'no.parquet').exists(), False)\n", "else: print('pandas or pyarrow not installed: to_parquet round trip skipped')" ] diff --git a/pyproject.toml b/pyproject.toml index 5de415c..a016e20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = ['torch', 'huggingface_hub>=0.25', 'safetensors', 'numpy'] test = ['torchvision', 'torch-pruning', 'onnx', 'fastcore'] dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore', 'pandas', 'pyarrow'] parquet = ['pandas', 'pyarrow'] +onnx = ['onnx'] [project.urls] Repository = "https://github.com/FasterAI-Labs/fastermodels" From 9457f2716a6e47fca658818342a9c7b8e37abf1c Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Tue, 29 Sep 2026 11:55:34 +0200 Subject: [PATCH 3/3] Write measurement Parquet files with an explicit Arrow schema to_parquet went through a pandas DataFrame, so the column types of the published file were whatever pandas inferred: a string column with a missing value read back as NaN under pandas 3 and as None under pandas 2, and a column nobody recorded had no type at all. A published file must not depend on the reader. The schema is now derived from FIELDS[kind]: str is a string column, int int64, float float64, bool bool, a dict a string holding canonical JSON, and a field that is not required is nullable. The table is built and written with pyarrow alone; pandas is no longer used, and the `parquet` extra is pyarrow only. The round-trip test reads the file back with pyarrow and compares it to the rows, None preserved, and pins the type and nullability of the columns. --- fastermodels/measurement.py | 17 +++++--- nbs/04_measurement.ipynb | 23 ++++++---- nbs/tests/test_measurement.ipynb | 72 +++++++++++++++++++++----------- pyproject.toml | 4 +- 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py index 251c675..cbc3d30 100644 --- a/fastermodels/measurement.py +++ b/fastermodels/measurement.py @@ -321,21 +321,26 @@ def write_row( with p.open('a') as f: f.write(('\n' if text and not text.endswith('\n') else '') + _json(row) + '\n') # %% ../nbs/04_measurement.ipynb #parquet +_ARROW = {str: 'string', int: 'int64', float: 'float64', bool: 'bool_', dict: 'string'} # a dict is stored as canonical JSON + + def to_parquet( rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows` path: str | Path, # the Parquet file to write ): - "Write `rows` as one Parquet table, one column per field of their kind, dict fields as canonical JSON" + "Write `rows` as one Parquet table with an explicit schema: one typed column per field of their kind" try: - import pandas as pd - import pyarrow # noqa: F401 - except ImportError: raise ImportError("to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'") from None + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError: raise ImportError("to_parquet needs pyarrow: pip install 'fastermodels[parquet]'") from None kinds = sorted({str(r.get('kind')) for r in rows}) if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or "no rows"}') for i, r in enumerate(rows): problems = validate(r) if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems)) fields = FIELDS[kinds[0]] + schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()]) + plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields} - for r in rows] - pd.DataFrame(table, columns=list(fields)).to_parquet(path, engine='pyarrow', index=False) + for r in plain] + pq.write_table(pa.Table.from_pylist(table, schema=schema), str(path)) diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb index 8af309d..5342d32 100644 --- a/nbs/04_measurement.ipynb +++ b/nbs/04_measurement.ipynb @@ -569,24 +569,29 @@ "outputs": [], "source": [ "#| export\n", + "_ARROW = {str: 'string', int: 'int64', float: 'float64', bool: 'bool_', dict: 'string'} # a dict is stored as canonical JSON\n", + "\n", + "\n", "def to_parquet(\n", " rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows`\n", " path: str | Path, # the Parquet file to write\n", "):\n", - " \"Write `rows` as one Parquet table, one column per field of their kind, dict fields as canonical JSON\"\n", + " \"Write `rows` as one Parquet table with an explicit schema: one typed column per field of their kind\"\n", " try:\n", - " import pandas as pd\n", - " import pyarrow # noqa: F401\n", - " except ImportError: raise ImportError(\"to_parquet needs pandas and pyarrow: pip install 'fastermodels[parquet]'\") from None\n", + " import pyarrow as pa\n", + " import pyarrow.parquet as pq\n", + " except ImportError: raise ImportError(\"to_parquet needs pyarrow: pip install 'fastermodels[parquet]'\") from None\n", " kinds = sorted({str(r.get('kind')) for r in rows})\n", " if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or \"no rows\"}')\n", " for i, r in enumerate(rows):\n", " problems = validate(r)\n", " if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems))\n", " fields = FIELDS[kinds[0]]\n", + " schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()])\n", + " plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers\n", " table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields}\n", - " for r in rows]\n", - " pd.DataFrame(table, columns=list(fields)).to_parquet(path, engine='pyarrow', index=False)" + " for r in plain]\n", + " pq.write_table(pa.Table.from_pylist(table, schema=schema), str(path))" ] }, { @@ -604,8 +609,10 @@ "id": "parquet-note", "metadata": {}, "source": [ - "`to_parquet` writes one kind per file and needs the optional `parquet` extra:\n", - "`pip install 'fastermodels[parquet]'`." + "`to_parquet` writes one kind per file, with an explicit Arrow schema derived from `FIELDS`: `str` is a\n", + "string column, `int` int64, `float` float64, `bool` bool, a `dict` a string holding canonical JSON, and a\n", + "field that is not required is nullable. A missing value is a null in the file, whatever reads it. It needs\n", + "the optional `parquet` extra: `pip install 'fastermodels[parquet]'`." ] }, { diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb index 4d50987..13851d0 100644 --- a/nbs/tests/test_measurement.ipynb +++ b/nbs/tests/test_measurement.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "from fastcore.test import *\n", - "import json, re, tempfile\n", + "import json, re, sys, tempfile\n", "from pathlib import Path\n", "\n", "import numpy as np\n", @@ -475,43 +475,67 @@ "source": [ "import importlib.util\n", "\n", - "# without the extra installed, a one-line error names it (pandas blocked before any import of it)\n", - "_saved = sys.modules.get('pandas')\n", - "sys.modules['pandas'] = None\n", + "# without the extra installed, a one-line error names it\n", + "_saved = sys.modules.get('pyarrow')\n", + "sys.modules['pyarrow'] = None\n", "try:\n", " with ExceptionExpected(ImportError, r\"fastermodels\\[parquet\\]\"): measurement.to_parquet([_LAT], 'x.parquet')\n", "finally:\n", - " if _saved is None: sys.modules.pop('pandas', None)\n", - " else: sys.modules['pandas'] = _saved\n", + " if _saved is None: sys.modules.pop('pyarrow', None)\n", + " else: sys.modules['pyarrow'] = _saved\n", "\n", - "if importlib.util.find_spec('pyarrow') and importlib.util.find_spec('pandas'):\n", - " import pandas as pd\n", + "\n", + "def stored(rows, kind):\n", + " \"What a Parquet file of `rows` must hold: every field of the kind, None where absent\"\n", + " return [{k: r.get(k) for k in measurement.FIELDS[kind]} for r in rows]\n", + "\n", + "\n", + "def read_back(path):\n", + " \"The rows of a Parquet file, dict fields decoded from their JSON\"\n", + " return [{k: json.loads(v) if k in ('recipe', 'layer_precisions') and v is not None else v for k, v in r.items()}\n", + " for r in pq.read_table(path).to_pylist()]\n", + "\n", + "\n", + "if importlib.util.find_spec('pyarrow'):\n", + " import pyarrow as pa\n", + " import pyarrow.parquet as pq\n", " with tempfile.TemporaryDirectory() as _d:\n", " _pq = Path(_d, 'latency.parquet')\n", " _sparse = {k: v for k, v in _LAT.items() if k not in ('series', 'latency_ms_p99', 'layer_precisions')}\n", - " measurement.to_parquet([_LAT, _RAND, {**_sparse, 'run_id': 'l3', 'batch_size': 1}], _pq)\n", - " _df = pd.read_parquet(_pq)\n", - " # one column per field of the kind, in schema order, one line per row\n", - " test_eq(list(_df.columns), list(measurement.FIELDS['latency']))\n", - " test_eq(len(_df), 3)\n", + " _rows = [_LAT, _RAND, {**_sparse, 'run_id': 'l3', 'batch_size': 1, 'latency_ms_p50': 8},\n", + " {**_LAT, 'run_id': 'l4', 'batch_size': np.int64(2), 'latency_ms_p50': np.float32(4.5)}]\n", + " measurement.to_parquet(_rows, _pq)\n", + " # every row reads back as written, a missing value as None, an int where a float goes as a float\n", + " test_eq(read_back(_pq), stored(_rows, 'latency'))\n", + " test_eq([r['model_id'] for r in read_back(_pq)], ['org/resnet50-p30', None, 'org/resnet50-p30', 'org/resnet50-p30'])\n", + " test_eq([type(r['latency_ms_p50']) for r in read_back(_pq)], [float] * 4)\n", " # dict fields are canonical JSON: key order is not a difference\n", - " test_eq(_df['recipe'][0], '{\"calib_n\":512,\"prune\":0.3,\"quant\":\"int8\"}')\n", - " test_eq(json.loads(_df['layer_precisions'][0]), {'int8': 52, 'fp16': 9})\n", - " test_eq(_df['weights'].tolist(), ['trained', 'random', 'trained'])\n", - " test_eq(_df['model_id'].tolist(), ['org/resnet50-p30', None, 'org/resnet50-p30'])\n", - " # an absent field is a null, not a hole\n", - " test_eq(_df['series'].isna().tolist(), [False, False, True])\n", - " test_eq(_df['layer_precisions'].isna().tolist(), [False, False, True])\n", - "\n", - " measurement.to_parquet([_QUAL], Path(_d, 'quality.parquet'))\n", - " test_eq(list(pd.read_parquet(Path(_d, 'quality.parquet')).columns), list(measurement.FIELDS['quality']))\n", + " test_eq(pq.read_table(_pq).column('recipe')[0].as_py(), '{\"calib_n\":512,\"prune\":0.3,\"quant\":\"int8\"}')\n", + "\n", + " # the schema is explicit, derived from FIELDS: the file does not depend on the reader\n", + " _schema = pq.read_schema(_pq)\n", + " test_eq(_schema.names, list(measurement.FIELDS['latency']))\n", + " for _f, _t in [('latency_ms_p50', pa.float64()), ('batch_size', pa.int64()), ('throttled', pa.bool_()),\n", + " ('model_id', pa.string()), ('recipe', pa.string()), ('layer_precisions', pa.string())]:\n", + " test_eq(_schema.field(_f).type, _t)\n", + " test_eq(_schema.field('model_id').nullable, True)\n", + " test_eq(_schema.field('throttled').nullable, True)\n", + " test_eq(_schema.field('batch_size').nullable, False)\n", + " # a column nobody recorded is still typed, not a null column\n", + " measurement.to_parquet([_RAND], Path(_d, 'rand.parquet'))\n", + " test_eq(pq.read_schema(Path(_d, 'rand.parquet')).field('model_id').type, pa.string())\n", + "\n", + " _q = [_QUAL, {**_QUAL, 'run_id': 'q2', 'engine_hash': None, 'measured_through': 'dev_model', 'device': None}]\n", + " measurement.to_parquet(_q, Path(_d, 'quality.parquet'))\n", + " test_eq(read_back(Path(_d, 'quality.parquet')), stored(_q, 'quality'))\n", + " test_eq(pq.read_schema(Path(_d, 'quality.parquet')).names, list(measurement.FIELDS['quality']))\n", "\n", " # one kind per file; no rows, no file; a malformed row is refused with its index; nothing is written\n", " with ExceptionExpected(ValueError, 'one kind per file'): measurement.to_parquet([_LAT, _QUAL], Path(_d, 'no.parquet'))\n", " with ExceptionExpected(ValueError, 'no rows'): measurement.to_parquet([], Path(_d, 'no.parquet'))\n", " with ExceptionExpected(ValueError, 'row 1 refused'): measurement.to_parquet([_LAT, {**_LAT, 'batch_size': 0}], Path(_d, 'no.parquet'))\n", " test_eq(Path(_d, 'no.parquet').exists(), False)\n", - "else: print('pandas or pyarrow not installed: to_parquet round trip skipped')" + "else: print('pyarrow not installed: to_parquet round trip skipped')" ] } ], diff --git a/pyproject.toml b/pyproject.toml index a016e20..38f30d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,8 @@ dependencies = ['torch', 'huggingface_hub>=0.25', 'safetensors', 'numpy'] [project.optional-dependencies] test = ['torchvision', 'torch-pruning', 'onnx', 'fastcore'] -dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore', 'pandas', 'pyarrow'] -parquet = ['pandas', 'pyarrow'] +dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore', 'pyarrow'] +parquet = ['pyarrow'] onnx = ['onnx'] [project.urls]