diff --git a/fastermodels/__init__.py b/fastermodels/__init__.py index dc88e53..03e93e2 100644 --- a/fastermodels/__init__.py +++ b/fastermodels/__init__.py @@ -5,3 +5,4 @@ params, macs, peak_activation_bytes) from fastermodels.card import render_card, check_card, FORBIDDEN from fastermodels.gate import run_gate, gate_passed, GateRow +from fastermodels import measurement as measurement diff --git a/fastermodels/_modidx.py b/fastermodels/_modidx.py index 4062ac7..700cdc5 100644 --- a/fastermodels/_modidx.py +++ b/fastermodels/_modidx.py @@ -36,6 +36,33 @@ 'fastermodels.gate._reload': ('gate.html#_reload', 'fastermodels/gate.py'), 'fastermodels.gate.gate_passed': ('gate.html#gate_passed', 'fastermodels/gate.py'), 'fastermodels.gate.run_gate': ('gate.html#run_gate', 'fastermodels/gate.py')}, + 'fastermodels.measurement': { 'fastermodels.measurement._canonical': ( 'measurement.html#_canonical', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._cross_problems': ( 'measurement.html#_cross_problems', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._field_problems': ( 'measurement.html#_field_problems', + 'fastermodels/measurement.py'), + 'fastermodels.measurement._is_date': ('measurement.html#_is_date', 'fastermodels/measurement.py'), + 'fastermodels.measurement._is_json': ('measurement.html#_is_json', 'fastermodels/measurement.py'), + 'fastermodels.measurement._item': ('measurement.html#_item', 'fastermodels/measurement.py'), + 'fastermodels.measurement._json': ('measurement.html#_json', 'fastermodels/measurement.py'), + 'fastermodels.measurement._pair': ('measurement.html#_pair', 'fastermodels/measurement.py'), + 'fastermodels.measurement._parse': ('measurement.html#_parse', 'fastermodels/measurement.py'), + 'fastermodels.measurement._typed': ('measurement.html#_typed', 'fastermodels/measurement.py'), + 'fastermodels.measurement.graph_hash': ( 'measurement.html#graph_hash', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.join': ('measurement.html#join', 'fastermodels/measurement.py'), + 'fastermodels.measurement.key': ('measurement.html#key', 'fastermodels/measurement.py'), + 'fastermodels.measurement.read_rows': ( 'measurement.html#read_rows', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.status': ('measurement.html#status', 'fastermodels/measurement.py'), + 'fastermodels.measurement.throughput': ( 'measurement.html#throughput', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.to_parquet': ( 'measurement.html#to_parquet', + 'fastermodels/measurement.py'), + 'fastermodels.measurement.validate': ('measurement.html#validate', 'fastermodels/measurement.py'), + 'fastermodels.measurement.write_row': ( 'measurement.html#write_row', + 'fastermodels/measurement.py')}, 'fastermodels.model': { 'fastermodels.model.FasterModel': ('model.html#fastermodel', 'fastermodels/model.py'), 'fastermodels.model.FasterModel.__init__': ('model.html#fastermodel.__init__', 'fastermodels/model.py'), 'fastermodels.model.FasterModel._from_pretrained': ( 'model.html#fastermodel._from_pretrained', diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py new file mode 100644 index 0000000..cbc3d30 --- /dev/null +++ b/fastermodels/measurement.py @@ -0,0 +1,346 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_measurement.ipynb. + +# %% ../nbs/04_measurement.ipynb #imports +from __future__ import annotations + +import hashlib +import json +import math +import numbers +import re +from datetime import datetime +from pathlib import Path + +# %% auto #0 +__all__ = ['SCHEMA_VERSION', 'FIELDS', 'validate', 'status', 'key', 'throughput', 'join', 'graph_hash', 'read_rows', 'write_row', + 'to_parquet'] + +# %% ../nbs/04_measurement.ipynb #schema +SCHEMA_VERSION = 1 +_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|\+00:00)' +_HEX = r'[0-9a-f]{64}' +_PCT = dict(ge=0, le=100) +_PRECISION = dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed')) + +# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None +_SHARED = { + 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)), + 'kind': dict(type=str, required=True, choices=('latency', 'quality')), + 'run_id': dict(type=str, required=True), + 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True), + 'series': dict(type=str, required=False), + 'harness_commit': dict(type=str, required=True), + 'fasterai_version': dict(type=str, required=False), + 'report_url': dict(type=str, required=False), +} + +_LATENCY = { + # graph + 'architecture': dict(type=str, required=True), + 'graph_hash': dict(type=str, required=True, pattern=_HEX), # `graph_hash`: the ONNX graph without its weight values + 'input_resolution': dict(type=int, required=True, gt=0), + 'macs': dict(type=int, required=True, gt=0), + 'params': dict(type=int, required=True, gt=0), + 'variant': dict(type=str, required=True), + 'recipe': dict(type=dict, required=True, json=True), + 'weights': dict(type=str, required=True, choices=('trained', 'random')), + 'model_id': dict(type=str, required=False), + 'model_revision': dict(type=str, required=False), + # artifact + 'precision': _PRECISION, + 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')), + 'engine_hash': dict(type=str, required=True, pattern=_HEX), # sha256 of the engine file that ran + 'size_mb': dict(type=float, required=True, gt=0), + 'engine_layers': dict(type=int, required=False, gt=0), + 'reformat_layers': dict(type=int, required=False, ge=0), + 'layer_precisions': dict(type=dict, required=False, counts=True), + # device and runtime + 'device': dict(type=str, required=True), + 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')), + 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')), + 'runtime_version': dict(type=str, required=True), + 'platform_version': dict(type=str, required=False), + 'power_mode': dict(type=str, required=False), + 'clocks_locked': dict(type=bool, required=False), + 'batch_size': dict(type=int, required=True, ge=1), + # conditions + 'temp_start_c': dict(type=float, required=False), + 'temp_end_c': dict(type=float, required=False), + 'gpu_freq_mhz': dict(type=float, required=False, gt=0), + 'throttled': dict(type=bool, required=False), + 'warmup_s': dict(type=float, required=False, ge=0), + 'duration_s': dict(type=float, required=False, ge=0), + 'repeats': dict(type=int, required=True, ge=1), + 'latency_spread_pct': dict(type=float, required=False, ge=0), + # latency + 'latency_ms_p50': dict(type=float, required=True, gt=0), + 'latency_ms_p90': dict(type=float, required=False, gt=0), + 'latency_ms_p99': dict(type=float, required=False, gt=0), + 'latency_scope': dict(type=str, required=True), + 'host_latency_ms': dict(type=float, required=False, gt=0), + 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')), +} + +_QUALITY = { + 'model_id': dict(type=str, required=True), + 'model_revision': dict(type=str, required=True), + 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned + 'weights_license': dict(type=str, required=True), + 'engine_hash': dict(type=str, required=False, pattern=_HEX), # the engine the accuracy was measured through + 'precision': _PRECISION, + 'device': dict(type=str, required=False), + 'metric': dict(type=str, required=True), + 'eval_set': dict(type=str, required=True), + 'eval_set_hash': dict(type=str, required=True), + 'dataset_license': dict(type=str, required=True), + 'eval_n': dict(type=int, required=True, gt=0), + 'accuracy': dict(type=float, required=True, **_PCT), + 'ci_low': dict(type=float, required=True, **_PCT), + 'ci_high': dict(type=float, required=True, **_PCT), + 'ci_method': dict(type=str, required=True, choices=('wilson', 'bootstrap')), + 'measured_through': dict(type=str, required=True, choices=('deployed_engine', 'dev_model')), + 'argmax_parity': dict(type=float, required=False, **_PCT), + 'parity_floor': dict(type=float, required=True, **_PCT), + 'baseline_run_id': dict(type=str, required=False), +} + +FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}} + + +def _item(o): + "A numpy scalar as the Python number JSON knows" + if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item() + raise TypeError(f'{type(o).__name__} is not JSON serializable') + + +def _json(obj, **kw): return json.dumps(obj, default=_item, allow_nan=False, **kw) +def _canonical(d): return _json(d, sort_keys=True, separators=(',', ':')) + + +def _is_json(v): + try: return _json(v) is not None + except (TypeError, ValueError): return False + + +def _is_date(v): + try: return datetime.fromisoformat(v.replace('Z', '+00:00')) is not None # Python 3.10 does not read 'Z' + except ValueError: return False + + +def _typed(v, t): + "`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite" + if isinstance(v, bool): return t is bool + if t is int: return isinstance(v, numbers.Integral) + if t is float: return isinstance(v, numbers.Real) and math.isfinite(v) + return isinstance(v, t) + + +_RULES = { + 'choices': (lambda v, c: v in c, 'not one of {}'), + 'gt': (lambda v, b: v > b, 'not > {}'), + 'ge': (lambda v, b: v >= b, 'not >= {}'), + 'le': (lambda v, b: v <= b, 'not <= {}'), + 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'), + 'date': (lambda v, _: _is_date(v), 'is not a date'), + 'json': (lambda v, _: _is_json(v), 'is not JSON'), + 'counts': (lambda v, _: all(isinstance(k, str) and _typed(n, int) and n >= 0 for k, n in v.items()), + 'is not a {{name: count}} dict'), +} +_ORDERED = {'latency': (('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'), ('reformat_layers', 'engine_layers')), + 'quality': (('ci_low', 'accuracy', 'ci_high'),)} + + +def _field_problems(name, spec, v): + "The problems of one field value, against its spec" + if v is None: return [f'{name}: missing'] if spec['required'] else [] + if not _typed(v, spec['type']): return [f'{name}: expected {spec["type"].__name__}, got {v!r}'] + return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])] + + +def _cross_problems(row): + "Problems between fields; each field involved is already well typed" + out, kind = [], row['kind'] + if kind == 'latency': + p, mode = row['precision'], row.get('int8_mode') + if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8') + if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}') + if row['weights'] == 'random' and row.get('model_id') is not None: + out.append(f"model_id: {row['model_id']!r} given for random weights") + elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None: + out.append('engine_hash: missing, required when measured through the deployed engine') + for fields in _ORDERED[kind]: + vals = [(f, row[f]) for f in fields if row.get(f) is not None] + if [v for _, v in vals] != sorted(v for _, v in vals): + out.append(f"{', '.join(fields)}: not in order, " + ', '.join(f'{f}={v}' for f, v in vals)) + return out + + +def validate( + row: dict, # one measurement row, of either kind +) -> list[str]: + "Every structural problem of `row`, each prefixed by its field name; an empty list means well formed" + if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}'] + kind = row.get('kind') + if kind is None: return ['kind: missing'] + if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}'] + fields = FIELDS[kind] + out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields] + for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name)) + return out or _cross_problems(row) + +# %% ../nbs/04_measurement.ipynb #status +# per kind: field, when its value makes the measurement invalid, and why; a field that is None is not recorded +_VALIDITY = { + 'latency': ( + ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'), + ('throttled', lambda v, r, m: v, 'the device throttled during the session'), + ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'), + ), + 'quality': ( + ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'), + ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'), + ), +} + + +def status( + row: dict, # one measurement row, of either kind + max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent +) -> tuple[str, list[str]]: + "('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget" + reasons = validate(row) + if reasons: return 'INVALID', reasons + for f, bad, why in _VALIDITY[row['kind']]: + v = row.get(f) + if v is None: reasons.append(f'{f}: not recorded') + elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct)) + return ('INVALID', reasons) if reasons else ('VALID', []) + +# %% ../nbs/04_measurement.ipynb #key +_KEYS = { + 'latency': ('graph_hash', 'weights', 'precision', 'int8_mode', 'batch_size', 'device', 'runtime', + 'runtime_version', 'platform_version', 'measured_at'), + 'quality': ('model_id', 'model_revision', 'engine_hash', 'eval_set_hash', 'metric', 'measured_at'), +} + + +def key( + row: dict, # one well-formed measurement row +) -> tuple: + "The identity a row is unique on, its kind first; a software update on the device gives a new latency key" + return (row['kind'],) + tuple(row.get(f) for f in _KEYS[row['kind']]) + + +def throughput( + row: dict, # one latency row +) -> float: + "Images per second at the timed batch, derived from the median latency and never stored" + if row.get('kind') != 'latency': raise ValueError(f"throughput needs a latency row, got kind {row.get('kind')!r}") + return row['batch_size'] * 1000 / row['latency_ms_p50'] + +# %% ../nbs/04_measurement.ipynb #join +_OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed +_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree + + +def _pair(lat, qual): + "One joined dict from a latency row and a quality row of the same engine" + out = {'schema_version': lat['schema_version']} + for side, r in (('latency', lat), ('quality', qual)): + out |= {f'{side}_{f}': r.get(f) for f in _OWN} + out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH} + for f in _BOTH: + a, b = lat.get(f), qual.get(f) + if None not in (a, b) and a != b: + raise ValueError(f"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine") + out[f] = b if a is None else a + return out + + +def join( + latency_rows: list[dict], # latency rows; rows of the other kind and invalid rows are skipped + quality_rows: list[dict], # quality rows; rows of the other kind and invalid rows are skipped +) -> list[dict]: + "One dict per pair of VALID latency and quality rows sharing an `engine_hash`" + by_engine = {} + for q in quality_rows: + if q.get('kind') == 'quality' and status(q)[0] == 'VALID': by_engine.setdefault(q['engine_hash'], []).append(q) + return [_pair(l, q) for l in latency_rows if l.get('kind') == 'latency' and status(l)[0] == 'VALID' + for q in by_engine.get(l['engine_hash'], [])] + +# %% ../nbs/04_measurement.ipynb #graph +def graph_hash( + onnx_path: str | Path, # an ONNX file +) -> str: + "sha256 of the ONNX graph with every initializer's values removed and its name, shape and type kept" + try: import onnx + except ImportError: raise ImportError("graph_hash needs onnx: pip install 'fastermodels[onnx]'") from None + m = onnx.load(str(onnx_path), load_external_data=False) + # ponytail: top-level initializers only; Constant nodes and subgraph initializers keep their values + for t in m.graph.initializer: + name, dtype, dims = t.name, t.data_type, list(t.dims) + t.Clear() + t.name, t.data_type = name, dtype + t.dims.extend(dims) + h = hashlib.sha256(m.graph.SerializeToString(deterministic=True)) + for o in sorted((o.domain, o.version) for o in m.opset_import): h.update(f'{o[0]}:{o[1]};'.encode()) + return h.hexdigest() + +# %% ../nbs/04_measurement.ipynb #io +def _parse(text, path): + "The rows of a JSON Lines text; a malformed line raises with its number" + rows = [] + for i, line in enumerate(text.splitlines(), 1): + if not line.strip(): continue + try: r = json.loads(line) + except json.JSONDecodeError as e: raise ValueError(f'{path}:{i}: not JSON ({e.msg})') from None + if not isinstance(r, dict): raise ValueError(f'{path}:{i}: not a JSON object') + rows.append(r) + return rows + + +def read_rows( + path: str | Path, # a JSON Lines file of measurement rows, of either kind +) -> list[dict]: + "The rows of `path`, as stored; `status` judges them" + return _parse(Path(path).read_text(), path) + + +def write_row( + path: str | Path, # a JSON Lines file, created if absent + row: dict, # the measurement row to append +): + "Append `row` to `path`; refuses a malformed row and a row whose key is already in the file" + problems = validate(row) + if problems: raise ValueError('row refused: ' + '; '.join(problems)) + p, k = Path(path), key(row) + text = p.read_text() if p.exists() else '' + if k in {key(r) for r in _parse(text, path) if r.get('kind') == row['kind']}: + raise ValueError(f'row refused: key {k} already in {path}') + # ponytail: no file lock, one writer at a time + with p.open('a') as f: f.write(('\n' if text and not text.endswith('\n') else '') + _json(row) + '\n') + +# %% ../nbs/04_measurement.ipynb #parquet +_ARROW = {str: 'string', int: 'int64', float: 'float64', bool: 'bool_', dict: 'string'} # a dict is stored as canonical JSON + + +def to_parquet( + rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows` + path: str | Path, # the Parquet file to write +): + "Write `rows` as one Parquet table with an explicit schema: one typed column per field of their kind" + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError: raise ImportError("to_parquet needs pyarrow: pip install 'fastermodels[parquet]'") from None + kinds = sorted({str(r.get('kind')) for r in rows}) + if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or "no rows"}') + for i, r in enumerate(rows): + problems = validate(r) + if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems)) + fields = FIELDS[kinds[0]] + schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()]) + plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers + table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields} + for r in plain] + pq.write_table(pa.Table.from_pylist(table, schema=schema), str(path)) diff --git a/nbs/00_model.ipynb b/nbs/00_model.ipynb index bcea8bf..f900e05 100644 --- a/nbs/00_model.ipynb +++ b/nbs/00_model.ipynb @@ -374,6 +374,7 @@ "- [Eval](01_eval.html) - the per-image harness the published numbers come from\n", "- [Card](02_card.html) - what the artifact says about itself\n", "- [Gate](03_gate.html) - the conditions an artifact meets before publication\n", + "- [Measurement](04_measurement.html) - one on-device measurement row of a published model\n", "\n", "Tests live in `nbs/tests/test_model.ipynb`." ] diff --git a/nbs/01_eval.ipynb b/nbs/01_eval.ipynb index 3011d53..30bcdb9 100644 --- a/nbs/01_eval.ipynb +++ b/nbs/01_eval.ipynb @@ -436,6 +436,7 @@ "- [Model](00_model.html) - the model these numbers are measured on\n", "- [Card](02_card.html) - where `k`, `n`, the interval and the delta are written down\n", "- [Gate](03_gate.html) - the condition that reads `lo` against the floor\n", + "- [Measurement](04_measurement.html) - where an on-device accuracy records its interval and `ci_method`\n", "\n", "Tests live in `nbs/tests/test_eval.ipynb`." ] diff --git a/nbs/02_card.ipynb b/nbs/02_card.ipynb index 0e17c64..436b517 100644 --- a/nbs/02_card.ipynb +++ b/nbs/02_card.ipynb @@ -294,6 +294,7 @@ "- [Eval](01_eval.html) - where `k`, `n`, the delta and the agreement come from\n", "- [Gate](03_gate.html) - condition 7 refuses to publish a card `check_card` flags\n", "- [Model](00_model.html) - the artifact the card describes\n", + "- [Measurement](04_measurement.html) - the on-device measurement rows a latency comes from\n", "\n", "Tests live in `nbs/tests/test_card.ipynb`." ] diff --git a/nbs/03_gate.ipynb b/nbs/03_gate.ipynb index c796619..470821f 100644 --- a/nbs/03_gate.ipynb +++ b/nbs/03_gate.ipynb @@ -325,6 +325,7 @@ "- [Model](00_model.html) - `state_hash`, the digest condition 1 compares\n", "- [Card](02_card.html) - `check_card`, which condition 7 runs\n", "- [Eval](01_eval.html) - the paired interval condition 3 reads\n", + "- [Measurement](04_measurement.html) - the validity checks of one on-device measurement row\n", "\n", "Tests live in `nbs/tests/test_gate.ipynb`." ] diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb new file mode 100644 index 0000000..5342d32 --- /dev/null +++ b/nbs/04_measurement.ipynb @@ -0,0 +1,690 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "frontmatter", + "metadata": {}, + "source": [ + "---\n", + "description: On-device latency rows and accuracy rows, and the checks that decide whether each is well formed and valid\n", + "output-file: measurement.html\n", + "title: Measurement\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "default-exp", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp measurement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "from __future__ import annotations\n", + "\n", + "import hashlib\n", + "import json\n", + "import math\n", + "import numbers\n", + "import re\n", + "from datetime import datetime\n", + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "showdoc-setup", + "metadata": {}, + "outputs": [], + "source": [ + "#| include: false\n", + "from nbdev.showdoc import *" + ] + }, + { + "cell_type": "markdown", + "id": "overview", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "Speed and accuracy do not depend on the same things. How fast an engine runs depends on its **graph** — the\n", + "architecture, the widths, the numeric precision — and on the device and software stack; it does not depend on\n", + "the values of the weights. How accurate it is depends on the **weights** and on the evaluation set. So a\n", + "measurement is one of two kinds of row, and neither needs the other:\n", + "\n", + "- a **latency row** times one engine on one device. It needs no accuracy, so a model with random weights\n", + " (`weights='random'`) is measured like any other: this is how a latency predictor gets trained on\n", + " architectures nobody has trained yet.\n", + "- a **quality row** measures one artifact's accuracy on one evaluation set, through the engine that runs it.\n", + "\n", + "Rows are stored as JSON Lines, one object per line, appended and never rewritten; a file may hold both kinds.\n", + "`FIELDS[kind]` is the schema of each kind: every field's type, whether it is required, and the rules its\n", + "value obeys. A field that is not required may be absent or `None`, which means *not recorded*.\n", + "\n", + "| Kind | Group | Fields |\n", + "|---|---|---|\n", + "| both | identity and provenance | `schema_version`, `kind`, `run_id`, `measured_at` (ISO-8601, UTC), `series`, `harness_commit`, `fasterai_version`, `report_url` |\n", + "| latency | graph | `architecture`, `graph_hash`, `input_resolution`, `macs`, `params`, `variant`, `recipe`, `weights` (`trained` or `random`), `model_id`, `model_revision` |\n", + "| latency | artifact | `precision`, `int8_mode` (iff `int8`), `engine_hash`, `size_mb`, `engine_layers`, `reformat_layers`, `layer_precisions` |\n", + "| latency | device and runtime | `device`, `device_host`, `runtime`, `runtime_version`, `platform_version`, `power_mode`, `clocks_locked`, `batch_size` |\n", + "| latency | conditions | `temp_start_c`, `temp_end_c`, `gpu_freq_mhz`, `throttled`, `warmup_s`, `duration_s`, `repeats`, `latency_spread_pct` |\n", + "| latency | latency | `latency_ms_p50`, `latency_ms_p90`, `latency_ms_p99`, `latency_scope`, `host_latency_ms`, `latency_source` |\n", + "| quality | model and artifact | `model_id`, `model_revision`, `model_hash`, `weights_license`, `engine_hash`, `precision`, `device` |\n", + "| quality | accuracy | `metric`, `eval_set`, `eval_set_hash`, `dataset_license`, `eval_n`, `accuracy`, `ci_low`, `ci_high`, `ci_method`, `measured_through`, `argmax_parity`, `parity_floor`, `baseline_run_id` |\n", + "\n", + "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of any key.\n", + "- `graph_hash` is the sha256 of the ONNX graph with the values of every initializer removed, so two models of the same shape share it, whatever their weights. `graph_hash(onnx_path)` computes it.\n", + "- `latency_scope` says what the latency covers, e.g. engine execution only, or host transfers included; two latency rows are only comparable when it matches.\n", + "- `layer_precisions` counts the engine's layers by the precision they actually ran in, e.g. `{'int8': 52, 'fp16': 9}`.\n", + "- `engine_hash` is the sha256 of the engine file that ran; `model_hash` is whatever fingerprint the exporter records, so it is only a string.\n", + "- A quality row names the engine it was measured through; `engine_hash` may be `None` only for an accuracy measured on the development model.\n", + "\n", + "Throughput is derived, never stored: `throughput(row)` is `batch_size * 1000 / latency_ms_p50` on a latency row.\n", + "\n", + "Two questions are kept apart: `validate` asks whether a row is **well formed**, `status` whether the\n", + "measurement it records is **valid**. Neither compares a row to a latency or accuracy budget: a row carries no\n", + "verdict column, and whoever reads it recomputes their own. A well-formed row is `INVALID` when:\n", + "\n", + "- latency: `latency_source` is not `measured`; `throttled` is true or not recorded; `latency_spread_pct` is\n", + " above `max_spread_pct` (5 by default) or not recorded;\n", + "- quality: `measured_through` is not `deployed_engine`; `argmax_parity` is below `parity_floor` or not recorded.\n", + "\n", + "`join(latency_rows, quality_rows)` pairs the VALID rows of both kinds that share an `engine_hash`: one dict per\n", + "pair, with the fields of both. The identity and provenance fields are kept per side, as `latency_run_id`,\n", + "`quality_run_id`, `latency_measured_at` and so on; `schema_version` is kept once. The fields both kinds carry\n", + "(`engine_hash`, `precision`, `device`, `model_id`, `model_revision`) are one column: a side that did not\n", + "record one takes the other's value, and two sides that disagree on the same engine raise." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "schema", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "SCHEMA_VERSION = 1\n", + "_UTC = r'\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|\\+00:00)'\n", + "_HEX = r'[0-9a-f]{64}'\n", + "_PCT = dict(ge=0, le=100)\n", + "_PRECISION = dict(type=str, required=True, choices=('fp32', 'fp16', 'int8', 'mixed'))\n", + "\n", + "# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None\n", + "_SHARED = {\n", + " 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)),\n", + " 'kind': dict(type=str, required=True, choices=('latency', 'quality')),\n", + " 'run_id': dict(type=str, required=True),\n", + " 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True),\n", + " 'series': dict(type=str, required=False),\n", + " 'harness_commit': dict(type=str, required=True),\n", + " 'fasterai_version': dict(type=str, required=False),\n", + " 'report_url': dict(type=str, required=False),\n", + "}\n", + "\n", + "_LATENCY = {\n", + " # graph\n", + " 'architecture': dict(type=str, required=True),\n", + " 'graph_hash': dict(type=str, required=True, pattern=_HEX), # `graph_hash`: the ONNX graph without its weight values\n", + " 'input_resolution': dict(type=int, required=True, gt=0),\n", + " 'macs': dict(type=int, required=True, gt=0),\n", + " 'params': dict(type=int, required=True, gt=0),\n", + " 'variant': dict(type=str, required=True),\n", + " 'recipe': dict(type=dict, required=True, json=True),\n", + " 'weights': dict(type=str, required=True, choices=('trained', 'random')),\n", + " 'model_id': dict(type=str, required=False),\n", + " 'model_revision': dict(type=str, required=False),\n", + " # artifact\n", + " 'precision': _PRECISION,\n", + " 'int8_mode': dict(type=str, required=False, choices=('implicit', 'explicit')),\n", + " 'engine_hash': dict(type=str, required=True, pattern=_HEX), # sha256 of the engine file that ran\n", + " 'size_mb': dict(type=float, required=True, gt=0),\n", + " 'engine_layers': dict(type=int, required=False, gt=0),\n", + " 'reformat_layers': dict(type=int, required=False, ge=0),\n", + " 'layer_precisions': dict(type=dict, required=False, counts=True),\n", + " # device and runtime\n", + " 'device': dict(type=str, required=True),\n", + " 'device_host': dict(type=str, required=True, choices=('own', 'rented', 'partner')),\n", + " 'runtime': dict(type=str, required=True, choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera')),\n", + " 'runtime_version': dict(type=str, required=True),\n", + " 'platform_version': dict(type=str, required=False),\n", + " 'power_mode': dict(type=str, required=False),\n", + " 'clocks_locked': dict(type=bool, required=False),\n", + " 'batch_size': dict(type=int, required=True, ge=1),\n", + " # conditions\n", + " 'temp_start_c': dict(type=float, required=False),\n", + " 'temp_end_c': dict(type=float, required=False),\n", + " 'gpu_freq_mhz': dict(type=float, required=False, gt=0),\n", + " 'throttled': dict(type=bool, required=False),\n", + " 'warmup_s': dict(type=float, required=False, ge=0),\n", + " 'duration_s': dict(type=float, required=False, ge=0),\n", + " 'repeats': dict(type=int, required=True, ge=1),\n", + " 'latency_spread_pct': dict(type=float, required=False, ge=0),\n", + " # latency\n", + " 'latency_ms_p50': dict(type=float, required=True, gt=0),\n", + " 'latency_ms_p90': dict(type=float, required=False, gt=0),\n", + " 'latency_ms_p99': dict(type=float, required=False, gt=0),\n", + " 'latency_scope': dict(type=str, required=True),\n", + " 'host_latency_ms': dict(type=float, required=False, gt=0),\n", + " 'latency_source': dict(type=str, required=True, choices=('measured', 'estimated')),\n", + "}\n", + "\n", + "_QUALITY = {\n", + " 'model_id': dict(type=str, required=True),\n", + " 'model_revision': dict(type=str, required=True),\n", + " 'model_hash': dict(type=str, required=False), # whatever fingerprint the exporter records, so not pinned\n", + " 'weights_license': dict(type=str, required=True),\n", + " 'engine_hash': dict(type=str, required=False, pattern=_HEX), # the engine the accuracy was measured through\n", + " 'precision': _PRECISION,\n", + " 'device': dict(type=str, required=False),\n", + " 'metric': dict(type=str, required=True),\n", + " 'eval_set': dict(type=str, required=True),\n", + " 'eval_set_hash': dict(type=str, required=True),\n", + " 'dataset_license': dict(type=str, required=True),\n", + " 'eval_n': dict(type=int, required=True, gt=0),\n", + " 'accuracy': dict(type=float, required=True, **_PCT),\n", + " 'ci_low': dict(type=float, required=True, **_PCT),\n", + " 'ci_high': dict(type=float, required=True, **_PCT),\n", + " 'ci_method': dict(type=str, required=True, choices=('wilson', 'bootstrap')),\n", + " 'measured_through': dict(type=str, required=True, choices=('deployed_engine', 'dev_model')),\n", + " 'argmax_parity': dict(type=float, required=False, **_PCT),\n", + " 'parity_floor': dict(type=float, required=True, **_PCT),\n", + " 'baseline_run_id': dict(type=str, required=False),\n", + "}\n", + "\n", + "FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}}\n", + "\n", + "\n", + "def _item(o):\n", + " \"A numpy scalar as the Python number JSON knows\"\n", + " if isinstance(o, numbers.Number) and hasattr(o, 'item'): return o.item()\n", + " raise TypeError(f'{type(o).__name__} is not JSON serializable')\n", + "\n", + "\n", + "def _json(obj, **kw): return json.dumps(obj, default=_item, allow_nan=False, **kw)\n", + "def _canonical(d): return _json(d, sort_keys=True, separators=(',', ':'))\n", + "\n", + "\n", + "def _is_json(v):\n", + " try: return _json(v) is not None\n", + " except (TypeError, ValueError): return False\n", + "\n", + "\n", + "def _is_date(v):\n", + " try: return datetime.fromisoformat(v.replace('Z', '+00:00')) is not None # Python 3.10 does not read 'Z'\n", + " except ValueError: return False\n", + "\n", + "\n", + "def _typed(v, t):\n", + " \"`v` is a `t`; a bool is not a number, a numpy integer is an int, an int is a float, a float is finite\"\n", + " if isinstance(v, bool): return t is bool\n", + " if t is int: return isinstance(v, numbers.Integral)\n", + " if t is float: return isinstance(v, numbers.Real) and math.isfinite(v)\n", + " return isinstance(v, t)\n", + "\n", + "\n", + "_RULES = {\n", + " 'choices': (lambda v, c: v in c, 'not one of {}'),\n", + " 'gt': (lambda v, b: v > b, 'not > {}'),\n", + " 'ge': (lambda v, b: v >= b, 'not >= {}'),\n", + " 'le': (lambda v, b: v <= b, 'not <= {}'),\n", + " 'pattern': (lambda v, p: re.fullmatch(p, v) is not None, 'does not match {}'),\n", + " 'date': (lambda v, _: _is_date(v), 'is not a date'),\n", + " 'json': (lambda v, _: _is_json(v), 'is not JSON'),\n", + " 'counts': (lambda v, _: all(isinstance(k, str) and _typed(n, int) and n >= 0 for k, n in v.items()),\n", + " 'is not a {{name: count}} dict'),\n", + "}\n", + "_ORDERED = {'latency': (('latency_ms_p50', 'latency_ms_p90', 'latency_ms_p99'), ('reformat_layers', 'engine_layers')),\n", + " 'quality': (('ci_low', 'accuracy', 'ci_high'),)}\n", + "\n", + "\n", + "def _field_problems(name, spec, v):\n", + " \"The problems of one field value, against its spec\"\n", + " if v is None: return [f'{name}: missing'] if spec['required'] else []\n", + " if not _typed(v, spec['type']): return [f'{name}: expected {spec[\"type\"].__name__}, got {v!r}']\n", + " return [f'{name}: {v!r} {msg.format(spec[r])}' for r, (ok, msg) in _RULES.items() if r in spec and not ok(v, spec[r])]\n", + "\n", + "\n", + "def _cross_problems(row):\n", + " \"Problems between fields; each field involved is already well typed\"\n", + " out, kind = [], row['kind']\n", + " if kind == 'latency':\n", + " p, mode = row['precision'], row.get('int8_mode')\n", + " if p == 'int8' and mode is None: out.append('int8_mode: missing, required when precision is int8')\n", + " if p != 'int8' and mode is not None: out.append(f'int8_mode: {mode!r} given for precision {p!r}')\n", + " if row['weights'] == 'random' and row.get('model_id') is not None:\n", + " out.append(f\"model_id: {row['model_id']!r} given for random weights\")\n", + " elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None:\n", + " out.append('engine_hash: missing, required when measured through the deployed engine')\n", + " for fields in _ORDERED[kind]:\n", + " vals = [(f, row[f]) for f in fields if row.get(f) is not None]\n", + " if [v for _, v in vals] != sorted(v for _, v in vals):\n", + " out.append(f\"{', '.join(fields)}: not in order, \" + ', '.join(f'{f}={v}' for f, v in vals))\n", + " return out\n", + "\n", + "\n", + "def validate(\n", + " row: dict, # one measurement row, of either kind\n", + ") -> list[str]:\n", + " \"Every structural problem of `row`, each prefixed by its field name; an empty list means well formed\"\n", + " if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}']\n", + " kind = row.get('kind')\n", + " if kind is None: return ['kind: missing']\n", + " if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}']\n", + " fields = FIELDS[kind]\n", + " out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields]\n", + " for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name))\n", + " return out or _cross_problems(row)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-validate", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(validate)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "status", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "# per kind: field, when its value makes the measurement invalid, and why; a field that is None is not recorded\n", + "_VALIDITY = {\n", + " 'latency': (\n", + " ('latency_source', lambda v, r, m: v != 'measured', 'latency not measured on the device'),\n", + " ('throttled', lambda v, r, m: v, 'the device throttled during the session'),\n", + " ('latency_spread_pct', lambda v, r, m: v > m, '{v} above {m}'),\n", + " ),\n", + " 'quality': (\n", + " ('measured_through', lambda v, r, m: v != 'deployed_engine', 'accuracy not measured through the deployed engine'),\n", + " ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'),\n", + " ),\n", + "}\n", + "\n", + "\n", + "def status(\n", + " row: dict, # one measurement row, of either kind\n", + " max_spread_pct: float = 5.0, # largest (max-min)/median latency spread over repeats, in percent\n", + ") -> tuple[str, list[str]]:\n", + " \"('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget\"\n", + " reasons = validate(row)\n", + " if reasons: return 'INVALID', reasons\n", + " for f, bad, why in _VALIDITY[row['kind']]:\n", + " v = row.get(f)\n", + " if v is None: reasons.append(f'{f}: not recorded')\n", + " elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct))\n", + " return ('INVALID', reasons) if reasons else ('VALID', [])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-status", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(status)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "key", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_KEYS = {\n", + " 'latency': ('graph_hash', 'weights', 'precision', 'int8_mode', 'batch_size', 'device', 'runtime',\n", + " 'runtime_version', 'platform_version', 'measured_at'),\n", + " 'quality': ('model_id', 'model_revision', 'engine_hash', 'eval_set_hash', 'metric', 'measured_at'),\n", + "}\n", + "\n", + "\n", + "def key(\n", + " row: dict, # one well-formed measurement row\n", + ") -> tuple:\n", + " \"The identity a row is unique on, its kind first; a software update on the device gives a new latency key\"\n", + " return (row['kind'],) + tuple(row.get(f) for f in _KEYS[row['kind']])\n", + "\n", + "\n", + "def throughput(\n", + " row: dict, # one latency row\n", + ") -> float:\n", + " \"Images per second at the timed batch, derived from the median latency and never stored\"\n", + " if row.get('kind') != 'latency': raise ValueError(f\"throughput needs a latency row, got kind {row.get('kind')!r}\")\n", + " return row['batch_size'] * 1000 / row['latency_ms_p50']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-key", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-throughput", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(throughput)" + ] + }, + { + "cell_type": "markdown", + "id": "key-note", + "metadata": {}, + "source": [ + "A latency row is unique on its graph, its weights (`trained` or `random`), precision, batch size,\n", + "device, software stack and date; a quality row on its model, revision, engine, evaluation set, metric and\n", + "date. Uniqueness is checked per kind." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "join", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed\n", + "_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree\n", + "\n", + "\n", + "def _pair(lat, qual):\n", + " \"One joined dict from a latency row and a quality row of the same engine\"\n", + " out = {'schema_version': lat['schema_version']}\n", + " for side, r in (('latency', lat), ('quality', qual)):\n", + " out |= {f'{side}_{f}': r.get(f) for f in _OWN}\n", + " out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH}\n", + " for f in _BOTH:\n", + " a, b = lat.get(f), qual.get(f)\n", + " if None not in (a, b) and a != b:\n", + " raise ValueError(f\"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine\")\n", + " out[f] = b if a is None else a\n", + " return out\n", + "\n", + "\n", + "def join(\n", + " latency_rows: list[dict], # latency rows; rows of the other kind and invalid rows are skipped\n", + " quality_rows: list[dict], # quality rows; rows of the other kind and invalid rows are skipped\n", + ") -> list[dict]:\n", + " \"One dict per pair of VALID latency and quality rows sharing an `engine_hash`\"\n", + " by_engine = {}\n", + " for q in quality_rows:\n", + " if q.get('kind') == 'quality' and status(q)[0] == 'VALID': by_engine.setdefault(q['engine_hash'], []).append(q)\n", + " return [_pair(l, q) for l in latency_rows if l.get('kind') == 'latency' and status(l)[0] == 'VALID'\n", + " for q in by_engine.get(l['engine_hash'], [])]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-join", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(join)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "graph", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def graph_hash(\n", + " onnx_path: str | Path, # an ONNX file\n", + ") -> str:\n", + " \"sha256 of the ONNX graph with every initializer's values removed and its name, shape and type kept\"\n", + " try: import onnx\n", + " except ImportError: raise ImportError(\"graph_hash needs onnx: pip install 'fastermodels[onnx]'\") from None\n", + " m = onnx.load(str(onnx_path), load_external_data=False)\n", + " # ponytail: top-level initializers only; Constant nodes and subgraph initializers keep their values\n", + " for t in m.graph.initializer:\n", + " name, dtype, dims = t.name, t.data_type, list(t.dims)\n", + " t.Clear()\n", + " t.name, t.data_type = name, dtype\n", + " t.dims.extend(dims)\n", + " h = hashlib.sha256(m.graph.SerializeToString(deterministic=True))\n", + " for o in sorted((o.domain, o.version) for o in m.opset_import): h.update(f'{o[0]}:{o[1]};'.encode())\n", + " return h.hexdigest()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-graph-hash", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(graph_hash)" + ] + }, + { + "cell_type": "markdown", + "id": "graph-note", + "metadata": {}, + "source": [ + "Initializer names are part of the graph: renaming a weight gives a new hash, as it does to the\n", + "engine a runtime builds from it. Opset versions are hashed too. `graph_hash` needs the optional `onnx`\n", + "extra: `pip install 'fastermodels[onnx]'`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "io", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _parse(text, path):\n", + " \"The rows of a JSON Lines text; a malformed line raises with its number\"\n", + " rows = []\n", + " for i, line in enumerate(text.splitlines(), 1):\n", + " if not line.strip(): continue\n", + " try: r = json.loads(line)\n", + " except json.JSONDecodeError as e: raise ValueError(f'{path}:{i}: not JSON ({e.msg})') from None\n", + " if not isinstance(r, dict): raise ValueError(f'{path}:{i}: not a JSON object')\n", + " rows.append(r)\n", + " return rows\n", + "\n", + "\n", + "def read_rows(\n", + " path: str | Path, # a JSON Lines file of measurement rows, of either kind\n", + ") -> list[dict]:\n", + " \"The rows of `path`, as stored; `status` judges them\"\n", + " return _parse(Path(path).read_text(), path)\n", + "\n", + "\n", + "def write_row(\n", + " path: str | Path, # a JSON Lines file, created if absent\n", + " row: dict, # the measurement row to append\n", + "):\n", + " \"Append `row` to `path`; refuses a malformed row and a row whose key is already in the file\"\n", + " problems = validate(row)\n", + " if problems: raise ValueError('row refused: ' + '; '.join(problems))\n", + " p, k = Path(path), key(row)\n", + " text = p.read_text() if p.exists() else ''\n", + " if k in {key(r) for r in _parse(text, path) if r.get('kind') == row['kind']}:\n", + " raise ValueError(f'row refused: key {k} already in {path}')\n", + " # ponytail: no file lock, one writer at a time\n", + " with p.open('a') as f: f.write(('\\n' if text and not text.endswith('\\n') else '') + _json(row) + '\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-read-rows", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(read_rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-write-row", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(write_row)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "parquet", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_ARROW = {str: 'string', int: 'int64', float: 'float64', bool: 'bool_', dict: 'string'} # a dict is stored as canonical JSON\n", + "\n", + "\n", + "def to_parquet(\n", + " rows: list[dict], # well-formed measurement rows of one kind, e.g. from `read_rows`\n", + " path: str | Path, # the Parquet file to write\n", + "):\n", + " \"Write `rows` as one Parquet table with an explicit schema: one typed column per field of their kind\"\n", + " try:\n", + " import pyarrow as pa\n", + " import pyarrow.parquet as pq\n", + " except ImportError: raise ImportError(\"to_parquet needs pyarrow: pip install 'fastermodels[parquet]'\") from None\n", + " kinds = sorted({str(r.get('kind')) for r in rows})\n", + " if len(kinds) != 1: raise ValueError(f'to_parquet writes one kind per file, got {kinds or \"no rows\"}')\n", + " for i, r in enumerate(rows):\n", + " problems = validate(r)\n", + " if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems))\n", + " fields = FIELDS[kinds[0]]\n", + " schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()])\n", + " plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers\n", + " table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields}\n", + " for r in plain]\n", + " pq.write_table(pa.Table.from_pylist(table, schema=schema), str(path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "show-to-parquet", + "metadata": {}, + "outputs": [], + "source": [ + "show_doc(to_parquet)" + ] + }, + { + "cell_type": "markdown", + "id": "parquet-note", + "metadata": {}, + "source": [ + "`to_parquet` writes one kind per file, with an explicit Arrow schema derived from `FIELDS`: `str` is a\n", + "string column, `int` int64, `float` float64, `bool` bool, a `dict` a string holding canonical JSON, and a\n", + "field that is not required is nullable. A missing value is a null in the file, whatever reads it. It needs\n", + "the optional `parquet` extra: `pip install 'fastermodels[parquet]'`." + ] + }, + { + "cell_type": "markdown", + "id": "usage", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Usage\n", + "\n", + "A latency row, here for an engine built from a model with random weights:\n", + "\n", + "```python\n", + "from fastermodels import measurement\n", + "\n", + "lat = dict(schema_version=1, kind='latency', run_id='2026-09-28-a', measured_at='2026-09-28T10:00:00Z',\n", + " harness_commit='deadbee', architecture='resnet50', graph_hash=measurement.graph_hash('model.onnx'),\n", + " input_resolution=224, macs=2_900_000_000, params=17_000_000, variant='p30', recipe={'prune': 0.30},\n", + " weights='random', precision='fp16', engine_hash='0' * 64, size_mb=34.1,\n", + " layer_precisions={'fp16': 61}, device='jetson-nano-4gb', device_host='own', runtime='tensorrt',\n", + " runtime_version='8.2.1', platform_version='jetpack-4.6.1', batch_size=1, throttled=False,\n", + " repeats=5, latency_spread_pct=1.2, latency_ms_p50=2.1,\n", + " latency_scope='engine execute, host transfers excluded', latency_source='measured')\n", + "\n", + "measurement.validate(lat) # [] : well formed\n", + "measurement.status(lat) # ('VALID', [])\n", + "measurement.throughput(lat) # 476.19... images per second, derived\n", + "measurement.write_row('rows.jsonl', lat) # appends one line; a second call with the same row is refused\n", + "```\n", + "\n", + "A quality row, and the join of the two kinds on the engine they share:\n", + "\n", + "```python\n", + "qual = dict(schema_version=1, kind='quality', run_id='2026-09-28-b', measured_at='2026-09-28T11:00:00Z',\n", + " harness_commit='deadbee', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", + " weights_license='apache-2.0', engine_hash='1' * 64, precision='fp16', device='jetson-nano-4gb',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:...', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0)\n", + "\n", + "rows = measurement.read_rows('rows.jsonl')\n", + "measurement.join(rows, rows) # each kind is picked from the list it belongs to\n", + "measurement.to_parquet([r for r in rows if r['kind'] == 'latency'], 'latency.parquet')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "see-also", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## See Also\n", + "\n", + "- [Model](00_model.html) - the published model a quality row measures\n", + "- [Eval](01_eval.html) - `wilson`, the interval `ci_method='wilson'` refers to, and `agreement`\n", + "- [Card](02_card.html) - where a published latency is written down\n", + "- [Gate](03_gate.html) - the conditions an artifact meets before it is published\n", + "\n", + "Tests live in `nbs/tests/test_measurement.ipynb`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/_quarto.yml b/nbs/_quarto.yml index 7b58c57..b23b528 100644 --- a/nbs/_quarto.yml +++ b/nbs/_quarto.yml @@ -34,5 +34,6 @@ website: - 01_eval.ipynb - 02_card.ipynb - 03_gate.ipynb + - 04_measurement.ipynb metadata-files: [nbdev.yml] diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb new file mode 100644 index 0000000..13851d0 --- /dev/null +++ b/nbs/tests/test_measurement.ipynb @@ -0,0 +1,551 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "title", + "metadata": {}, + "source": [ + "# Tests: `fastermodels.measurement` (source `nbs/04_measurement.ipynb`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "from fastcore.test import *\n", + "import json, re, sys, tempfile\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "\n", + "import fastermodels\n", + "from fastermodels import measurement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fixture", + "metadata": {}, + "outputs": [], + "source": [ + "_ENGINE = 'ab' * 32\n", + "_LAT = dict(schema_version=1, kind='latency', run_id='l1', measured_at='2026-09-28T10:00:00Z', series='2026-09-28',\n", + " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None,\n", + " architecture='resnet50', graph_hash='cd' * 32, input_resolution=224, macs=2_900_000_000,\n", + " params=17_000_000, variant='p30-int8', recipe={'prune': 0.30, 'quant': 'int8', 'calib_n': 512},\n", + " weights='trained', model_id='org/resnet50-p30', model_revision='a1b2c3d',\n", + " precision='int8', int8_mode='explicit', engine_hash=_ENGINE, size_mb=18.2, engine_layers=61,\n", + " reformat_layers=2, layer_precisions={'int8': 52, 'fp16': 9},\n", + " device='jetson-nano-4gb', device_host='own', runtime='tensorrt', runtime_version='8.2.1',\n", + " platform_version='jetpack-4.6.1', power_mode='MAXN', clocks_locked=True, batch_size=4,\n", + " temp_start_c=41.0, temp_end_c=47.5, gpu_freq_mhz=921.6, throttled=False, warmup_s=10.0,\n", + " duration_s=60.0, repeats=5, latency_spread_pct=1.2,\n", + " latency_ms_p50=8.0, latency_ms_p90=8.4, latency_ms_p99=9.1, latency_scope='engine execute',\n", + " host_latency_ms=8.5, latency_source='measured')\n", + "\n", + "# the same graph with random weights: no model behind it, no accuracy, a first-class latency row\n", + "_RAND = {**{k: v for k, v in _LAT.items() if k not in ('model_id', 'model_revision')},\n", + " 'run_id': 'l2', 'weights': 'random', 'engine_hash': 'ef' * 32}\n", + "\n", + "_QUAL = dict(schema_version=1, kind='quality', run_id='q1', measured_at='2026-09-28T11:00:00Z', series='2026-09-28',\n", + " harness_commit='deadbee', fasterai_version='0.4.0', report_url=None,\n", + " model_id='org/resnet50-p30', model_revision='a1b2c3d', model_hash=None, weights_license='apache-2.0',\n", + " engine_hash=_ENGINE, precision='int8', device='jetson-nano-4gb',\n", + " metric='top1', eval_set='imagenet-val', eval_set_hash='sha256:0f', dataset_license='custom',\n", + " eval_n=50_000, accuracy=78.9, ci_low=78.5, ci_high=79.3, ci_method='wilson',\n", + " measured_through='deployed_engine', argmax_parity=99.7, parity_floor=99.0, baseline_run_id='q0')\n", + "\n", + "\n", + "def edit(row, **change):\n", + " \"`row` with some fields changed; a value of `...` removes the field\"\n", + " return {k: v for k, v in {**row, **change}.items() if v is not ...}\n", + "\n", + "\n", + "def problems(row, **change): return measurement.validate(edit(row, **change))\n", + "def names(ps): return {p.split(':')[0] for p in ps}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "valid", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(set(_LAT), set(measurement.FIELDS['latency']))\n", + "test_eq(set(_QUAL), set(measurement.FIELDS['quality']))\n", + "test_eq(measurement.SCHEMA_VERSION, 1)\n", + "for _r in (_LAT, _RAND, _QUAL): test_eq(measurement.validate(_r), [])\n", + "\n", + "# the shared fields are defined once, and the schema is data\n", + "_shared = ('schema_version', 'kind', 'run_id', 'measured_at', 'series', 'harness_commit', 'fasterai_version', 'report_url')\n", + "for _f in _shared: test_is(measurement.FIELDS['latency'][_f], measurement.FIELDS['quality'][_f])\n", + "\n", + "# a latency row needs no accuracy, a quality row no latency\n", + "test_eq(problems(_LAT, accuracy=78.9), ['accuracy: unknown field for a latency row'])\n", + "test_eq(problems(_QUAL, latency_ms_p50=8.0), ['latency_ms_p50: unknown field for a quality row'])\n", + "\n", + "# optional fields may be absent: absent means not recorded\n", + "test_eq(problems(_LAT, series=..., fasterai_version=..., platform_version=..., temp_start_c=..., latency_ms_p90=...,\n", + " latency_ms_p99=..., engine_layers=..., reformat_layers=..., layer_precisions=..., model_id=...,\n", + " model_revision=...), [])\n", + "test_eq(problems(_QUAL, model_hash=..., device=..., baseline_run_id=..., argmax_parity=...), [])\n", + "# JSON writes 8 as 8: an int is accepted where a float is expected\n", + "test_eq(problems(_LAT, latency_ms_p50=8), [])\n", + "\n", + "# numpy scalars, as a harness produces them, are numbers; a numpy or Python bool is not\n", + "test_eq(problems(_LAT, batch_size=np.int64(4), macs=np.uint64(2_900_000_000), latency_ms_p50=np.float64(8.0)), [])\n", + "test_eq(problems(_QUAL, eval_n=np.int32(50_000), accuracy=np.float32(78.9)), [])\n", + "test_eq(names(problems(_LAT, batch_size=np.bool_(True))), {'batch_size'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=np.float64('nan'))), {'latency_ms_p50'})\n", + "\n", + "# the module is reached by its name, its generic function names are not at the package top level\n", + "test_eq(fastermodels.measurement is measurement, True)\n", + "test_eq(hasattr(fastermodels, 'status') or hasattr(fastermodels, 'key'), False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "kind", + "metadata": {}, + "outputs": [], + "source": [ + "# the kind decides the schema: missing or unknown, one clear problem and nothing else\n", + "test_eq(problems(_LAT, kind=...), ['kind: missing'])\n", + "test_eq(problems(_LAT, kind=None), ['kind: missing'])\n", + "test_eq(problems(_LAT, kind='speed'), [\"kind: 'speed' not one of ('latency', 'quality')\"])\n", + "test_eq(problems(_LAT, kind=['latency']), [\"kind: ['latency'] not one of ('latency', 'quality')\"])\n", + "# a row of one kind is not a row of the other\n", + "test_eq(names(problems(_LAT, kind='quality')) >= {'architecture', 'accuracy', 'weights_license'}, True)\n", + "# not a row at all\n", + "test_eq(measurement.validate([_LAT]), ['row: expected a dict, got list'])\n", + "test_eq(measurement.validate(None), ['row: expected a dict, got NoneType'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fields", + "metadata": {}, + "outputs": [], + "source": [ + "# missing required, by name\n", + "test_eq(problems(_LAT, run_id=...), ['run_id: missing'])\n", + "test_eq(problems(_LAT, latency_ms_p50=None), ['latency_ms_p50: missing'])\n", + "test_eq(problems(_LAT, graph_hash=...), ['graph_hash: missing'])\n", + "test_eq(problems(_LAT, weights=...), ['weights: missing'])\n", + "test_eq(problems(_QUAL, weights_license=...), ['weights_license: missing'])\n", + "test_eq(names(problems(_QUAL, model_id=..., eval_set_hash=...)), {'model_id', 'eval_set_hash'})\n", + "\n", + "# wrong type\n", + "test_eq(problems(_LAT, batch_size='4'), [\"batch_size: expected int, got '4'\"])\n", + "test_eq(names(problems(_LAT, recipe=[0.3])), {'recipe'})\n", + "test_eq(names(problems(_LAT, variant={'prune': 0.3})), {'variant'})\n", + "test_eq(names(problems(_LAT, runtime_version=8.2)), {'runtime_version'})\n", + "test_eq(names(problems(_LAT, throttled='no')), {'throttled'})\n", + "test_eq(names(problems(_LAT, batch_size=4.0)), {'batch_size'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=float('inf'))), {'latency_ms_p50'})\n", + "test_eq(names(problems(_LAT, layer_precisions=[52, 9])), {'layer_precisions'})\n", + "\n", + "# a bool is an int in Python, and neither a count nor a measure here\n", + "test_eq(problems(_LAT, batch_size=True), ['batch_size: expected int, got True'])\n", + "test_eq(names(problems(_LAT, schema_version=True)), {'schema_version'})\n", + "test_eq(names(problems(_LAT, latency_ms_p50=True)), {'latency_ms_p50'})\n", + "test_eq(names(problems(_LAT, engine_layers=True)), {'engine_layers'})\n", + "test_eq(names(problems(_QUAL, eval_n=False)), {'eval_n'})\n", + "\n", + "# out of range\n", + "for _f, _v in [('batch_size', 0), ('repeats', 0), ('input_resolution', 0), ('latency_ms_p50', 0.0),\n", + " ('latency_ms_p99', 0.0), ('size_mb', -1.0), ('latency_spread_pct', -0.1), ('warmup_s', -1.0),\n", + " ('duration_s', -1.0), ('host_latency_ms', 0), ('engine_layers', 0), ('reformat_layers', -1)]:\n", + " test_eq(names(problems(_LAT, **{_f: _v})), {_f})\n", + "test_eq(problems(_LAT, warmup_s=0, reformat_layers=0), [])\n", + "test_eq(names(problems(_QUAL, eval_n=0)), {'eval_n'})\n", + "test_eq(names(problems(_QUAL, argmax_parity=100.5)), {'argmax_parity'})\n", + "test_eq(names(problems(_QUAL, accuracy=150.0, ci_high=160.0)), {'accuracy', 'ci_high'})\n", + "\n", + "# a count per precision\n", + "test_eq(problems(_LAT, layer_precisions={}), [])\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': -1})), {'layer_precisions'})\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': 2.5})), {'layer_precisions'})\n", + "test_eq(names(problems(_LAT, layer_precisions={'int8': True})), {'layer_precisions'})\n", + "test_eq(problems(_LAT, layer_precisions={'int8': np.int64(52)}), [])\n", + "\n", + "# closed vocabularies\n", + "for _r, _f, _v in [(_LAT, 'precision', 'bf16'), (_LAT, 'weights', 'pretrained'), (_LAT, 'runtime', 'TensorRT 8.2.1'),\n", + " (_LAT, 'device_host', 'borrowed'), (_LAT, 'latency_source', 'guessed'),\n", + " (_QUAL, 'precision', 'bf16'), (_QUAL, 'ci_method', 'normal'), (_QUAL, 'measured_through', 'notebook')]:\n", + " test_eq(names(problems(_r, **{_f: _v, **({'int8_mode': None} if _f == 'precision' and _r is _LAT else {})})), {_f})\n", + "\n", + "# schema version mismatch\n", + "test_eq(problems(_LAT, schema_version=2), ['schema_version: 2 not one of (1,)'])\n", + "\n", + "# unknown fields: a derived throughput, a verdict, and a name from before the split\n", + "test_eq(problems(_LAT, throughput_img_s=500.0), ['throughput_img_s: unknown field for a latency row'])\n", + "test_eq(problems(_QUAL, passed=True), ['passed: unknown field for a quality row'])\n", + "test_eq(problems(_LAT, latency_ms=8.0), ['latency_ms: unknown field for a latency row'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cross", + "metadata": {}, + "outputs": [], + "source": [ + "# int8_mode is required iff the precision is int8\n", + "test_eq(problems(_LAT, int8_mode=None), ['int8_mode: missing, required when precision is int8'])\n", + "test_eq(problems(_LAT, precision='fp16'), [\"int8_mode: 'explicit' given for precision 'fp16'\"])\n", + "test_eq(problems(_LAT, precision='mixed'), [\"int8_mode: 'explicit' given for precision 'mixed'\"])\n", + "test_eq(problems(_LAT, precision='fp16', int8_mode=None), [])\n", + "test_eq(names(problems(_LAT, int8_mode='dynamic')), {'int8_mode'})\n", + "\n", + "# random weights come from no named model\n", + "test_eq(problems(_RAND, model_id='org/resnet50-p30'), [\"model_id: 'org/resnet50-p30' given for random weights\"])\n", + "test_eq(problems(_LAT, model_id=None, model_revision=None), []) # trained weights need not name their model\n", + "\n", + "# a quality row names the engine it was measured through, unless it was measured on the development model\n", + "test_eq(problems(_QUAL, engine_hash=None), ['engine_hash: missing, required when measured through the deployed engine'])\n", + "test_eq(problems(_QUAL, engine_hash=None, measured_through='dev_model'), [])\n", + "\n", + "# ordered fields, among those recorded\n", + "test_eq(names(problems(_QUAL, accuracy=80.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(names(problems(_QUAL, ci_low=79.0)), {'ci_low, accuracy, ci_high'})\n", + "test_eq(problems(_QUAL, ci_low=78.9, ci_high=78.9), [])\n", + "test_eq(names(problems(_LAT, latency_ms_p90=7.9)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(names(problems(_LAT, latency_ms_p90=..., latency_ms_p99=7.0)), {'latency_ms_p50, latency_ms_p90, latency_ms_p99'})\n", + "test_eq(problems(_LAT, latency_ms_p90=8.0, latency_ms_p99=8.0), [])\n", + "test_eq(problems(_LAT, latency_ms_p90=None), [])\n", + "test_eq(names(problems(_LAT, reformat_layers=62)), {'reformat_layers, engine_layers'})\n", + "\n", + "# digests are 64 lowercase hex characters\n", + "for _r, _f in [(_LAT, 'engine_hash'), (_LAT, 'graph_hash'), (_QUAL, 'engine_hash')]:\n", + " for _bad in ('ab' * 31, 'AB' * 32, 'sha256:' + 'ab' * 32): test_eq(names(problems(_r, **{_f: _bad})), {_f})\n", + "\n", + "# the date is ISO-8601 in UTC, and a real date\n", + "test_eq(problems(_LAT, measured_at='2026-09-28T10:00:00.123+00:00'), [])\n", + "test_eq(names(problems(_LAT, measured_at='2026-09-28')), {'measured_at'})\n", + "test_eq(names(problems(_QUAL, measured_at='2026-09-28T10:00:00+02:00')), {'measured_at'})\n", + "test_eq(names(problems(_LAT, measured_at='2026-13-28T10:00:00Z')), {'measured_at'})\n", + "\n", + "# recipe keys are free, but the recipe is stored as JSON\n", + "test_eq(problems(_LAT, recipe={}), [])\n", + "test_eq(names(problems(_LAT, recipe={'x': object()})), {'recipe'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "status", + "metadata": {}, + "outputs": [], + "source": [ + "for _r in (_LAT, _RAND, _QUAL): test_eq(measurement.status(_r), ('VALID', []))\n", + "\n", + "\n", + "def reasons(row, **change):\n", + " \"measurement.status() reasons for `row` with some fields changed\"\n", + " v, rs = measurement.status(edit(row, **change))\n", + " test_eq(v, 'INVALID')\n", + " return rs\n", + "\n", + "\n", + "# latency: the device, not the weights\n", + "test_eq(reasons(_LAT, latency_source='estimated'), ['latency_source: latency not measured on the device'])\n", + "test_eq(reasons(_LAT, throttled=True), ['throttled: the device throttled during the session'])\n", + "test_eq(reasons(_LAT, throttled=None), ['throttled: not recorded'])\n", + "test_eq(reasons(_RAND, throttled=...), ['throttled: not recorded'])\n", + "test_eq(reasons(_LAT, latency_spread_pct=5.1), ['latency_spread_pct: 5.1 above 5.0'])\n", + "test_eq(reasons(_LAT, latency_spread_pct=None), ['latency_spread_pct: not recorded'])\n", + "test_eq(measurement.status(edit(_LAT, latency_spread_pct=5.0)), ('VALID', []))\n", + "test_eq(measurement.status(edit(_LAT, latency_spread_pct=5.1), max_spread_pct=6.0), ('VALID', []))\n", + "# the informational conditions do not decide validity\n", + "test_eq(measurement.status(edit(_LAT, temp_start_c=None, temp_end_c=None, gpu_freq_mhz=None, clocks_locked=None,\n", + " warmup_s=None, layer_precisions=None)), ('VALID', []))\n", + "test_eq(names(reasons(_LAT, throttled=None, latency_spread_pct=None, latency_source='estimated')),\n", + " {'throttled', 'latency_spread_pct', 'latency_source'})\n", + "\n", + "# quality: through the engine, with parity\n", + "test_eq(reasons(_QUAL, measured_through='dev_model'), ['measured_through: accuracy not measured through the deployed engine'])\n", + "test_eq(reasons(_QUAL, argmax_parity=98.9), ['argmax_parity: 98.9 below parity_floor 99.0'])\n", + "test_eq(reasons(_QUAL, argmax_parity=None), ['argmax_parity: not recorded'])\n", + "test_eq(measurement.status(edit(_QUAL, argmax_parity=99.0)), ('VALID', [])) # at the floor holds\n", + "test_eq(names(reasons(_QUAL, measured_through='dev_model', engine_hash=None, argmax_parity=...)),\n", + " {'measured_through', 'argmax_parity'})\n", + "\n", + "# a malformed row is invalid for its structural problems\n", + "test_eq(reasons(_LAT, batch_size=True), ['batch_size: expected int, got True'])\n", + "test_eq(reasons(_LAT, kind=...), ['kind: missing'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "key-throughput", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(measurement.key(_LAT)[0], 'latency')\n", + "test_eq(measurement.key(_QUAL)[0], 'quality')\n", + "test_eq(measurement.key(_LAT), measurement.key(dict(_LAT)))\n", + "# a latency row is unique on its graph and how it ran; a software update on the device is a new key\n", + "for _f, _v in [('graph_hash', '12' * 32), ('weights', 'random'), ('precision', 'fp16'), ('int8_mode', 'implicit'),\n", + " ('batch_size', 1), ('device', 'raspberry-pi-5'), ('runtime', 'onnxruntime'),\n", + " ('runtime_version', '8.5.2'), ('platform_version', 'jetpack-5.1.3'),\n", + " ('measured_at', '2026-09-29T10:00:00Z')]:\n", + " test_ne(measurement.key({**_LAT, _f: _v}), measurement.key(_LAT))\n", + "# the weights' values are not: another engine of the same graph and weights kind, the same key\n", + "test_eq(measurement.key({**_LAT, 'engine_hash': '12' * 32, 'model_id': None, 'run_id': 'l9', 'latency_ms_p50': 9.0,\n", + " 'series': None, 'variant': 'renamed'}), measurement.key(_LAT))\n", + "# a quality row is unique on its model, engine, evaluation set, metric and date\n", + "for _f, _v in [('model_id', 'org/other'), ('model_revision', 'e4f5a6b'), ('engine_hash', '12' * 32),\n", + " ('eval_set_hash', 'sha256:10'), ('metric', 'top5'), ('measured_at', '2026-09-29T10:00:00Z')]:\n", + " test_ne(measurement.key({**_QUAL, _f: _v}), measurement.key(_QUAL))\n", + "test_eq(measurement.key({**_QUAL, 'accuracy': 70.0, 'run_id': 'q9', 'device': None}), measurement.key(_QUAL))\n", + "\n", + "# throughput is derived from a latency row: batch_size * 1000 / latency_ms_p50\n", + "test_eq(measurement.throughput(_LAT), 500.0)\n", + "test_close(measurement.throughput({**_LAT, 'batch_size': 1, 'latency_ms_p50': 2.1}), 476.19, eps=0.01)\n", + "with ExceptionExpected(ValueError, 'latency row'): measurement.throughput(_QUAL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "join", + "metadata": {}, + "outputs": [], + "source": [ + "_j = measurement.join([_LAT, _RAND], [_QUAL])\n", + "test_eq(len(_j), 1) # the random-weights engine has no accuracy to join, and needs none\n", + "_row = _j[0]\n", + "# identity kept per side, schema version once, the kind gone\n", + "test_eq((_row['latency_run_id'], _row['quality_run_id']), ('l1', 'q1'))\n", + "test_eq((_row['latency_measured_at'], _row['quality_measured_at']), (_LAT['measured_at'], _QUAL['measured_at']))\n", + "test_eq(_row['schema_version'], 1)\n", + "test_eq('kind' in _row or 'run_id' in _row, False)\n", + "# every field of both sides, nothing overwritten\n", + "_both = [f for f in measurement.FIELDS['latency'] if f in measurement.FIELDS['quality']]\n", + "_own = [f for f in _both if f not in ('schema_version', 'kind', 'engine_hash', 'precision', 'device', 'model_id',\n", + " 'model_revision')]\n", + "test_eq(len(_row), len(measurement.FIELDS['latency']) + len(measurement.FIELDS['quality']) - len(_both) + len(_own) - 1)\n", + "test_eq((_row['latency_ms_p50'], _row['accuracy'], _row['engine_hash']), (8.0, 78.9, _ENGINE))\n", + "test_eq(measurement.throughput({**_row, 'kind': 'latency'}), 500.0)\n", + "\n", + "# only VALID rows of the right kind are paired, so the file read back can be given as both arguments\n", + "test_eq(len(measurement.join([_LAT, _RAND, _QUAL], [_LAT, _RAND, _QUAL])), 1)\n", + "test_eq(measurement.join([{**_LAT, 'throttled': True}], [_QUAL]), [])\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'argmax_parity': 90.0}]), [])\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'engine_hash': '12' * 32}]), [])\n", + "# one latency row, two quality rows of the same engine: two pairs\n", + "test_eq(len(measurement.join([_LAT], [_QUAL, {**_QUAL, 'run_id': 'q2', 'eval_set_hash': 'sha256:10'}])), 2)\n", + "\n", + "# a field only one side recorded is taken from it; two sides that disagree on one engine raise\n", + "test_eq(measurement.join([{**_LAT, 'model_id': None, 'model_revision': None}], [_QUAL])[0]['model_id'], 'org/resnet50-p30')\n", + "test_eq(measurement.join([_LAT], [{**_QUAL, 'device': None}])[0]['device'], 'jetson-nano-4gb')\n", + "with ExceptionExpected(ValueError, 'precision'): measurement.join([_LAT], [{**_QUAL, 'precision': 'fp16'}])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "graph-hash", + "metadata": {}, + "outputs": [], + "source": [ + "import sys, torch\n", + "\n", + "# without onnx installed, a one-line error names the extra\n", + "_saved = sys.modules.get('onnx')\n", + "sys.modules['onnx'] = None\n", + "try:\n", + " with ExceptionExpected(ImportError, r\"fastermodels\\[onnx\\]\"): measurement.graph_hash('x.onnx')\n", + "finally:\n", + " if _saved is None: sys.modules.pop('onnx', None)\n", + " else: sys.modules['onnx'] = _saved\n", + "\n", + "import onnx\n", + "\n", + "\n", + "def export(width, seed, path):\n", + " \"A small conv net with `width` channels and weights drawn from `seed`, exported to ONNX\"\n", + " torch.manual_seed(seed)\n", + " net = torch.nn.Sequential(torch.nn.Conv2d(3, width, 3), torch.nn.ReLU(), torch.nn.Flatten(),\n", + " torch.nn.Linear(width * 6 * 6, 10)).eval()\n", + " torch.onnx.export(net, torch.randn(1, 3, 8, 8), str(path), dynamo=False, opset_version=17,\n", + " input_names=['input'], output_names=['output'])\n", + " return path\n", + "\n", + "\n", + "with tempfile.TemporaryDirectory() as _d:\n", + " _a, _b, _c = (export(w, s, Path(_d, f'{n}.onnx')) for n, w, s in (('a', 8, 0), ('b', 8, 1), ('c', 16, 0)))\n", + " test_ne(Path(_a).read_bytes(), Path(_b).read_bytes()) # the files differ: the weights differ\n", + " _h = measurement.graph_hash(_a)\n", + " test_eq(bool(re.fullmatch(r'[0-9a-f]{64}', _h)), True)\n", + " test_eq(measurement.graph_hash(str(_a)), _h) # deterministic, str or Path\n", + " test_eq(measurement.graph_hash(_b), _h) # same shape, other weights: same graph\n", + " test_ne(measurement.graph_hash(_c), _h) # another width: another graph\n", + " # the file is read, not modified\n", + " _before = Path(_a).read_bytes(); measurement.graph_hash(_a); test_eq(Path(_a).read_bytes(), _before)\n", + "\n", + " # initializer names are part of the graph\n", + " _m = onnx.load(str(_a))\n", + " _old = _m.graph.initializer[0].name\n", + " _m.graph.initializer[0].name = 'renamed'\n", + " for _n in _m.graph.node: _n.input[:] = ['renamed' if i == _old else i for i in _n.input]\n", + " onnx.save(_m, str(Path(_d, 'r.onnx')))\n", + " test_ne(measurement.graph_hash(Path(_d, 'r.onnx')), _h)\n", + "\n", + " # so is the element type: the same graph in fp16 is another graph\n", + " _m = onnx.load(str(_a))\n", + " _m.graph.initializer[0].data_type = onnx.TensorProto.FLOAT16\n", + " onnx.save(_m, str(Path(_d, 'h.onnx')))\n", + " test_ne(measurement.graph_hash(Path(_d, 'h.onnx')), _h)\n", + "\n", + " # weights stored outside the file are not read, and the hash does not depend on where they are stored\n", + " _m = onnx.load(str(_b))\n", + " onnx.save(_m, str(Path(_d, 'ext.onnx')), save_as_external_data=True, all_tensors_to_one_file=True,\n", + " location='ext.data', size_threshold=0)\n", + " test_eq(measurement.graph_hash(Path(_d, 'ext.onnx')), _h)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "io", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as _d:\n", + " _p = Path(_d, 'rows.jsonl')\n", + "\n", + " # rows of both kinds are appended to one file, and read back equal\n", + " for _r in (_LAT, _RAND, _QUAL): measurement.write_row(_p, _r)\n", + " test_eq(measurement.read_rows(_p), [_LAT, _RAND, _QUAL])\n", + " test_eq(len(_p.read_text().splitlines()), 3)\n", + "\n", + " # an invalid row is refused, naming its problems, and nothing is written\n", + " with ExceptionExpected(ValueError, 'batch_size'): measurement.write_row(_p, {**_LAT, 'batch_size': 0, 'run_id': 'l3'})\n", + " with ExceptionExpected(ValueError, 'kind: missing'): measurement.write_row(_p, edit(_LAT, kind=...))\n", + " # a duplicate key is refused, even with other fields changed\n", + " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_LAT, 'run_id': 'again', 'latency_ms_p50': 7.0})\n", + " with ExceptionExpected(ValueError, 'already in'): measurement.write_row(_p, {**_QUAL, 'run_id': 'again', 'accuracy': 79.0})\n", + " test_eq(measurement.read_rows(_p), [_LAT, _RAND, _QUAL])\n", + "\n", + " # uniqueness is per kind: a quality row never collides with a latency row\n", + " _r4 = {**_LAT, 'run_id': 'l4', 'batch_size': 1}\n", + " measurement.write_row(_p, _r4)\n", + "\n", + " # numpy scalars are written as plain JSON numbers\n", + " _np = {**_LAT, 'run_id': 'l5', 'batch_size': np.int64(2), 'latency_ms_p50': np.float64(4.0),\n", + " 'recipe': {'prune': np.float64(0.3), 'calib_n': np.int64(512)}, 'layer_precisions': {'int8': np.int64(61)}}\n", + " measurement.write_row(_p, _np)\n", + " test_eq(json.loads(_p.read_text().splitlines()[-1])['batch_size'], 2)\n", + " test_eq(measurement.read_rows(_p)[-1]['recipe'], {'prune': 0.3, 'calib_n': 512})\n", + "\n", + " # a row written by hand without a final newline does not swallow the next one\n", + " _p.write_text(_p.read_text().rstrip('\\n'))\n", + " _r6 = {**_QUAL, 'run_id': 'q6', 'metric': 'top5'}\n", + " measurement.write_row(_p, _r6)\n", + " test_eq([r['run_id'] for r in measurement.read_rows(_p)], ['l1', 'l2', 'q1', 'l4', 'l5', 'q6'])\n", + "\n", + " # a malformed line is reported with its line number; blank lines are skipped\n", + " _bad = Path(_d, 'bad.jsonl')\n", + " _bad.write_text(json.dumps(_LAT) + '\\n\\n{\"run_id\": \\n')\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.read_rows(_bad)\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:3'): measurement.write_row(_bad, _QUAL)\n", + " _bad.write_text('[1, 2]\\n')\n", + " with ExceptionExpected(ValueError, 'bad.jsonl:1: not a JSON object'): measurement.read_rows(_bad)\n", + "\n", + " # reading a file that is not there is an error, not an empty table\n", + " with ExceptionExpected(FileNotFoundError): measurement.read_rows(Path(_d, 'none.jsonl'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "parquet", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "\n", + "# without the extra installed, a one-line error names it\n", + "_saved = sys.modules.get('pyarrow')\n", + "sys.modules['pyarrow'] = None\n", + "try:\n", + " with ExceptionExpected(ImportError, r\"fastermodels\\[parquet\\]\"): measurement.to_parquet([_LAT], 'x.parquet')\n", + "finally:\n", + " if _saved is None: sys.modules.pop('pyarrow', None)\n", + " else: sys.modules['pyarrow'] = _saved\n", + "\n", + "\n", + "def stored(rows, kind):\n", + " \"What a Parquet file of `rows` must hold: every field of the kind, None where absent\"\n", + " return [{k: r.get(k) for k in measurement.FIELDS[kind]} for r in rows]\n", + "\n", + "\n", + "def read_back(path):\n", + " \"The rows of a Parquet file, dict fields decoded from their JSON\"\n", + " return [{k: json.loads(v) if k in ('recipe', 'layer_precisions') and v is not None else v for k, v in r.items()}\n", + " for r in pq.read_table(path).to_pylist()]\n", + "\n", + "\n", + "if importlib.util.find_spec('pyarrow'):\n", + " import pyarrow as pa\n", + " import pyarrow.parquet as pq\n", + " with tempfile.TemporaryDirectory() as _d:\n", + " _pq = Path(_d, 'latency.parquet')\n", + " _sparse = {k: v for k, v in _LAT.items() if k not in ('series', 'latency_ms_p99', 'layer_precisions')}\n", + " _rows = [_LAT, _RAND, {**_sparse, 'run_id': 'l3', 'batch_size': 1, 'latency_ms_p50': 8},\n", + " {**_LAT, 'run_id': 'l4', 'batch_size': np.int64(2), 'latency_ms_p50': np.float32(4.5)}]\n", + " measurement.to_parquet(_rows, _pq)\n", + " # every row reads back as written, a missing value as None, an int where a float goes as a float\n", + " test_eq(read_back(_pq), stored(_rows, 'latency'))\n", + " test_eq([r['model_id'] for r in read_back(_pq)], ['org/resnet50-p30', None, 'org/resnet50-p30', 'org/resnet50-p30'])\n", + " test_eq([type(r['latency_ms_p50']) for r in read_back(_pq)], [float] * 4)\n", + " # dict fields are canonical JSON: key order is not a difference\n", + " test_eq(pq.read_table(_pq).column('recipe')[0].as_py(), '{\"calib_n\":512,\"prune\":0.3,\"quant\":\"int8\"}')\n", + "\n", + " # the schema is explicit, derived from FIELDS: the file does not depend on the reader\n", + " _schema = pq.read_schema(_pq)\n", + " test_eq(_schema.names, list(measurement.FIELDS['latency']))\n", + " for _f, _t in [('latency_ms_p50', pa.float64()), ('batch_size', pa.int64()), ('throttled', pa.bool_()),\n", + " ('model_id', pa.string()), ('recipe', pa.string()), ('layer_precisions', pa.string())]:\n", + " test_eq(_schema.field(_f).type, _t)\n", + " test_eq(_schema.field('model_id').nullable, True)\n", + " test_eq(_schema.field('throttled').nullable, True)\n", + " test_eq(_schema.field('batch_size').nullable, False)\n", + " # a column nobody recorded is still typed, not a null column\n", + " measurement.to_parquet([_RAND], Path(_d, 'rand.parquet'))\n", + " test_eq(pq.read_schema(Path(_d, 'rand.parquet')).field('model_id').type, pa.string())\n", + "\n", + " _q = [_QUAL, {**_QUAL, 'run_id': 'q2', 'engine_hash': None, 'measured_through': 'dev_model', 'device': None}]\n", + " measurement.to_parquet(_q, Path(_d, 'quality.parquet'))\n", + " test_eq(read_back(Path(_d, 'quality.parquet')), stored(_q, 'quality'))\n", + " test_eq(pq.read_schema(Path(_d, 'quality.parquet')).names, list(measurement.FIELDS['quality']))\n", + "\n", + " # one kind per file; no rows, no file; a malformed row is refused with its index; nothing is written\n", + " with ExceptionExpected(ValueError, 'one kind per file'): measurement.to_parquet([_LAT, _QUAL], Path(_d, 'no.parquet'))\n", + " with ExceptionExpected(ValueError, 'no rows'): measurement.to_parquet([], Path(_d, 'no.parquet'))\n", + " with ExceptionExpected(ValueError, 'row 1 refused'): measurement.to_parquet([_LAT, {**_LAT, 'batch_size': 0}], Path(_d, 'no.parquet'))\n", + " test_eq(Path(_d, 'no.parquet').exists(), False)\n", + "else: print('pyarrow not installed: to_parquet round trip skipped')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 1c37268..38f30d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,9 @@ dependencies = ['torch', 'huggingface_hub>=0.25', 'safetensors', 'numpy'] [project.optional-dependencies] test = ['torchvision', 'torch-pruning', 'onnx', 'fastcore'] -dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore'] +dev = ['nbdev', 'torchvision', 'torch-pruning', 'onnx', 'fastcore', 'pyarrow'] +parquet = ['pyarrow'] +onnx = ['onnx'] [project.urls] Repository = "https://github.com/FasterAI-Labs/fastermodels"