From 5d66020269a0a71d8232d45af1952117169126c4 Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Tue, 27 Oct 2026 17:11:00 -0500 Subject: [PATCH 1/6] feat: add Brev Nano launchable with native local scoring Signed-off-by: Jon Patterson --- .../cosmos3/reasoner/vlmevalkit/.dockerignore | 21 + .../vlmevalkit/EXTERNAL_CONFIGURATION.md | 29 + .../cosmos3/reasoner/vlmevalkit/README.md | 16 +- .../reasoner/vlmevalkit/launchable/.gitignore | 3 + .../reasoner/vlmevalkit/launchable/Dockerfile | 21 + .../launchable/Evaluate_Cosmos3.ipynb | 200 ++++++ .../reasoner/vlmevalkit/launchable/README.md | 142 ++++ .../vlmevalkit/launchable/compose.yaml | 85 +++ .../vlmevalkit/launchable/jupyter_config.py | 21 + .../vlmevalkit/launchable/launchable.py | 287 ++++++++ .../reasoner/vlmevalkit/launchable/setup.sh | 91 +++ .../vlmevalkit/launchable/test_launchable.py | 58 ++ .../launchable/test_local_scoring.py | 388 +++++++++++ .../dataset/metropolis2d/astro_2d_dataset.py | 30 + .../vlmeval/dataset/metropolis_dvc.py | 69 +- .../dataset/metropolis_event_verification.py | 10 + .../vlmeval/dataset/metropolis_temporal.py | 16 +- .../vlmeval/dataset/metropolis_vqa.py | 16 +- .../vlmevalkit/vlmeval/local_scoring.py | 525 +++++++++++++++ .../vlmevalkit/vlmeval/scoring_assets.py | 630 ++++++++++++++++++ 20 files changed, 2626 insertions(+), 32 deletions(-) create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore new file mode 100644 index 000000000..0013b61a8 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore @@ -0,0 +1,21 @@ +.git +.venv +__pycache__ +*.pyc +*.egg-info +**/__pycache__ +**/.ipynb_checkpoints +launchable/state +**/.env* +**/.aws +**/.ssh +**/*credentials* +**/*.pem +**/*.key +**/scoring-assets.json +**/scoring-assets +**/results +**/env.sh +cosmos_eval_out +out +outputs diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md index 24023fdb7..138263b8d 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md @@ -40,3 +40,32 @@ constructor or dataset configuration. Original private dataset YAML defaults remain available. Missing or nonexistent grounding annotation files produce a clear `ValueError`. Omni3D retains its existing private configuration and can be pointed at another authorized dataset through `raw_image_path` and `json_path`. + +## Brev Nano and fully local VANTAGE scoring + +The [Brev launchable](launchable/README.md) uses this Framework-native kit, the +pinned Nano revision, and a caller-supplied, labeled `scoring-assets.json` bundle. +Prepare authorized sources offline with `python -m vlmeval.scoring_assets`; that +step writes canonical tables and a fingerprinted manifest, not original labels +or media. Keep the bundle and runtime state outside every source checkout. +During evaluation Compose mounts the completed bundle read-only; local scoring +does not use an API judge, dataset downloader, credential mount, or external +submission workflow. + +For `MetropolisVQA`, `MetropolisTemporal`, `MetropolisEventVerification`, and +`MetropolisDVC`, provide `local_data_file` and `local_video_root` together to +bypass cache and remote-source discovery. The local table must include genuine +ground truth and bundle-relative media identities; missing labels or media fail. +`Astro2DBenchDataset` accepts a `data_roots` mapping covering all registered child +datasets. Existing explicit local grounding, pointing, and SOT constructor paths +are reused. Local DVC scoring requires a fingerprinted, revisioned local +roberta-large checkpoint and never consults the remote BERTScore endpoint. + +Use `python -m vlmeval.local_scoring` for existing native predictions without a +model endpoint or GPU. Prediction identity columns must match trusted rows; +prediction-supplied answers and paths cannot replace ground truth. Subsets +require an explicit opt-in and are reported as subsets. There is no aggregate +cross-task score or claim of official parity. Dataset distribution approval and +a named reference result remain release gates. Original labels and native +filtering are preserved; annotation anomalies are recorded in private local +diagnostics rather than repaired or silently omitted. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md index a01f18d98..a6bd89ffa 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md @@ -5,14 +5,20 @@ SPDX-License-Identifier: Apache-2.0 # TAR and VANTAGE Benchmarks for Cosmos3 Model -This repository is an **NVIDIA-customized [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)** that -releases the NVIDIA Metropolis "smart infra" benchmarks and makes the cosmos-reasoner scores on them -**publicly reproducible**. The reproduction kit lives in [`cosmos_eval/`](cosmos_eval/); the stock -VLMEvalKit inference + scoring engine it drives is the rest of the tree (upstream toolkit docs: -[`README-vlmevalkit.md`](README-vlmevalkit.md)). +This directory is an **NVIDIA-customized [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)** +maintained in Cosmos Framework. It contains reference settings for NVIDIA +Metropolis "smart infra" evaluation; [`cosmos_eval/`](cosmos_eval/) composes the +native inference and scoring engine. Toolkit docs live in +[`README-vlmevalkit.md`](README-vlmevalkit.md). These settings do not make private +datasets publicly reproducible or establish official score parity. For user-hosted endpoints and local dataset inputs, see [Deployment and dataset configuration](EXTERNAL_CONFIGURATION.md). +For a Nano-only Brev notebook and fully local VANTAGE scoring, use the +[Brev launchable](launchable/README.md). It requires an authorized labeled bundle +prepared outside the checkout and mounted read-only; it has no external scoring +submission workflow. Data redistribution approval and a named reference result +remain release gates. See that guide for validation status and limits. The kit ships, for each benchmark, reference `run.py --config` settings — split into a shared dataset layer (`cosmos_eval/data/`) and a per-family model layer diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore new file mode 100644 index 000000000..dd26ad7de --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore @@ -0,0 +1,3 @@ +state/ +.env +.ipynb_checkpoints/ diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile new file mode 100644 index 000000000..a9729aef7 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +FROM python:3.11.14-slim-bookworm@sha256:65a93d69fa75478d554f4ad27c85c1e69fa184956261b4301ebaf6dbb0a3543d +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ffmpeg git libgl1 libglib2.0-0 libcairo2 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /opt/vlmevalkit +COPY requirements.txt setup.py setup.cfg README-vlmevalkit.md ./ +# Evaluation is API-based; keep its PyTorch stack off the serving GPU. +RUN pip install --no-cache-dir torch==2.8.0 torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu \ + && pip install --no-cache-dir -r requirements.txt einops accelerate 'setuptools<81' evaluate jupyterlab==4.4.10 +COPY . . +RUN pip install --no-cache-dir --no-deps -e . \ + && python -c 'import vlmeval' \ + && pip freeze > /opt/evaluation-packages.txt +ARG COSMOS_COMMIT=unknown +ENV COSMOS_COMMIT=${COSMOS_COMMIT} PYTHONUNBUFFERED=1 HF_HOME=/cache/huggingface \ + AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null \ + AWS_EC2_METADATA_DISABLED=true SCORING_ASSETS_DIR=/scoring-assets LMUData=/cache/lmudata +WORKDIR /opt/vlmevalkit/launchable +CMD ["python", "jupyter_config.py"] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb new file mode 100644 index 000000000..533ed9eb4 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Evaluate Cosmos3-Nano locally\n", + "\n", + "**Validate labeled assets → run five VQA predictions → score locally.** Full eight-task inference is opt-in. Existing predictions can be scored without a model endpoint or GPU.\n", + "\n", + "This notebook uses Cosmos Framework's native VLMEvalKit datasets and metrics. It does not upload outputs or prepare an external scoring submission. A labeled, fingerprinted scoring bundle is required. Dataset access does **not** grant permission to redistribute labels or media. Public release remains gated on approved data licensing/distribution and a named reference result.\n", + "\n", + "The serving configuration targets the reasoner in `nvidia/Cosmos3-Nano` at `e59a53c25979a090fa8706c9acc0c254a6e89b92`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1 · Configure\n", + "\n", + "`SCORING_ASSETS_DIR` is mounted read-only by Compose. Prepare the bundle outside every source checkout before setup; it must contain `scoring-assets.json`, canonical tables, original labels, and media. DVC also requires its pinned local roberta-large metric model. `LAUNCHABLE_STATE_DIR` must likewise be outside the checkout and separate from the bundle. See the README for preparation. No AWS credentials are mounted.\n", + "\n", + "For existing predictions, set `SCORE_ONLY = True`, `PREDICTIONS_FILE` to a native TSV/XLSX/JSON table, and `TASK` appropriately. Native identity columns must be retained. Set `ALLOW_SUBSET = True` only when intentionally scoring a partial dataset; coverage is recorded explicitly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import subprocess\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from IPython.display import FileLink, display\n", + "\n", + "SCORE_ONLY = os.environ.get('EVAL_SCORE_ONLY') == '1'\n", + "PREDICTIONS_FILE = None\n", + "TASK = 'VANTAGE_VQA'\n", + "ALLOW_SUBSET = False\n", + "RUN_FULL = False\n", + "ASSETS = Path(os.environ.get('SCORING_ASSETS_DIR', '/scoring-assets'))\n", + "STAMP = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')\n", + "SMOKE_WORK = Path('/results') / f'local-vqa-smoke-{STAMP}'\n", + "FULL_WORK = Path('/results') / f'local-full-{STAMP}'\n", + "SCORE_DIR = Path('/results') / f'local-score-only-{STAMP}'\n", + "CLI = [sys.executable, str(Path('launchable.py').resolve())]\n", + "\n", + "def invoke(command):\n", + " result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n", + " print(result.stdout, end='')\n", + " result.check_returncode()\n", + "\n", + "if SCORE_ONLY and not PREDICTIONS_FILE:\n", + " raise ValueError('Set PREDICTIONS_FILE to an existing native prediction table before score-only execution.')\n", + "print('Existing-prediction scoring' if SCORE_ONLY else 'Five-question inference and local scoring')\n", + "print('Assets:', ASSETS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Validate local labels and media\n", + "\n", + "Missing labels, altered annotation fingerprints, or unavailable media fail before contacting the model. Public inference-only inputs cannot substitute for ground truth." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "selected = TASK if SCORE_ONLY else ('all' if RUN_FULL else 'VANTAGE_VQA')\n", + "invoke(CLI + ['prepare', '--assets-dir', str(ASSETS), '--benchmarks', selected])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3 · Connect only when generating predictions\n", + "\n", + "For BYOM, set the complete `COSMOS_API_BASE` chat-completions URL and `COSMOS_MODEL` in the evaluator environment before running the notebook. Keep any API key in the environment, not notebook source. Score-only mode skips this step entirely." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not SCORE_ONLY:\n", + " invoke(CLI + ['check', '--wait', '1800'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4 · Generate and score, or score existing predictions\n", + "\n", + "The default smoke runs five deterministically selected VQA examples and then the native local evaluator. It is a **subset**, not a full benchmark or published-score reproduction. Score-only invokes the native scoring CLI directly and never constructs an inference model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if SCORE_ONLY:\n", + " command = [sys.executable, '-m', 'vlmeval.local_scoring', '--task', TASK,\n", + " '--predictions', str(PREDICTIONS_FILE), '--assets-dir', str(ASSETS),\n", + " '--output-dir', str(SCORE_DIR)]\n", + " if ALLOW_SUBSET:\n", + " command.append('--allow-subset')\n", + " invoke(command)\n", + " SUMMARY = SCORE_DIR / 'summary.json'\n", + "else:\n", + " invoke(CLI + ['run', '--assets-dir', str(ASSETS), '--benchmarks', 'VANTAGE_VQA',\n", + " '--mode', 'all', '--smoke', '--work-dir', str(SMOKE_WORK)])\n", + " SUMMARY = SMOKE_WORK / 'local-scores' / 'bench_00' / 'summary.json'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5 · Inspect the local score and provenance\n", + "\n", + "Each summary records the task's native metric, explicit units, aggregation rule, row coverage, asset and prediction hashes, and scorer-source hashes. DVC includes metric-model fingerprints. There is no invented cross-task overall score or claim of official leaderboard parity. All result files are private and can contain answer-bearing diagnostics; review before sharing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary = json.loads(SUMMARY.read_text())\n", + "print(json.dumps({key: summary[key] for key in ('task', 'status', 'coverage', 'primary_metric')}, indent=2))\n", + "display(FileLink(str(SUMMARY), url_prefix='/files'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6 · Optional full eight-task evaluation\n", + "\n", + "Only after every requested task validates, set `RUN_FULL = True` in configuration and rerun from validation. This generates all eight tasks and scores them locally in a fresh directory. It can take substantially longer than the smoke. The labeled bundle must cover every task, including the real local DVC metric model. Astro preserves original labels and native filtering and denominators, with annotation diagnostics recorded privately. A full run does not by itself establish official parity: native per-task results require an agreed release protocol and named reference result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if RUN_FULL and not SCORE_ONLY:\n", + " invoke(CLI + ['run', '--assets-dir', str(ASSETS), '--benchmarks', 'all',\n", + " '--mode', 'all', '--work-dir', str(FULL_WORK)])\n", + " invoke(CLI + ['results', '--work-dir', str(FULL_WORK)])\n", + " display(FileLink(str(FULL_WORK / 'run.json'), url_prefix='/files'))\n", + "else:\n", + " print('Full eight-task inference disabled.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stop and retain results\n", + "\n", + "From the launchable directory, `docker compose --profile local stop cosmos-endpoint` stops serving without deleting state. To restart an approved inference run, use `bash setup.sh` (or `bash setup.sh --brev` on Brev). `bash setup.sh --score-only` starts only the notebook for existing-prediction scoring. Stopping a container does not stop billing for the underlying Brev instance; stop that instance separately when finished.\n", + "\n", + "Clear notebook outputs before sharing. Do not upload restricted labels, media, credentials, or prediction artifacts to the PR." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md new file mode 100644 index 000000000..91fdba159 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -0,0 +1,142 @@ +# Cosmos3 VANTAGE evaluation on Brev + +Jupyter and CLI orchestration for native VLMEvalKit inference **and fully local scoring**. Use a labeled, fingerprinted local asset bundle; score existing predictions without a model endpoint or GPU. There is no external scoring submission, upload, or separate public harness. + +**Release gates:** Dataset access does not grant permission to redistribute labels or media. Public inference-only inputs are insufficient for local scoring. Do not advertise published-score reproducibility until the labeled bundle is approved for distribution and a named reference result and release protocol are agreed. No private data, credentials, or benchmark outputs belong in this repository or PR. + +This work targets the [16B BF16 Cosmos3-Nano checkpoint](https://huggingface.co/nvidia/Cosmos3-Nano/blob/e59a53c25979a090fa8706c9acc0c254a6e89b92/README.md), `nvidia/Cosmos3-Nano` at `e59a53c25979a090fa8706c9acc0c254a6e89b92`, and evaluates its reasoner. Other models and deployment tracks are outside this launchable's scope. + +## Supply the scoring bundle + +Set absolute **host** paths outside every source checkout before setup: + +```bash +export SCORING_ASSETS_DIR=/absolute/path/to/authorized-scoring-bundle +export LAUNCHABLE_STATE_DIR=/absolute/path/to/private-runtime-state +``` + +The bundle contains `scoring-assets.json`, original labels, canonical tables, and media. DVC additionally requires the pinned local roberta-large checkpoint. Compose mounts the bundle read-only at `/scoring-assets`; frame caches use `/cache/lmudata`, and results use `/results`. Do not mount AWS profiles, home directories, or credential files. No shared NVIDIA profile, NGC key, dataset-service credential, or API judge is required at evaluation time. + +Preparation requires a writable authorized source directory; execution requires the completed bundle mounted **read-only**, never a source checkout, credential store, or writable cache/results directory. Setup requires both exported paths and rejects checkout-local paths. Keep these exports in the host shell for later Compose lifecycle commands. + +Each requested task must be complete. Missing labels/media or modified fingerprints fail before inference. A VQA-only bundle is sufficient for the five-question smoke; full evaluation requires all eight tasks. Internal access and release approval are separate requirements. + +### Maintainers: prepare authorized sources + +Preparation is an offline conversion, not a dataset downloader. Keep the original filtered release layout, including `VQA`, `Temporal`, `Dense Video Caption`, `Spatial`, `2dbbox`, `datasets/Metropolis2DPointing`, `event_verification_subset`, and `tracking`. It does not copy media or rewrite original labels. Never put the source tree inside the code checkout. + +From the maintained Framework kit root (`.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit`), build the CPU evaluator: + +```bash +docker build --platform linux/amd64 -t cosmos3-vantage-eval:local -f launchable/Dockerfile . +``` + +For DVC, stage the genuine public metric model first. This download needs network access but no model-serving endpoint. It does not download benchmark data: + +```bash +docker run --rm --platform linux/amd64 \ + -v "$SCORING_ASSETS_DIR:/scoring-assets" --entrypoint python cosmos3-vantage-eval:local \ + -c 'from huggingface_hub import snapshot_download; snapshot_download("FacebookAI/roberta-large", revision="722cf37b1afa9454edce342e7895e588b6ff1d59", local_dir="/scoring-assets/metric-models/roberta-large", allow_patterns=["config.json","merges.txt","model.safetensors","tokenizer.json","tokenizer_config.json","vocab.json"])' + +docker run --rm --platform linux/amd64 --network none \ + -v "$SCORING_ASSETS_DIR:/scoring-assets" --entrypoint python cosmos3-vantage-eval:local \ + -m vlmeval.scoring_assets --assets-dir /scoring-assets --asset-id YOUR_IMMUTABLE_DATASET_VERSION \ + --bert-score-model metric-models/roberta-large \ + --bert-score-revision 722cf37b1afa9454edce342e7895e588b6ff1d59 +``` + +For VQA only, omit the metric model and pass `--tasks VQA`. Preparation validates original ground truth before native conversion, including complete tracking-frame labels. Explicit VQA labels are trimmed/uppercased, never defaulted to A; binary and four-choice source questions are supported. Invalid/missing labels fail rather than manufacturing answers. Scorer-source fingerprints record this normalization; compare results only under an agreed reference protocol. + +Do not regenerate or modify the manifest between inference and scoring. The controller records its hash and rejects a different snapshot. DVC model files are fingerprinted separately and checked before use. + +## Start the launchable + +Use Linux x86-64, Docker Engine/Compose v2, and NVIDIA Container Toolkit for local serving. The source repository is [NVIDIA/cosmos-framework](https://github.com/NVIDIA/cosmos-framework), not `NVIDIA/cosmos`. Select the approved branch/commit containing this launchable and record its exact Framework commit; upstream `main` must not be assumed to include an unpublished branch. Brev's managed Jupyter must be disabled because Compose provides it. + +From the Framework checkout root, with the authorized bundle prepared and both host paths exported: + +```bash +bash .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh --brev +``` + +On a non-Brev host, omit `--brev`. For **existing-prediction scoring only**, add `--score-only`; this starts Jupyter without starting the GPU endpoint. Normal setup builds the evaluator and validates VQA assets before starting serving. + +Brev access must use its authenticated/secure port-8888 link. `--brev` discovers the private `wt0` mesh IPv4 address and fails without it; non-Brev defaults to loopback. An explicit private `JUPYTER_BIND_ADDRESS` takes precedence. There is no public/wildcard fallback. Setup persists only its notebook token and non-secret bind address in permission-restricted `.env`. Retrieve the login token on the host with `docker compose exec vantage-eval jupyter server list`; never share it or capture it in screenshots. + +Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb`. Jupyter's authenticated file root is `/`, allowing links to `/results`. The evaluator has no Docker socket. The model API has no host port. Container lifecycle commands belong in the host terminal. + +Serving uses `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` (override with `VLLM_IMAGE`). Allow space for model weights, images, the labeled bundle, frame caches, and results. + +### Validation status and limits + +- The runtime code passed 20 native-scoring/artifact regressions in the packaged evaluator on native Linux x86-64. The committed fixtures are synthetic and use temporary directories; no benchmark labels, media, or executed notebook outputs are included. +- A private five-question Brev notebook smoke exercised Nano inference and native local scoring. Saved predictions were also rescored in a CPU evaluator with networking disabled. This demonstrates the subset workflow, not a full benchmark or published-score reproduction. +- DVC's genuine pinned local metric model was exercised offline with controlled captions, not model-generated benchmark answers. No DVC model-quality claim is made. +- Preparation preserves original labels, native filtering, and ground-truth denominators. Annotation anomalies are recorded in private local diagnostics rather than repaired or silently omitted. +- Full eight-task model evaluation, immutable one-click cold launch, end-to-end BYOM, and named reference-score reproduction remain unverified. Scoring assets, detailed operational evidence, and benchmark results must be supplied and retained separately. + +## Run and score + +The notebook defaults to a deterministic five-question VQA inference-and-scoring smoke. `RUN_FULL = False` prevents Run All from launching full eight-task inference. Score-only mode requires an existing native prediction table and explicitly selected task; it skips endpoint checks and model construction. + +CLI equivalents inside the evaluator: + +```bash +python launchable.py prepare --benchmarks VANTAGE_VQA +python launchable.py run --benchmarks VANTAGE_VQA --smoke --mode all --work-dir /results/smoke-unique +python launchable.py results --work-dir /results/smoke-unique + +# Full native inference and local scoring, only after all task assets validate. +python launchable.py prepare --benchmarks all +python launchable.py run --benchmarks all --mode all --work-dir /results/full-unique + +# Separate inference from scoring; the latter never contacts a model endpoint. +python launchable.py run --benchmarks VANTAGE_VQA --mode infer --work-dir /results/infer-unique +python launchable.py score --work-dir /results/infer-unique + +# Native prediction files from another run: retain original identities/questions. +python -m vlmeval.local_scoring --task VANTAGE_VQA --predictions /results/native-predictions.tsv \ + --assets-dir /scoring-assets --output-dir /results/score-unique +``` + +All output directories must be fresh. Direct scoring requires full task coverage unless `--allow-subset` is explicitly supplied. A smoke/subset is always identified in coverage and is not a reference benchmark result. Duplicate, unknown, mismatched, blank, or failed predictions are rejected. Only verified prediction strings are transferred into trusted dataset rows: prediction-supplied labels, image dimensions, or paths cannot override ground truth. + +`--mode all` always performs native inference first, then the same local scorer used for existing outputs. It never substitutes inference-only behavior or runs a remote scoring service. `--dry-run` composes configuration but does not prove inference or media compatibility. + +## Metric semantics and provenance + +| Task | Native primary metric | Unit / aggregation | +| --- | --- | --- | +| VQA | accuracy | Fraction; question micro | +| EventVerification | macro F1 | Fraction; class macro | +| Temporal | temporal IoU | Fraction; video macro of question means | +| DVC | SODA-c | Fraction; video macro, genuine local BERTScore | +| 2DGrounding | Acc@0.5 | Percent; expression micro | +| 2DPointing | accuracy | Fraction; question micro | +| SOT | success AUC | Fraction; sequence macro; initialization excluded | +| Astro2D | F1 | Percent; unweighted sequence macro | + +Each private `summary.json` includes native metrics, explicit units/aggregation, expected/scored row counts, asset ID and annotation hashes, prediction snapshot hash, scorer-source hashes, and DVC metric-model fingerprints. There is **no invented cross-task overall VANTAGE score** or unverified leaderboard-parity claim. DVC requires a real local model; remote endpoints and dummy semantic scores are not allowed in local scoring. Missing or silently skipped examples fail scoring. Native failures leave a private `scoring-error.log`, not a success summary. + +`run.json` records controller state and the exact scoring-asset manifest hash. Generated native configuration and `/opt/evaluation-packages.txt` identify runtime inputs. A BYOM endpoint's revision cannot be inferred from its display name; record its deployment revision separately. Treat predictions, native diagnostics, and summaries as private until reviewed. Nothing is uploaded automatically. + +## BYOM, stop, and restart + +At initial deployment, set `COSMOS_API_BASE` to the full `/v1/chat/completions` URL, `COSMOS_MODEL` to its exact served ID, and `COSMOS_API_KEY` only if required. Setup skips local serving for BYOM. The endpoint must support the native image/video requests and be reachable from the evaluator; container `localhost` is not the host. Native recipe exports intentionally retain environment placeholders; the controller supplies the checked endpoint URL before inference. The API adapter reads its key from the environment, not the generated configuration. Keep credentials in the environment, never notebook source or URLs. + +From the launchable directory on the host: + +```bash +docker compose --profile local ps +docker compose --profile local stop +# Recreate with the same bundle and intended mode: +bash setup.sh --score-only +``` + +Stopping a container does not stop underlying Brev VM billing. Stop the instance separately after preserving authorized results. Do not delete state or volumes to stop a run. Clear notebook outputs before sharing. + +## Local development checks + +In the installed evaluator environment, run `python -m unittest discover -s launchable -p 'test_*.py'`. Behavioral fixtures exercise native scorers without private data or inference services; DVC additionally needs a smoke with the real metric checkpoint. The committed notebook contains no executed outputs. + +Platform limit: local ARM64 Docker's `linux/amd64` Jupyter kernel previously stalled even in a clean upstream image. Native Linux x86-64 is the supported runtime. CLI/native metric checks under emulation are not proof of notebook execution or a fresh Brev GPU deployment. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml new file mode 100644 index 000000000..e1ae93b95 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +name: cosmos3-vantage +services: + cosmos-endpoint: + profiles: [local] + platform: linux/amd64 + image: ${VLLM_IMAGE:-vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55} + restart: unless-stopped + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + ipc: host + environment: + HF_TOKEN: ${HF_TOKEN:-} + HF_HOME: /cache/huggingface + volumes: + - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache + - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/data:/data:ro + # Deliberately no host port: only the evaluator can reach this API. + command: + - nvidia/Cosmos3-Nano + - --revision + - e59a53c25979a090fa8706c9acc0c254a6e89b92 + - --async-scheduling + - --allowed-local-media-path + - / + - --port + - '8080' + - --max-model-len + - '128000' + - --tensor-parallel-size + - '1' + - --gpu-memory-utilization + - '0.9' + - --enable-chunked-prefill + - --mm-processor-cache-gb + - '0' + - --mm-encoder-tp-mode + - data + - --media-io-kwargs + - '{"video": {"num_frames": -1, "fps": -1}}' + healthcheck: + test: [CMD, python3, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/v1/models', timeout=5)"] + interval: 15s + timeout: 10s + retries: 120 + start_period: 60s + vantage-eval: + platform: linux/amd64 + build: + context: .. + dockerfile: launchable/Dockerfile + args: + COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} + restart: unless-stopped + ports: + - '${JUPYTER_BIND_ADDRESS:-127.0.0.1}:8888:8888' + environment: + HF_TOKEN: ${HF_TOKEN:-} + COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} + COSMOS_API_BASE: ${COSMOS_API_BASE:-http://cosmos-endpoint:8080/v1/chat/completions} + COSMOS_MODEL: ${COSMOS_MODEL:-nvidia/Cosmos3-Nano} + COSMOS_API_KEY: ${COSMOS_API_KEY:-EMPTY} + JUPYTER_TOKEN: ${JUPYTER_TOKEN:?Run bash setup.sh to generate a Jupyter token} + LMUData: /cache/lmudata + SCORING_ASSETS_DIR: /scoring-assets + EVAL_SCORE_ONLY: ${EVAL_SCORE_ONLY:-0} + AWS_CONFIG_FILE: /dev/null + AWS_SHARED_CREDENTIALS_FILE: /dev/null + AWS_EC2_METADATA_DISABLED: 'true' + volumes: + - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache + - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/data:/data:ro + - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/results:/results + - ${SCORING_ASSETS_DIR:?Set an external absolute prepared bundle directory}:/scoring-assets:ro + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8888/login', timeout=5)"] + interval: 15s + timeout: 10s + retries: 20 diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py new file mode 100644 index 000000000..b0fd3a6ef --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Start Jupyter without placing the authentication token in process arguments.""" +import os + +from jupyter_server.serverapp import ServerApp +from traitlets.config import Config + +config = Config() +config.ServerApp.ip = "0.0.0.0" +config.ServerApp.port = 8888 +config.ServerApp.port_retries = 0 +config.ServerApp.open_browser = False +config.ServerApp.allow_root = True +# Brev's reverse proxy preserves a non-local Host header. Token auth remains on. +config.ServerApp.allow_remote_access = True +config.ServerApp.root_dir = "/" +config.ServerApp.default_url = "/lab/tree/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb" +config.ServerApp.jpserver_extensions = {"jupyterlab": True} +config.IdentityProvider.token = os.environ["JUPYTER_TOKEN"] +ServerApp.launch_instance(config=config) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py new file mode 100644 index 000000000..cc52c2cdf --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Notebook/CLI orchestration; inference and metrics remain native VLMEvalKit's.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +KIT = ROOT / "cosmos_eval" + + +def manifest(): + return json.loads((KIT / "manifest.json").read_text()) + + +def benchmarks(value): + available = [key.split("/", 1)[1] for key in manifest() if "/VANTAGE_" in key] + selected = available if value == "all" else value.split(",") + unknown = set(selected) - set(available) + if unknown or not selected: + raise ValueError(f"Unknown VANTAGE benchmark(s): {', '.join(sorted(unknown))}") + return [bench for bench in available if bench in selected] + + +def endpoint(wait=0): + base = os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions") + if not base.endswith("/chat/completions"): + raise ValueError("COSMOS_API_BASE must be the complete /v1/chat/completions URL") + os.environ["COSMOS_API_BASE"] = base + url = base.removesuffix("/chat/completions") + "/models" + key = os.environ.get("COSMOS_API_KEY", "EMPTY") + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}"}) + deadline = time.monotonic() + wait + while True: + try: + with urllib.request.urlopen(request, timeout=10) as response: + models = json.load(response)["data"] + if not models: + raise ValueError("The endpoint returned no served models") + requested = os.environ.get("COSMOS_MODEL") + ids = [model["id"] for model in models] + if requested and requested not in ids: + raise ValueError(f"COSMOS_MODEL {requested!r} is not served; available: {ids}") + if not requested and len(ids) != 1: + raise ValueError(f"Set COSMOS_MODEL to one of {ids}") + os.environ["COSMOS_MODEL"] = requested or ids[0] + print(f"Endpoint ready; model: {os.environ['COSMOS_MODEL']}", flush=True) + return + except urllib.error.HTTPError as exc: + if exc.code not in (502, 503, 504): + raise RuntimeError(f"Model endpoint returned HTTP {exc.code}") from exc + if time.monotonic() >= deadline: + raise RuntimeError(f"Model endpoint not ready after {wait}s (HTTP {exc.code})") from exc + except (urllib.error.URLError, TimeoutError) as exc: + if time.monotonic() >= deadline: + raise RuntimeError(f"Model endpoint not reachable after {wait}s") from exc + time.sleep(min(5, max(0, deadline - time.monotonic()))) + + +def prepare(selected, assets_dir): + from vlmeval.local_scoring import build_local_dataset, load_assets + + bundle = load_assets(assets_dir, selected) + for bench in selected: + dataset = build_local_dataset(assets_dir, bench) + if dataset is None or dataset.data.empty: + raise RuntimeError(f"{bench}: no labeled dataset rows") + print(f"{bench}: {len(dataset.data)} local labeled rows ready", flush=True) + diagnostics = bundle["manifest"]["tasks"][bench.removeprefix("VANTAGE_")].get("annotation_diagnostics") + if diagnostics: + print(f"{bench}: original annotation diagnostics: {json.dumps(diagnostics, sort_keys=True)}", flush=True) + + +def run_command(command, **kwargs): + env = {key: value for key, value in os.environ.items() if not key.startswith("AWS_")} + env.update(AWS_CONFIG_FILE="/dev/null", AWS_SHARED_CREDENTIALS_FILE="/dev/null", + AWS_EC2_METADATA_DISABLED="true", MPLBACKEND="Agg") + subprocess.run(command, cwd=ROOT, env=env, check=True, **kwargs) + + +def prediction_files(work): + return sorted(p for p in work.rglob("cosmos_*") + if p.parent.name.startswith("T") and p.suffix in {".tsv", ".xlsx", ".json"} + and p.stem in {f"cosmos_{bench}" for bench in benchmarks("all")}) + + +def verify_predictions(work, bench, expected_rows=None): + import pandas as pd + + files = [p for p in prediction_files(work) if p.stem == f"cosmos_{bench}"] + if len(files) != 1: + raise RuntimeError(f"{bench}: expected one prediction table, found {len(files)}") + path = files[0] + if path.suffix == ".tsv": + frame = pd.read_csv(path, sep="\t") + elif path.suffix == ".json": + frame = pd.read_json(path) + else: + frame = pd.read_excel(path) + if "prediction" not in frame or frame.empty: + raise RuntimeError(f"{bench}: empty or invalid prediction output") + if expected_rows is not None and len(frame) != expected_rows: + raise RuntimeError(f"{bench}: expected {expected_rows} predictions, found {len(frame)}") + text = frame["prediction"].astype(str).str.strip().str.lower() + failed = (frame["prediction"].isna() + | text.isin(("", "nan", "none", "null", "")) + | text.str.contains("failed to obtain answer", regex=False) + | text.str.contains("video_load_failed", regex=False)) + if failed.any(): + raise RuntimeError(f"{bench}: {int(failed.sum())} failed/empty predictions; inspect the run log") + print(f"{bench}: {len(frame)} predictions verified", flush=True) + return path + + +def _write_state(work, state): + (work / "run.json").write_text(json.dumps(state, indent=2) + "\n") + + +def run(selected, work, mode, smoke, dry_run, assets_dir): + if smoke and selected != ["VANTAGE_VQA"]: + raise ValueError("--smoke requires --benchmarks VANTAGE_VQA") + if work.exists(): + raise ValueError(f"Use a fresh --work-dir (already exists: {work}) to avoid stale results") + from vlmeval.local_scoring import dataset_config, load_assets + + # Missing labels fail before allocating inference work or contacting the model. + bundle = load_assets(assets_dir, selected) + local_configs = {bench: dataset_config(assets_dir, bench) for bench in selected} + if not dry_run: + prepare(selected, assets_dir) + endpoint(wait=1800) + work.mkdir(parents=True, mode=0o700) + configs = work / "_configs" + run_command([sys.executable, str(KIT / "run_all.py"), "--benchmarks", ",".join(selected), + "--export-configs", str(configs)]) + state = {"format_version": 2, "source": "local", "status": "planned" if dry_run else "running", + "cosmos_commit": os.environ.get("COSMOS_COMMIT", "unknown"), + "model": os.environ.get("COSMOS_MODEL", "${COSMOS_MODEL}"), + "benchmarks": selected, "mode": mode, "smoke": smoke, "dry_run": dry_run, + "scoring_assets": {"asset_id": bundle["manifest"]["asset_id"], + "manifest_sha256": bundle["manifest_sha256"]}, + "work_dirs": {}, "predictions": {}, "scores": {}} + _write_state(work, state) + entries = manifest() + try: + for index, bench in enumerate(selected): + config_path = configs / f"{bench}.json" + cfg = json.loads(config_path.read_text()) + # Recipe exports retain placeholders; supply the endpoint already checked above. + cfg["model"]["cosmos"]["api_base"] = os.environ.get( + "COSMOS_API_BASE", cfg["model"]["cosmos"]["api_base"]) + cfg["data"] = {bench: local_configs[bench]} + if smoke: + cfg["data"][bench].update(limit=5, random_state=0) + config_path.write_text(json.dumps(cfg, indent=2) + "\n") + out = work / f"bench_{index:02d}" + state["work_dirs"][bench] = out.name + flags = entries[f"metropolis/{bench}"]["run"] + command = [sys.executable, str(ROOT / "run.py"), "--config", str(config_path), + "--mode", "infer", "--work-dir", str(out), + "--api-nproc", str(flags["api_nproc"]), "--verbose"] + if dry_run: + print(" ".join(command)) + continue + out.mkdir(mode=0o700) + with (out / "run.log").open("w") as log: + run_command(command, stdout=log, stderr=subprocess.STDOUT) + path = verify_predictions(out, bench, expected_rows=5 if smoke else None) + state["predictions"][bench] = str(path.relative_to(work)) + _write_state(work, state) + if not dry_run: + state["status"] = "inferred" + _write_state(work, state) + if mode == "all": + score(work, assets_dir) + else: + results(work) + except Exception: + state = json.loads((work / "run.json").read_text()) + state["status"] = "failed" + _write_state(work, state) + raise + + +def score(work, assets_dir, score_dir=None): + from vlmeval.local_scoring import load_assets, score_predictions + + state = json.loads((work / "run.json").read_text()) + if state.get("source") != "local": + raise ValueError("Use the native score-only CLI for predictions from older or other inference runs") + if state.get("dry_run") or state.get("status") not in {"inferred", "complete"}: + raise ValueError("Only successful inference runs can be scored") + selected = state["benchmarks"] + bundle = load_assets(assets_dir, selected) + if state.get("scoring_assets", {}).get("manifest_sha256") != bundle["manifest_sha256"]: + raise ValueError("Scoring assets changed since inference; use the original fingerprinted bundle") + score_dir = Path(score_dir) if score_dir is not None else work / "local-scores" + if score_dir.exists(): + raise ValueError("Scoring output already exists; choose a fresh --score-dir") + predictions = {bench: verify_predictions(work / state["work_dirs"][bench], bench, + expected_rows=5 if state.get("smoke") else None) + for bench in selected} + score_dir.mkdir(parents=True, mode=0o700) + state["status"] = "scoring" + state["scores"] = {} + _write_state(work, state) + try: + for index, bench in enumerate(selected): + output = score_dir / f"bench_{index:02d}" + summary = score_predictions(bench, predictions[bench], assets_dir, output, + allow_subset=bool(state.get("smoke"))) + state["scores"][bench] = {"summary": str(output / "summary.json"), + "primary_metric": summary["primary_metric"], + "coverage": summary["coverage"]} + _write_state(work, state) + state["status"] = "complete" + _write_state(work, state) + except Exception: + state["status"] = "failed" + _write_state(work, state) + raise + results(work) + + +def results(work): + state = json.loads((work / "run.json").read_text()) + print(f"Local run: {state.get('status', 'unknown')}" + + (" (five-question smoke subset)" if state.get("smoke") else "")) + for bench in state["benchmarks"]: + result = state.get("scores", {}).get(bench) + if result: + metric, coverage = result["primary_metric"], result["coverage"] + print(f"{bench}: {metric['name']}={metric['value']} {metric['unit']}; " + f"{coverage['scored_rows']}/{coverage['expected_rows']} rows") + print(f" Local score: {result['summary']}") + else: + print(f"{bench}: predictions only; local scoring has not completed") + print("Per-task native metrics only; no external scoring submission or unverified leaderboard parity.") + + +def main(): + os.environ["MPLBACKEND"] = "Agg" + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check", help="Wait for /v1/models and verify the requested model") + check.add_argument("--wait", type=int, default=1800) + prep = sub.add_parser("prepare", help="Validate explicitly supplied local labeled scoring assets") + runner = sub.add_parser("run", help="Run native inference and optionally local scoring") + scorer = sub.add_parser("score", help="Score existing predictions locally, without a model endpoint") + for child in (prep, runner, scorer): + child.add_argument("--assets-dir", type=Path, + default=Path(os.environ.get("SCORING_ASSETS_DIR", "/scoring-assets"))) + for child in (prep, runner): + child.add_argument("--benchmarks", default="VANTAGE_VQA") + runner.add_argument("--mode", choices=["infer", "all"], default="all") + runner.add_argument("--smoke", action="store_true", help="Five seeded VQA samples, not a full benchmark") + runner.add_argument("--dry-run", action="store_true") + runner.add_argument("--work-dir", type=Path, required=True) + scorer.add_argument("--work-dir", type=Path, required=True) + scorer.add_argument("--score-dir", type=Path) + report = sub.add_parser("results", help="Show verified local metric summaries") + report.add_argument("--work-dir", type=Path, required=True) + args = parser.parse_args() + if args.command == "check": + endpoint(args.wait) + elif args.command == "prepare": + prepare(benchmarks(args.benchmarks), args.assets_dir.resolve()) + elif args.command == "run": + run(benchmarks(args.benchmarks), args.work_dir.resolve(), args.mode, + args.smoke, args.dry_run, args.assets_dir.resolve()) + elif args.command == "score": + score(args.work_dir.resolve(), args.assets_dir.resolve(), args.score_dir) + else: + results(args.work_dir.resolve()) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh new file mode 100644 index 000000000..20686600a --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +umask 077 +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +command -v docker >/dev/null +command -v python3 >/dev/null +docker compose version +docker info >/dev/null +export COSMOS_COMMIT +COSMOS_COMMIT=$(git rev-parse HEAD) +: "${LAUNCHABLE_STATE_DIR:?Set an absolute state directory outside the Framework checkout}" +: "${SCORING_ASSETS_DIR:?Set an absolute authorized scoring bundle directory outside the Framework checkout}" +export LAUNCHABLE_STATE_DIR SCORING_ASSETS_DIR +python3 - "$LAUNCHABLE_STATE_DIR" "$SCORING_ASSETS_DIR" "$(git rev-parse --show-toplevel)" <<'PY' +from pathlib import Path +import sys + +checkout = Path(sys.argv[3]).resolve() +state, assets = (Path(value) for value in sys.argv[1:3]) +if any(not path.is_absolute() or path.resolve().is_relative_to(checkout) for path in (state, assets)): + raise SystemExit("State and scoring assets must use absolute paths outside the Framework checkout") +if not assets.is_dir(): + raise SystemExit("Scoring bundle directory is absent; prepare authorized sources first") +if any(assets.resolve().is_relative_to((state / name).resolve()) for name in ("cache", "results")): + raise SystemExit("Scoring assets must not be nested in writable cache/results mounts") +PY +mkdir -p "$LAUNCHABLE_STATE_DIR"/{cache,data,results} +export EVAL_SCORE_ONLY=0 +for argument in "$@"; do + if [[ "$argument" == "--score-only" ]]; then EVAL_SCORE_ONLY=1; fi +done +# Persist notebook authentication and its non-secret bind address, never model/judge credentials. +export JUPYTER_BIND_ADDRESS +JUPYTER_BIND_ADDRESS=$(python3 - "$@" <<'PY' +import ipaddress +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys + +if set(sys.argv[1:]) - {"--brev", "--score-only"}: + raise SystemExit("Usage: bash setup.sh [--brev] [--score-only]") +path = Path(".env") +lines = path.read_text().splitlines() if path.exists() else [ + "JUPYTER_TOKEN=" + secrets.token_hex(32) +] +stored = dict(line.split("=", 1) for line in lines if line.startswith("JUPYTER_BIND_ADDRESS=")) +address = os.environ.get("JUPYTER_BIND_ADDRESS") or stored.get("JUPYTER_BIND_ADDRESS", "127.0.0.1") +if "--brev" in sys.argv[1:] and not os.environ.get("JUPYTER_BIND_ADDRESS"): + try: + interfaces = json.loads(subprocess.check_output( + ["ip", "-j", "-4", "addr", "show", "dev", "wt0"], timeout=5)) + addresses = [item["local"] for interface in interfaces for item in interface["addr_info"] + if item["family"] == "inet" and item.get("scope") == "global"] + if len(addresses) != 1: + raise ValueError("expected exactly one global wt0 IPv4 address") + address = addresses[0] + if ipaddress.IPv4Address(address) not in ipaddress.IPv4Network("100.64.0.0/10"): + raise ValueError("wt0 is not in the private Brev mesh range") + except (OSError, subprocess.SubprocessError, ValueError, KeyError) as error: + raise SystemExit(f"Brev private notebook binding unavailable: {error}") from error +allowed = ("127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10") +if not any(ipaddress.IPv4Address(address) in ipaddress.IPv4Network(network) for network in allowed): + raise SystemExit("JUPYTER_BIND_ADDRESS must be a loopback or private IPv4 address, never a public/wildcard address") +lines = [line for line in lines if not line.startswith("JUPYTER_BIND_ADDRESS=")] +path.write_text("\n".join(lines + ["JUPYTER_BIND_ADDRESS=" + address]) + "\n") +print(address) +PY +) +chmod 600 .env +docker compose build vantage-eval +if [[ "$EVAL_SCORE_ONLY" == "0" ]]; then + # Reject missing/incomplete labels before starting a paid model service. + docker compose run --rm --no-deps vantage-eval python launchable.py prepare +fi +if [[ "$EVAL_SCORE_ONLY" == "0" && -z "${COSMOS_API_BASE:-}" ]]; then + nvidia-smi + docker compose --profile local pull cosmos-endpoint + docker compose --profile local up -d +else + # Existing-prediction scoring and BYOM never start a local model/GPU. + docker compose up -d vantage-eval +fi +printf 'Jupyter is starting on %s:8888. Open its authenticated secure link or SSH tunnel.\n' "${JUPYTER_BIND_ADDRESS:-127.0.0.1}" +printf '%s\n' \ + 'Retrieve the notebook login token with: docker compose exec vantage-eval jupyter server list' \ + 'Use the notebook to score existing predictions or run native inference and local scoring.' diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py new file mode 100644 index 000000000..89e0e233f --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Artifact safety regressions; no GPU, credentials, or network required.""" +import contextlib +import io +from pathlib import Path +import tempfile +import unittest + +import launchable + + +class ArtifactTests(unittest.TestCase): + def test_existing_output_is_rejected_before_network_or_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + with self.assertRaisesRegex(ValueError, "fresh --work-dir"): + launchable.run(["VANTAGE_VQA"], Path(temp), "infer", True, False, Path(temp) / "assets") + + def test_predictions_reject_empty_and_failure_markers(self): + for value in (" ", "VIDEO_LOAD_FAILED", "prefix VIDEO_LOAD_FAILED", + "Failed to obtain answer via API.", "null"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as temp: + work = Path(temp) + output = work / "cosmos" / "T20261023" + output.mkdir(parents=True) + (output / "cosmos_VANTAGE_VQA.tsv").write_text( + "index\tprediction\n0\t" + value + "\n") + with self.assertRaisesRegex(RuntimeError, "failed/empty"): + launchable.verify_predictions(work, "VANTAGE_VQA") + + def test_smoke_requires_exactly_five_predictions(self): + for count in (4, 5, 6): + with self.subTest(count=count), tempfile.TemporaryDirectory() as temp: + work = Path(temp) + output = work / "cosmos" / "T20261023" + output.mkdir(parents=True) + (output / "cosmos_VANTAGE_VQA.tsv").write_text( + "index\tprediction\n" + "".join(f"{i}\tA\n" for i in range(count))) + if count == 5: + with contextlib.redirect_stdout(io.StringIO()): + launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) + else: + with self.assertRaisesRegex(RuntimeError, "expected 5 predictions"): + launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) + + def test_multiple_dated_prediction_tables_are_ambiguous(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) + for stamp in ("T20261023", "T20261024"): + output = work / "cosmos" / stamp + output.mkdir(parents=True) + (output / "cosmos_VANTAGE_VQA.tsv").write_text("index\tprediction\n0\tA\n") + with self.assertRaisesRegex(RuntimeError, "expected one prediction table, found 2"): + launchable.verify_predictions(work, "VANTAGE_VQA") + + +if __name__ == "__main__": + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py new file mode 100644 index 000000000..c69e4ff1e --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic, nonsecret local bundles exercising real native scoring behavior. + +Run from the installed VLMEvalKit environment. No GPU, network, credentials, +model endpoint, or fabricated DVC semantic scorer is used. +""" +import contextlib +import hashlib +import io +import json +from pathlib import Path +import tempfile +import unittest + +import pandas as pd +from PIL import Image + +from vlmeval.local_scoring import ( + build_local_dataset, dataset_config, load_assets, score_predictions, +) +from vlmeval.scoring_assets import EVENT_FILES, SOURCE_DIRS, VQA_FILES, prepare_assets + + +def _json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + +def _image(path, size=(100, 100)): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new('RGB', size, color='white').save(path) + + +def _video(path): + # Only file identity is used by these scoring-only fixtures. No decoder or + # inference is involved; duration is explicitly supplied by annotations. + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'synthetic scoring fixture: media is not decoded') + + +def _prepare(root, tasks): + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + return prepare_assets(root, 'synthetic-nonsecret-v1', tasks=tasks) + + +def _grounding(root, count=2): + source = root / SOURCE_DIRS['2DGrounding'] + _image(source / 'images/example.png') + rows = [] + for index in range(count): + rows.append({ + 'media': 'images/example.png', + 'conversations': [{'from': 'human', 'value': 'Locate the square.'}, + {'from': 'gpt', 'value': '[[10, 10, 50, 50]]'}], + '_meta': {'sentence': f'square {index}', 'image_width': 100, 'image_height': 100}, + }) + _json(source / 'refdrone_test_llava.json', rows) + + +def _sot(root): + source = root / SOURCE_DIRS['SOT'] + for sequence in ('first', 'second'): + directory = source / sequence + _json(directory / 'gt.json', { + 'frame_ids': [0, 10, 20], 'label': f'camera/{sequence}', + 'init_bbox': [100, 100, 500, 500], + 'gt_bboxes': {'0': [100, 100, 500, 500], '10': [100, 100, 500, 500], '20': None}, + }) + for name in ('f00.png', 'f01.png', 'f02.png', 'crop.png'): + _image(directory / 'frames' / name) + + +def _astro(root): + from vlmeval.dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset + for child, layout in Astro2DDetectionDataset._LAYOUT.items(): + source = root / SOURCE_DIRS['Astro2D'] / layout['data_root'].rsplit('/', 1)[1] + # Unequal child sizes make micro-vs-sequence-macro mistakes observable. + count = 2 if child.endswith('_C0065') else 1 + for index in range(count): + _image(source / 'images' / f'{index}.png') + labels = source / 'labels' / f'{index}.txt' + labels.parent.mkdir(parents=True, exist_ok=True) + labels.write_text('Pedestrian 0 0 0 10 10 50 50 0 0 0 0 0 0 0\n') + + +def _events(root): + source = root / SOURCE_DIRS['EventVerification'] + for index, relative in enumerate(EVENT_FILES): + annotation = source / relative + _video(annotation.parent / 'clip.mp4') + _json(annotation, {'bcq': [{'id': f'item-{index}', 'video': 'clip.mp4', + 'question': 'Is the square visible?', 'answer': 'yes', + 'system_prompt': 'Answer yes or no.'}]}) + + +def _vqa(root): + source = root / SOURCE_DIRS['VQA'] + files = [name for name in VQA_FILES if not name.startswith('VANTAGE_')] + for index, name in enumerate(files): + video = f'clip-{index}' + _video(source / 'videos' / (video + '.mp4')) + _json(source / 'data_jsons/annotations' / name, [{ + 'q_uid': video, 'question_id': f'q-{index}', 'question': 'Choose the square.', + 'options': ['square', 'circle', 'triangle', 'star'], 'gt_option': 'A', + }]) + + +def _temporal(root): + source = root / SOURCE_DIRS['Temporal'] + rows = [] + for video, question, answer in ( + ('first', 'q1', '<1><3>'), ('first', 'q2', '<1><3>'), ('second', 'q3', '<1><3>')): + _video(source / 'videos' / (video + '.mp4')) + rows.append({'vid': video, 'question_id': question, 'question': question, + 'answer': answer, 'duration': 10}) + _json(source / 'data_jsons/annotations/examples.json', rows) + + +def _pointing(root): + source = root / SOURCE_DIRS['2DPointing'] + _image(source / 'images/example.png') + pd.DataFrame([ + {'index': index, 'question_id': f'point-{index}', 'question': 'Choose the marked point.', + 'answer': answer, 'target_point': '[20,30]', 'image_path': 'images/example.png', + 'A': 'upper left', 'B': 'upper right', 'C': 'lower left', 'D': 'lower right'} + for index, answer in ((10, 'A'), (11, 'B')) + ]).to_csv(source / 'Metropolis2DPointing.tsv', sep='\t', index=False) + + +class LocalScoringTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.assets = self.root / 'assets' + self.assets.mkdir() + + def predictions(self, task, values, name='predictions.tsv'): + table = build_local_dataset(self.assets, task).data.copy() + table['prediction'] = values + path = self.root / name + table.to_csv(path, sep='\t', index=False) + return path + + def score(self, task, path, name='result', **kwargs): + return score_predictions(task, path, self.assets, self.root / name, **kwargs) + + def test_grounding_ignores_injected_gt_paths_and_reports_native_percent(self): + _grounding(self.assets) + _prepare(self.assets, ['2DGrounding']) + path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) + table = pd.read_csv(path, sep='\t') + table['gt_bboxes'] = '[[70,70,90,90]]' + table['image_path'] = '/untrusted/does-not-exist.png' + table['image_width'] = 98765 + table['image_height'] = 98765 + table.to_csv(path, sep='\t', index=False) + diagnostic = io.StringIO() + with contextlib.redirect_stdout(diagnostic), contextlib.redirect_stderr(diagnostic): + summary = self.score('2DGrounding', path) + self.assertEqual(diagnostic.getvalue(), '') + self.assertEqual(summary['primary_metric']['value'], 100.0) + self.assertEqual(summary['primary_metric']['unit'], 'percent') + self.assertEqual(summary['native_metrics']['Mean_IoU'], 100.0) + scored = pd.read_csv(self.root / 'result/predictions_VANTAGE_2DGrounding.tsv', sep='\t') + self.assertTrue(scored['image_path'].map(lambda value: Path(value).is_file()).all()) + self.assertEqual(summary['provenance']['predictions_sha256'], hashlib.sha256(path.read_bytes()).hexdigest()) + self.assertEqual(json.loads((self.root / 'result/summary.json').read_text()), summary) + + def test_partial_requires_explicit_subset_and_never_claims_full_coverage(self): + _grounding(self.assets) + _prepare(self.assets, ['2DGrounding']) + path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) + table = pd.read_csv(path, sep='\t').iloc[:1] + table.to_csv(path, sep='\t', index=False) + with self.assertRaisesRegex(ValueError, 'Full local dataset coverage'): + self.score('2DGrounding', path) + self.assertFalse((self.root / 'result').exists()) + summary = self.score('2DGrounding', path, allow_subset=True) + self.assertEqual(summary['coverage'], {'expected_rows': 2, 'scored_rows': 1, 'subset': True}) + + def test_duplicate_unknown_and_mismatched_identity_fail_before_output(self): + _grounding(self.assets) + _prepare(self.assets, ['2DGrounding']) + path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) + original = pd.read_csv(path, sep='\t') + for kind in ('duplicate', 'unknown', 'mismatch', 'missing_identity'): + with self.subTest(kind=kind): + table = original.copy() + if kind == 'duplicate': + table.loc[1, 'index'] = table.loc[0, 'index'] + elif kind == 'unknown': + table.loc[1, 'index'] = 999 + elif kind == 'mismatch': + table.loc[1, 'sentence'] = 'another example' + else: + table = table.drop(columns=['sentence']) + table.to_csv(path, sep='\t', index=False) + with self.assertRaises(ValueError): + self.score('2DGrounding', path) + self.assertFalse((self.root / 'result').exists()) + + def test_native_sot_uses_integer_cache_and_absent_frame_semantics(self): + _sot(self.assets) + _prepare(self.assets, ['SOT']) + predictions = [json.dumps({'frame_1': [100, 100, 500, 500], 'frame_2': None}), + json.dumps({'frame_1': None, 'frame_2': None})] + path = self.predictions('SOT', predictions) + summary = self.score('SOT', path) + # Native threshold grid includes zero: a missed visible frame has AUC + # 1/21, not exactly zero. The absent frame is excluded, not rewarded. + self.assertAlmostEqual(summary['native_metrics']['camera/first'], 1.0) + self.assertAlmostEqual(summary['native_metrics']['camera/second'], 1 / 21) + self.assertAlmostEqual(summary['primary_metric']['value'], (1 + 1 / 21) / 2) + self.assertEqual(summary['primary_metric']['unit'], 'fraction') + + def test_sot_missing_gt_frame_is_rejected_even_with_updated_fingerprint(self): + _sot(self.assets) + _prepare(self.assets, ['SOT']) + path = self.assets / SOURCE_DIRS['SOT'] / 'first/gt.json' + annotation = json.loads(path.read_text()) + del annotation['gt_bboxes']['20'] + path.write_text(json.dumps(annotation)) + manifest_path = self.assets / 'scoring-assets.json' + manifest = json.loads(manifest_path.read_text()) + manifest['files'][path.relative_to(self.assets).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaises(ValueError): + load_assets(self.assets, ['SOT']) + + def test_astro_distinct_roots_and_unweighted_native_sequence_macro(self): + _astro(self.assets) + _prepare(self.assets, ['Astro2D']) + config = dataset_config(self.assets, 'Astro2D') + self.assertEqual(len(set(config['data_roots'].values())), 3) + dataset = build_local_dataset(self.assets, 'Astro2D') + values = ['[{"bbox_2d":[100,100,500,500],"label":"person"}]' + if row['SUB_DATASET'].endswith('_C0065') else '[]' + for _, row in dataset.data.iterrows()] + path = self.predictions('Astro2D', values) + # Including VANTAGE_Astro2D in a parent directory must not corrupt the + # native child file paths (ConcatDataset's historical string trap). + summary = self.score('Astro2D', path, name='VANTAGE_Astro2D-output') + self.assertAlmostEqual(summary['primary_metric']['value'], 100 / 3) + self.assertEqual(summary['primary_metric']['aggregation'], 'unweighted_sequence_macro') + self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:f1'], 100) + self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_IVAKPI05:f1'], 0) + + def test_astro_inverted_source_boxes_remain_in_native_denominator(self): + from vlmeval.dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset + _astro(self.assets) + layout = Astro2DDetectionDataset._LAYOUT['VANTAGE_Astro2D_C0065'] + label = self.assets / SOURCE_DIRS['Astro2D'] / layout['data_root'].rsplit('/', 1)[1] / 'labels/0.txt' + label.write_text(label.read_text() + 'Person 0 0 0 50 10 10 50 0 0 0 0 0 0 0\n') + _prepare(self.assets, ['Astro2D']) + count = len(build_local_dataset(self.assets, 'Astro2D').data) + path = self.predictions('Astro2D', ['[{"bbox_2d":[100,100,500,500],"label":"person"}]'] * count) + summary = self.score('Astro2D', path) + self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:total_gt_objects'], 3) + self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:false_negatives'], 1) + self.assertAlmostEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:f1'], 80) + self.assertAlmostEqual(summary['primary_metric']['value'], (80 + 100 + 100) / 3) + + def test_event_invalid_predictions_are_not_silently_removed(self): + _events(self.assets) + _prepare(self.assets, ['EventVerification']) + path = self.predictions('EventVerification', ['yes', 'yes', 'yes', 'unparseable']) + with self.assertRaisesRegex(ValueError, 'silent fallback'): + self.score('EventVerification', path) + self.assertFalse((self.root / 'result').exists()) + path = self.predictions('EventVerification', ['yes'] * 4) + summary = self.score('EventVerification', path) + self.assertEqual(summary['native_metrics']['Valid Predictions'], 4) + self.assertEqual(summary['primary_metric']['value'], 1) + + def test_vqa_uses_local_ground_truth_not_prediction_answer(self): + _vqa(self.assets) + _prepare(self.assets, ['VQA']) + dataset = build_local_dataset(self.assets, 'VQA') + path = self.predictions('VQA', ['B'] * len(dataset.data)) + table = pd.read_csv(path, sep='\t') + table['answer'] = 'B' + table.to_csv(path, sep='\t', index=False) + summary = self.score('VQA', path) + self.assertEqual(summary['primary_metric']['value'], 0) + + def test_vqa_binary_explicit_label_is_normalized_without_defaulting(self): + _vqa(self.assets) + source = self.assets / SOURCE_DIRS['VQA'] / 'data_jsons/annotations' / VQA_FILES[0] + records = json.loads(source.read_text()) + records[0].update(options=['square', 'circle'], gt_option=' b ') + _json(source, records) + _prepare(self.assets, ['VQA']) + count = len(build_local_dataset(self.assets, 'VQA').data) + summary = self.score('VQA', self.predictions('VQA', ['A'] * count)) + self.assertEqual(summary['primary_metric']['value'], (count - 1) / count) + + def test_vqa_reused_fallback_qid_cannot_hide_wrong_question(self): + _vqa(self.assets) + source = self.assets / SOURCE_DIRS['VQA'] + _video(source / 'videos/shared.mp4') + _json(source / 'data_jsons/annotations' / VQA_FILES[0], [ + {'q_uid': 'shared.mp4', 'question': question, + 'options': ['square', 'circle', 'triangle', 'star'], 'gt_option': answer} + for question, answer in [('Choose the square.', 'A'), ('Choose the circle.', 'B')] + ]) + _prepare(self.assets, ['VQA']) + count = len(build_local_dataset(self.assets, 'VQA').data) + path = self.predictions('VQA', ['A'] * count) + table = pd.read_csv(path, sep='\t') + shared = table.index[table['video'] == 'shared.mp4'] + self.assertEqual(table.loc[shared, 'qid'].nunique(), 1) + table.loc[shared, 'question'] = table.loc[shared, 'question'].tolist()[::-1] + table.to_csv(path, sep='\t', index=False) + with self.assertRaisesRegex(ValueError, 'stable identity'): + self.score('VQA', path) + self.assertFalse((self.root / 'result').exists()) + + def test_temporal_preserves_video_macro_not_question_micro(self): + _temporal(self.assets) + _prepare(self.assets, ['Temporal']) + dataset = build_local_dataset(self.assets, 'Temporal') + values = [json.dumps({'start': '1', 'end': '3'}) if row['video'] == 'first' + else json.dumps({'start': '5', 'end': '7'}) for _, row in dataset.data.iterrows()] + path = self.predictions('Temporal', values) + summary = self.score('Temporal', path) + self.assertEqual(summary['primary_metric']['value'], 0.5) + self.assertEqual(summary['native_metrics']['overall']['count'], 3) + + def test_temporal_malformed_prediction_cannot_receive_full_duration_default(self): + _temporal(self.assets) + _prepare(self.assets, ['Temporal']) + path = self.predictions('Temporal', ['no parseable interval'] * 3) + with self.assertRaisesRegex(ValueError, 'silent fallback'): + self.score('Temporal', path) + self.assertFalse((self.root / 'result').exists()) + + def test_pointing_uses_native_accuracy_and_rejects_textual_id_alias(self): + _pointing(self.assets) + _prepare(self.assets, ['2DPointing']) + path = self.predictions('2DPointing', ['A', 'D']) + table = pd.read_csv(path, sep='\t') + table['answer'] = ['A', 'D'] + table.to_csv(path, sep='\t', index=False) + summary = self.score('2DPointing', path) + self.assertEqual(summary['primary_metric']['value'], 0.5) + self.assertEqual(summary['primary_metric']['unit'], 'fraction') + table['index'] = ['010', '11'] + table.to_csv(path, sep='\t', index=False) + with self.assertRaisesRegex(ValueError, 'identity does not match'): + self.score('2DPointing', path, name='alias') + + def test_asset_hash_and_escape_and_existing_output_are_rejected(self): + _grounding(self.assets) + _prepare(self.assets, ['2DGrounding']) + path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) + (self.root / 'existing').mkdir() + with self.assertRaisesRegex(ValueError, 'fresh output'): + self.score('2DGrounding', path, name='existing') + manifest_path = self.assets / 'scoring-assets.json' + original = manifest_path.read_text() + manifest = json.loads(original) + annotation = self.assets / SOURCE_DIRS['2DGrounding'] / 'refdrone_test_llava.json' + annotation.write_text(annotation.read_text() + '\n') + with self.assertRaisesRegex(ValueError, 'fingerprint mismatch'): + load_assets(self.assets, ['2DGrounding']) + manifest['tasks']['2DGrounding']['dataset_kwargs']['data_root'] = '../' + manifest['files'][annotation.relative_to(self.assets).as_posix()] = hashlib.sha256(annotation.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'escapes'): + load_assets(self.assets, ['2DGrounding']) + + def test_dvc_without_genuine_local_model_is_rejected(self): + from vlmeval.scoring_assets import validate_bert_score_model + empty_model = self.assets / 'model' + empty_model.mkdir() + with self.assertRaises((ValueError, FileNotFoundError)): + validate_bert_score_model(empty_model) + # Preparation must not create a manifest or silently use a remote or + # dummy semantic scorer when a local checkpoint was not provisioned. + with self.assertRaises(ValueError): + _prepare(self.assets, ['DVC']) + self.assertFalse((self.assets / 'scoring-assets.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py index f69d3941c..1230daf9c 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py @@ -588,6 +588,36 @@ class Astro2DBenchDataset(concat_base): ], } + def __init__(self, dataset='VANTAGE_Astro2D', data_roots=None, **kwargs): + if data_roots is None: + super().__init__(dataset=dataset, **kwargs) + return + children = self.DATASET_SETS[dataset] + if set(data_roots) != set(children): + raise ValueError('data_roots must specify every registered ASTRO child') + self.dataset_name = dataset + self.datasets = children + self.dataset_map = {} + tables = [] + for name in children: + root = os.fspath(data_roots[name]) + if not os.path.isdir(root): + raise ValueError('ASTRO data_roots must be existing local directories') + child = Astro2DDetectionDataset( + dataset=name, data_root=root, + min_bbox_area=400 if name.endswith('_IVAKPI05') else 0, + **kwargs, + ) + self.dataset_map[name] = child + table = child.data.copy() + table['SUB_DATASET'] = name + tables.append(table) + self.data = pd.concat(tables, ignore_index=True) + self.data['original_index'] = self.data.pop('index') + self.data['index'] = np.arange(len(self.data)) + self.TYPE = next(iter(self.dataset_map.values())).TYPE + self.MODALITY = next(iter(self.dataset_map.values())).MODALITY + def evaluate(self, eval_file, **judge_kwargs): result = super().evaluate(eval_file, **judge_kwargs) # f1 (F1@0.5) first so dict iteration / Format-D parsers pick it diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py index c022ebf1c..8b2ef8f3f 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py @@ -113,15 +113,19 @@ class MetropolisDVC(VideoBaseDataset): def __init__(self, dataset='MetropolisDVC', pack=False, nframe=0, fps=0, total_pixels=8192 * 32 * 32, max_pixels=None, max_frames=128, test_mode=False, limit=None, - random_state=None, include_categories=None): + random_state=None, include_categories=None, local_data_file=None, local_video_root=None): self.test_mode = test_mode + self.local_data_file = local_data_file + self.local_video_root = local_video_root + if (local_data_file is None) != (local_video_root is None): + raise ValueError("local_data_file and local_video_root must be supplied together") self.limit = limit self.max_pixels = max_pixels self.max_frames = max_frames self.random_state = random_state self.include_categories = set(include_categories) if include_categories else None - if not test_mode: + if not test_mode or local_data_file is not None: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -324,6 +328,10 @@ def prepare_dataset(self, dataset_name='MetropolisDVC'): - category: category label - qid: question ID """ + if self.local_data_file is not None: + from ..scoring_assets import validate_local_video_inputs + return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'DVC') + if dataset_name == 'VANTAGE_DVC': return self._prepare_vantage_from_s3(dataset_name) @@ -479,6 +487,19 @@ def evaluate(self, eval_file, **judge_kwargs): - Temporal IoU for localization accuracy - BERTScore for caption semantic similarity """ + local_scoring = judge_kwargs.get('local_scoring', False) + local_model = judge_kwargs.get('bert_score_model') + if local_scoring: + if judge_kwargs.get('bert_score_endpoint'): + raise ValueError("Local DVC scoring does not permit a BERTScore endpoint") + from ..scoring_assets import validate_bert_score_model + local_model = str(validate_bert_score_model(local_model)) + try: + import torch + from bert_score import BERTScorer + except ImportError as exc: + raise ImportError("Local DVC scoring requires torch and bert-score") from exc + data = load(eval_file) # Convert data to SODA format @@ -534,6 +555,11 @@ def evaluate(self, eval_file, **judge_kwargs): if gt_timestamps: gts[vid] = {"timestamps": gt_timestamps, "sentences": gt_sentences} + if local_scoring and ( + not len(data) or parse_fail_count or len(preds) != len(data) or len(gts) != len(data) + ): + raise ValueError("Local DVC scoring rejects unmatched, duplicate, or unparseable scoring rows") + if parse_fail_count > 0: print(f'[MetropolisDVC] WARNING: {parse_fail_count} events failed to parse ' f'(missing or malformed start/end timestamps). ' @@ -548,29 +574,27 @@ def evaluate(self, eval_file, **judge_kwargs): print(f"\nEvaluating {len(gt_vids)} videos with SODA-c metric...") - # Check for remote BERTScore endpoint (set via env var or judge_kwargs) - bert_endpoint = judge_kwargs.get('bert_score_endpoint') or os.environ.get('BERT_SCORE_ENDPOINT') - - # Initialize BERTScorer (local or remote) + # Explicit local scoring never consults the remote endpoint environment. + bert_endpoint = None if local_scoring else ( + judge_kwargs.get('bert_score_endpoint') or os.environ.get('BERT_SCORE_ENDPOINT') + ) bert_scorer = None - use_remote = False - - if bert_endpoint: - # Use remote BERTScore endpoint - print(f"Using remote BERTScore endpoint: {bert_endpoint}") - use_remote = True - else: - # Try local BERTScorer + use_remote = bool(bert_endpoint) + if not use_remote: try: import torch from bert_score import BERTScorer - device = "cuda" if torch.cuda.is_available() else "cpu" - bert_scorer = BERTScorer(model_type="roberta-large", device=device) - print(f"BERTScorer initialized locally on {device}") - except ImportError: - print("Warning: bert_score not available, using dummy scorer (F1=0.5)") - print("Install with: pip install bert-score") - print("Or set BERT_SCORE_ENDPOINT env var for remote evaluation") + except ImportError as exc: + raise ImportError("Real DVC semantic scoring requires torch and bert-score") from exc + device = "cuda" if torch.cuda.is_available() else "cpu" + scorer_kwargs = dict(device=device) + if local_scoring: + # Native roberta-large BERTScore uses layer 17. An absolute existing + # model directory makes Transformers resolve every file locally. + scorer_kwargs['num_layers'] = 17 + bert_scorer = BERTScorer( + model_type=local_model if local_scoring else "roberta-large", **scorer_kwargs + ) def compute_bert_scores_remote(candidates: List[str], references: List[str]) -> List[float]: """Call remote BERTScore endpoint.""" @@ -619,8 +643,7 @@ def compute_bert_scores_remote(candidates: List[str], references: List[str]) -> _, _, F1 = bert_scorer.score(pred_sentences, refs) score_matrix[gi, :] = F1.cpu().numpy() else: - # Dummy scores if BERTScore unavailable - score_matrix = np.ones_like(iou_matrix) * 0.5 + raise RuntimeError("DVC semantic scorer was not initialized") # SODA-c: optimal matching using IoU × BERTScore combined_matrix = iou_matrix * score_matrix diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py index 14abc2b42..1b5d25345 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py @@ -60,6 +60,8 @@ def __init__( max_pixels=None, max_frames=None, system_prompt_option='merged', + local_data_file=None, + local_video_root=None, ): """ MetropolisEventVerification dataset for event verification in videos. @@ -69,6 +71,10 @@ def __init__( self.max_pixels = max_pixels self.max_frames = max_frames self.system_prompt_option = system_prompt_option + self.local_data_file = local_data_file + self.local_video_root = local_video_root + if (local_data_file is None) != (local_video_root is None): + raise ValueError("local_data_file and local_video_root must be supplied together") super().__init__( dataset=dataset, nframe=nframe, @@ -81,6 +87,10 @@ def supported_datasets(cls): return list(cls._S3_PATHS.keys()) def prepare_dataset(self, dataset_name='MetropolisEventVerification'): + if self.local_data_file is not None: + from ..scoring_assets import validate_local_video_inputs + return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'EventVerification') + from s3fs import S3FileSystem bucket, prefix = self._S3_PATHS[dataset_name] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py index 0a3d2185a..d8f2740cc 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py @@ -119,8 +119,12 @@ class MetropolisTemporal(VideoBaseDataset): # Question generation prefix (must match metropolis-benchmarks/code/run_qwen3_temporal_api_pt.py) QUESTION_PREFIX = """Localize a series of activity events in the video, output the start and end timestamp for each event. Provide the result in json format with 'mm:ss.ff' format for time depiction for this event. Use keywords 'start' and 'end' in the json output.""" - def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, random_state=None, include_categories=None): + def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, random_state=None, include_categories=None, local_data_file=None, local_video_root=None): self.test_mode = test_mode + self.local_data_file = local_data_file + self.local_video_root = local_video_root + if (local_data_file is None) != (local_video_root is None): + raise ValueError("local_data_file and local_video_root must be supplied together") self.category_mapping = {} self.limit = limit # Store limit parameter self.verbose = verbose # Enable debug logging @@ -134,8 +138,10 @@ def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, to self.S3_BUCKET = bucket self.S3_PREFIX = prefix self.S3_PROFILE = profile - if not test_mode: + if not test_mode or local_data_file is not None: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) + if local_data_file is not None and 'category' in self.data: + self.category_mapping = dict(zip(self.data['video'], self.data['category'])) # Track original size original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -212,6 +218,10 @@ def prepare_dataset(self, dataset_name='MetropolisTemporal'): """ Prepare the dataset by loading from S3 or local cache. """ + if self.local_data_file is not None: + from ..scoring_assets import validate_local_video_inputs + return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'Temporal') + def check_integrity(pth): data_file = osp.join(pth, f'{dataset_name}.tsv') if not osp.exists(data_file): @@ -964,6 +974,8 @@ def evaluate(self, eval_file, **judge_kwargs): def _get_category(self, video_id: str) -> str: """Get category for a video based on its ID.""" + if self.local_data_file is not None and video_id in self.category_mapping: + return self.category_mapping[video_id] if not self.category_mapping: return "Other" diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py index a2caafd8c..e31b1fe65 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py @@ -90,8 +90,12 @@ class MetropolisVQA(VideoBaseDataset): def __init__(self, dataset='MetropolisVQA', pack=False, nframe=0, fps=1.0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, - random_state=None, include_categories=None): + random_state=None, include_categories=None, local_data_file=None, local_video_root=None): self.test_mode = test_mode + self.local_data_file = local_data_file + self.local_video_root = local_video_root + if (local_data_file is None) != (local_video_root is None): + raise ValueError("local_data_file and local_video_root must be supplied together") self.category_mapping = {} self.limit = limit self.verbose = verbose @@ -107,8 +111,10 @@ def __init__(self, dataset='MetropolisVQA', pack=False, nframe=0, fps=1.0, total self.S3_PREFIX = prefix self.S3_PROFILE = profile - if not test_mode: + if not test_mode or local_data_file is not None: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) + if local_data_file is not None and 'category' in self.data: + self.category_mapping = dict(zip(self.data['video'], self.data['category'])) # Track original size original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -166,6 +172,10 @@ def prepare_dataset(self, dataset_name='MetropolisVQA'): """ Prepare the dataset by loading from S3 or local cache. """ + if self.local_data_file is not None: + from ..scoring_assets import validate_local_video_inputs + return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'VQA') + def check_integrity(pth): data_file = osp.join(pth, f'{dataset_name}.tsv') if not osp.exists(data_file): @@ -587,6 +597,8 @@ def get_category(self, vid: str) -> str: def _get_category(self, video_id: str) -> str: """Get category for a video based on its ID from mapping.""" + if self.local_data_file is not None and video_id in self.category_mapping: + return self.category_mapping[video_id] if not self.category_mapping: return "Other" diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py new file mode 100644 index 000000000..999fe7c70 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py @@ -0,0 +1,525 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Credential-free native scoring against a fingerprinted, private local asset bundle. + +Prediction files are not annotation sources. Only their verified identities and +prediction values cross into a new table reconstructed from the local dataset. +The output directory contains private native artifacts and must not be published. +""" +import argparse +from contextlib import contextmanager, redirect_stderr, redirect_stdout +import hashlib +import inspect +from io import BytesIO +import json +import logging +import math +import numbers +import os +from pathlib import Path +import re +import traceback + +import pandas as pd + +TASKS = ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DGrounding', + '2DPointing', 'SOT', 'Astro2D') +_CONFIG_DIR = Path(__file__).resolve().parents[1] / 'cosmos_eval/data/metropolis' +_CLASSES = { + 'VQA': 'MetropolisVQA', 'EventVerification': 'MetropolisEventVerification', + 'Temporal': 'MetropolisTemporal', 'DVC': 'MetropolisDVC', + '2DGrounding': 'Metropolis2DGroundingDataset', '2DPointing': 'VANTAGE_2DPointing', + 'SOT': 'VANTAGE_SOT', 'Astro2D': 'Astro2DBenchDataset', +} +_KWARGS = { + **{task: {'local_data_file', 'local_video_root'} + for task in ('VQA', 'EventVerification', 'Temporal', 'DVC')}, + '2DGrounding': {'data_root', 'annotation_file'}, '2DPointing': {'data_root'}, + 'SOT': {'prepared_data_dir', 'metadata_path'}, 'Astro2D': {'data_roots'}, +} +_ASTRO_CHILDREN = ('VANTAGE_Astro2D_C0065', 'VANTAGE_Astro2D_IVAKPI05', + 'VANTAGE_Astro2D_IVAKPI11') + + +def _task(task): + if task.startswith('VANTAGE_'): + task = task[len('VANTAGE_'):] + if task not in TASKS: + raise ValueError('Unknown local scoring task') + return task + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, 'rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def _inside(root, value, directory=False): + if not isinstance(value, str) or not value or Path(value).is_absolute(): + raise ValueError('Asset paths must be nonempty bundle-relative paths') + path = (root / value).resolve() + if not path.is_relative_to(root): + raise ValueError('Asset path escapes the local bundle') + if not (path.is_dir() if directory else path.is_file()): + raise ValueError('Required local asset is absent') + return path + + +def _require_fingerprint(bundle, path): + path = Path(path).resolve() + if not path.is_relative_to(bundle['root']): + raise ValueError('Ground-truth path escapes the local bundle') + key = path.relative_to(bundle['root']).as_posix() + if key not in bundle['manifest']['files']: + raise ValueError('Required ground truth is not fingerprinted') + + +@contextmanager +def _private_native_output(): + # Native scorers print categories, sequence identities and occasionally GT. + # Do not collect these in a public log or interpolate their exceptions. + previous = logging.root.manager.disable + with open(os.devnull, 'w') as sink, redirect_stdout(sink), redirect_stderr(sink): + logging.disable(logging.CRITICAL) + try: + yield + finally: + logging.disable(previous) + + +def _validate_source(bundle, task): + from .scoring_assets import validate_local_video_inputs, validate_task_sources + config = bundle['configs'][task] + with _private_native_output(): + originals = validate_task_sources( + bundle['root'], task, + bundle['manifest']['tasks'][task]['dataset_kwargs'], + ) + for path in originals: + _require_fingerprint(bundle, path) + if task in ('VQA', 'EventVerification', 'Temporal', 'DVC'): + _require_fingerprint(bundle, config['local_data_file']) + with _private_native_output(): + validate_local_video_inputs(config['local_data_file'], config['local_video_root'], task) + elif task == '2DPointing': + table = Path(config['data_root']) / 'Metropolis2DPointing.tsv' + _require_fingerprint(bundle, table) + if not table.is_file(): + raise ValueError('Local pointing table is absent; downloading is not permitted') + elif task == '2DGrounding': + _require_fingerprint(bundle, config['annotation_file']) + elif task == 'SOT' and 'metadata_path' in config: + _require_fingerprint(bundle, config['metadata_path']) + + +def load_assets(assets_dir, tasks): + """Validate local paths, annotation hashes, and task label coverage. + + Returns bundle metadata with ``root`` (Path), ``manifest``, ``configs`` and + ``manifest_sha256``. No credentials, cache lookup, endpoint, or model is used. + """ + if isinstance(tasks, str): + tasks = [tasks] + selected = tuple(dict.fromkeys(_task(task) for task in tasks)) + if not selected: + raise ValueError('At least one local task is required') + root = Path(assets_dir).expanduser().resolve() + manifest_file = root / 'scoring-assets.json' + with open(manifest_file) as stream: + manifest = json.load(stream) + if (manifest.get('format_version') != 1 or not isinstance(manifest.get('asset_id'), str) + or not manifest['asset_id'].strip() or not isinstance(manifest.get('tasks'), dict) + or not isinstance(manifest.get('files'), dict) or not manifest['files']): + raise ValueError('Invalid local scoring asset manifest') + if set(manifest['tasks']) - set(TASKS): + raise ValueError('Unknown task in local scoring manifest') + for relative, expected in manifest['files'].items(): + path = _inside(root, relative) + if not isinstance(expected, str) or not re.fullmatch('[a-f0-9]{64}', expected) or _sha256(path) != expected: + raise ValueError('Local asset fingerprint mismatch') + configs = {} + for task in selected: + entry = manifest['tasks'].get(task) + if (not isinstance(entry, dict) or type(entry.get('rows')) is not int or entry['rows'] <= 0 + or not isinstance(entry.get('dataset_kwargs'), dict)): + raise ValueError('Requested task is absent or incomplete in local assets') + kwargs = entry['dataset_kwargs'] + required = _KWARGS[task] - ({'metadata_path'} if task == 'SOT' else set()) + if not required <= set(kwargs) or set(kwargs) - _KWARGS[task]: + raise ValueError('Manifest may supply only the native local dataset paths') + with open(_CONFIG_DIR / f'VANTAGE_{task}.json') as stream: + config = json.load(stream) + if config.get('class') != _CLASSES[task] or config.get('dataset') != f'VANTAGE_{task}': + raise ValueError('Committed native dataset config is inconsistent') + if task not in ('VQA', 'EventVerification', 'Temporal', 'DVC'): + config['model_family'] = 'cr' + for key, value in kwargs.items(): + if key == 'data_roots': + if not isinstance(value, dict) or set(value) != set(_ASTRO_CHILDREN): + raise ValueError('ASTRO requires all three registered local child roots') + config[key] = {name: str(_inside(root, path, directory=True)) for name, path in value.items()} + else: + config[key] = str(_inside(root, value, directory=key in { + 'data_root', 'local_video_root', 'prepared_data_dir'})) + configs[task] = config + bundle = {'root': root, 'manifest': manifest, 'configs': configs, + 'manifest_sha256': _sha256(manifest_file)} + for task in selected: + _validate_source(bundle, task) + if 'DVC' in selected: + model = manifest.get('metric_models', {}).get('roberta_large', {}) + if not isinstance(model, dict) or not isinstance(model.get('revision'), str) or not model['revision'].strip(): + raise ValueError('DVC requires a revisioned, genuine local roberta-large model') + bundle['bert_score_model'] = str(_inside(root, model.get('path'), directory=True)) + from .scoring_assets import validate_bert_score_model + validate_bert_score_model(bundle['bert_score_model']) + model_root = Path(bundle['bert_score_model']) + files = {path.relative_to(model_root).as_posix(): path + for path in model_root.rglob('*') + if path.is_file() and '.cache' not in path.relative_to(model_root).parts} + expected = model.get('files') + if not isinstance(expected, dict) or not expected or set(files) != set(expected): + raise ValueError('DVC metric-model files do not match the fingerprinted bundle') + for relative, path in files.items(): + if not path.resolve().is_relative_to(root) or _sha256(path) != expected[relative]: + raise ValueError('DVC metric-model fingerprint mismatch') + return bundle + + +def dataset_config(assets_dir, task): + """Return the committed native inference config with absolute local paths.""" + task = _task(task) + return load_assets(assets_dir, [task])['configs'][task] + + +def _build_dataset(bundle, task): + from . import dataset as native + config = dict(bundle['configs'][task]) + cls = getattr(native, config.pop('class')) + with _private_native_output(): + dataset = cls(**config) + expected = bundle['manifest']['tasks'][task]['rows'] + if len(dataset.data) != expected or 'index' not in dataset.data: + raise ValueError('Native dataset silently omitted required local examples') + keys = [_identity(value) for value in dataset.data['index']] + if len(set(keys)) != expected: + raise ValueError('Native dataset identities are not unique') + if task == 'SOT': + if set(dataset.data['index']) != set(dataset._gt_cache) or any( + type(key) is not int for key in dataset._gt_cache): + raise ValueError('SOT native integer identity cache is incomplete') + return dataset + + +def build_local_dataset(assets_dir, task): + """Build and validate the native dataset without constructing a model.""" + task = _task(task) + return _build_dataset(load_assets(assets_dir, [task]), task) + + +def _identity(value): + # Integral numeric indexes and their canonical decimal spelling are equal; + # distinct textual IDs (e.g. '00' and '0') are deliberately not collapsed. + if isinstance(value, bool) or value is None: + raise ValueError('Missing or invalid example identity') + if isinstance(value, numbers.Integral): + return str(int(value)) + if isinstance(value, numbers.Real): + if math.isfinite(value) and value.is_integer(): + return str(int(value)) + raise ValueError('Invalid numeric example identity') + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError('Missing or invalid example identity') + return value + + +def _stable_identity(value): + if isinstance(value, str): + if not value.strip(): + raise ValueError('Missing stable example identity') + return value + return _identity(value) + + +def _read_predictions(predictions): + path = Path(predictions).expanduser().resolve() + contents = path.read_bytes() + stream = BytesIO(contents) + if path.suffix.lower() in {'.tsv', '.csv'}: + table = pd.read_csv(stream, sep='\t' if path.suffix.lower() == '.tsv' else ',', + dtype=str, keep_default_na=False) + elif path.suffix.lower() == '.xlsx': + table = pd.read_excel(stream, dtype=object, keep_default_na=False) + elif path.suffix.lower() == '.json': + rows = json.loads(contents) + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + raise ValueError('Prediction JSON must be an array of row objects') + table = pd.DataFrame(rows) + else: + raise ValueError('Predictions must be TSV, CSV, XLSX, or JSON rows') + if table.empty or not {'index', 'prediction'} <= set(table): + raise ValueError('Predictions require nonempty index and prediction columns') + return table, hashlib.sha256(contents).hexdigest() + + +def _identity_columns(task, trusted): + if task == 'Astro2D': + return ['SUB_DATASET', 'original_index', 'image_filename'] + if task == 'SOT': + return ['label'] + if task == '2DGrounding': + return ['sentence'] + if task == '2DPointing': + return [column for column in ('question_id', 'question', 'A', 'B', 'C', 'D') if column in trusted] + # VQA's native fallback qid is shared by every question in one video. + # The actual question (including options) must match, not just video/qid. + return [column for column in ('video', 'qid', 'id', 'question', 'options') if column in trusted] + + +def _trusted_predictions(task, dataset, predictions, allow_subset): + trusted = dataset.data + keys = [_identity(value) for value in trusted['index']] + lookup = {key: position for position, key in enumerate(keys)} + incoming = [_identity(value) for value in predictions['index']] + if len(set(incoming)) != len(incoming): + raise ValueError('Duplicate prediction identity') + if set(incoming) - set(keys): + raise ValueError('Prediction identity does not match the local dataset') + if not allow_subset and set(incoming) != set(keys): + raise ValueError('Full local dataset coverage is required; use allow_subset explicitly') + identities = _identity_columns(task, trusted) + if not set(identities) <= set(predictions): + raise ValueError('Predictions must retain the native stable identity columns: ' + ', '.join(identities)) + rows = trusted.iloc[[lookup[key] for key in incoming]].copy().reset_index(drop=True) + for column in identities: + if any(_stable_identity(left) != _stable_identity(right) + for left, right in zip(predictions[column], rows[column])): + raise ValueError('Prediction stable identity does not match its native index') + values = list(predictions['prediction']) + if any(not isinstance(value, str) or not value.strip() or value.strip().lower() in {'null', 'nan', 'none'} + or any(marker in value for marker in ('VIDEO_LOAD_FAILED', 'Failed to obtain answer via API.')) + for value in values): + raise ValueError('Missing or failed prediction; no score was produced') + rows['prediction'] = values + # Normalize only after verified native identities. Both sides of native + # joins then survive TSV numeric inference, including textual '00' IDs. + if task in ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DPointing'): + rows['index'] = [lookup[key] for key in incoming] + dataset.data = trusted.copy() + dataset.data['index'] = list(range(len(trusted))) + if task == 'SOT': + rows['index'] = rows['index'].astype(int) + return rows + + +def _validate_predictions(task, dataset, rows): + for position, row in rows.iterrows(): + text = row['prediction'] + try: + if task == 'EventVerification': + if dataset._extract_answer(text) not in ('yes', 'no'): + raise ValueError('Unparseable event prediction would be silently skipped') + elif task == 'DVC': + events = dataset.parse_events_from_json(text) + if not events: + raise ValueError('DVC prediction has no scorable events') + for event in events: + if not (('start' in event or 'start_time' in event) + and ('end' in event or 'end_time' in event)): + raise ValueError('DVC event is missing timestamps') + start = dataset.parse_timestamp(event.get('start') or event.get('start_time', '0')) + end = dataset.parse_timestamp(event.get('end') or event.get('end_time', '0')) + if (not math.isfinite(start) or not math.isfinite(end) or end <= start + or not (event.get('caption') or event.get('description') or '').strip()): + raise ValueError('DVC contains an invalid or skipped event') + elif task == 'Temporal': + # Check the same branch native evaluate uses, but disallow its + # silent [0,duration] fallback for malformed responses. + try: + stamps = dataset.parse_timestamps_json(text, row['duration'], strict=True) + except Exception: + stamps = dataset.parse_timestamps(text, row['duration'], strict=True) + # Native JSON's permissive fallback returns a full interval + # rather than trying angle timestamps. Reject that mismatch. + try: + native_stamps = dataset.parse_timestamps_json(text, row['duration']) + except Exception: + native_stamps = stamps + if native_stamps != stamps: + raise ValueError('Temporal prediction would trigger a silent full-duration fallback') + if not all(math.isfinite(value) for value in stamps) or stamps[1] <= stamps[0]: + raise ValueError('Invalid temporal prediction interval') + elif task == 'SOT': + from .dataset.vantage_sot import parse_sot_response + parsed = parse_sot_response(text, int(row['n_frames']), model_family='cr') + if not parsed and text.strip() != '{}': + raise ValueError('Unparseable SOT prediction') + # MCQ, grounding and detection keep malformed nonempty outputs in + # their native denominators (zero / empty detection), not skip them. + except Exception: + raise ValueError(f'{task} prediction at row {position} cannot be scored without silent fallback') from None + + +def _evaluate(task, dataset, rows, output, bundle): + path = output / f'predictions_VANTAGE_{task}.tsv' + rows.to_csv(path, sep='\t', index=False) + with _private_native_output(): + if task == 'Astro2D': + # Invoke children directly: ConcatDataset replaces dataset names in + # the entire path, which can corrupt user-chosen output directories. + metrics = {} + present = [] + for child in dataset.datasets: + subset = rows[rows['SUB_DATASET'] == child].copy() + if subset.empty: + continue + subset['index'] = subset.pop('original_index') + subset = subset.drop(columns=['SUB_DATASET']) + child_path = output / f'{child}.tsv' + subset.to_csv(child_path, sep='\t', index=False) + result = dataset.dataset_map[child].evaluate(str(child_path), verbose=False) + if result.get('error') or result.get('total_predictions') != len(subset): + raise ValueError('ASTRO native scorer omitted examples') + metrics.update({f'{child}:{key}': value for key, value in result.items()}) + present.append(child) + for key in ('f1', 'precision', 'recall', 'f1_0.95', 'f1_mIOU'): + metrics[key] = sum(metrics[f'{child}:{key}'] for child in present) / len(present) + else: + kwargs = {'verbose': False} + if task == 'DVC': + kwargs.update(local_scoring=True, bert_score_model=bundle['bert_score_model']) + metrics = dataset.evaluate(str(path), **kwargs) + if isinstance(metrics, pd.DataFrame): + if len(metrics) != 1: + raise ValueError('Unexpected native metric table') + metrics = metrics.iloc[0].to_dict() + if not isinstance(metrics, dict) or metrics.get('error'): + raise ValueError('Native scoring failed') + if task == 'EventVerification' and metrics.get('Valid Predictions') != len(rows): + raise ValueError('Native event scorer silently omitted predictions') + if task == 'Temporal' and metrics.get('overall', {}).get('count') != len(rows): + raise ValueError('Native temporal scorer silently omitted predictions') + if task == 'DVC' and metrics.get('overall', {}).get('count') != len(rows): + raise ValueError('Native DVC scorer silently omitted predictions') + if task == '2DGrounding' and metrics.get('total_samples') != len(rows): + raise ValueError('Native grounding scorer silently omitted predictions') + if task == 'SOT' and len(metrics) != len(rows) + 1: + raise ValueError('Native tracking scorer silently omitted predictions') + return metrics, path + + +def _primary(task, metrics): + name, unit, aggregation = { + 'VQA': ('accuracy', 'fraction', 'question_micro'), + 'EventVerification': ('macro_f1', 'fraction', 'class_macro'), + 'Temporal': ('iou', 'fraction', 'video_macro_of_question_means'), + 'DVC': ('SODA_c', 'fraction', 'video_macro'), + '2DGrounding': ('Acc@0.5', 'percent', 'expression_micro'), + '2DPointing': ('accuracy', 'fraction', 'question_micro'), + 'SOT': ('success_auc', 'fraction', 'sequence_macro'), + 'Astro2D': ('f1', 'percent', 'unweighted_sequence_macro'), + }[task] + if task in ('VQA', '2DPointing', 'SOT'): + value = metrics['Overall'] + elif task in ('Temporal', 'DVC'): + value = metrics['overall'][name] + else: + value = metrics[name] + if not isinstance(value, numbers.Real) or not math.isfinite(value): + raise ValueError('Native primary metric is not finite') + return {'name': name, 'value': float(value), 'unit': unit, 'aggregation': aggregation} + + +def _json_value(value): + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + if hasattr(value, 'item'): + return _json_value(value.item()) + if isinstance(value, float) and not math.isfinite(value): + raise ValueError('Native metrics contain a nonfinite value') + return value + + +def score_predictions(task, predictions, assets_dir, output_dir, allow_subset=False): + """Score native predictions locally; write private artifacts and summary.json. + + Stable identity columns from the native inference table must be retained. + A subset is never labeled full coverage, even when explicitly permitted. + Existing output directories are rejected rather than reused or overwritten. + """ + task = _task(task) + output = Path(output_dir).expanduser().resolve() + if output.exists(): + raise ValueError('Local scoring requires a fresh output directory') + bundle = load_assets(assets_dir, [task]) + predictions, prediction_sha256 = _read_predictions(predictions) + dataset = _build_dataset(bundle, task) + rows = _trusted_predictions(task, dataset, predictions, allow_subset) + _validate_predictions(task, dataset, rows) + output.mkdir(mode=0o700, parents=True, exist_ok=False) + try: + metrics, scoring_table = _evaluate(task, dataset, rows, output, bundle) + source_paths = {Path(__file__).resolve(), Path(inspect.getfile(type(dataset))).resolve(), + Path(__file__).with_name('scoring_assets.py').resolve(), + _CONFIG_DIR / f'VANTAGE_{task}.json'} + if task == 'Astro2D': + source_paths.update(Path(inspect.getfile(type(child))).resolve() + for child in dataset.dataset_map.values()) + if task in ('Astro2D', '2DGrounding'): + from .dataset.metropolis2d import utils as geometry + source_paths.add(Path(inspect.getfile(geometry)).resolve()) + provenance = { + 'asset_id': bundle['manifest']['asset_id'], 'manifest_sha256': bundle['manifest_sha256'], + 'asset_files': bundle['manifest']['files'], 'predictions_sha256': prediction_sha256, + 'scoring_table_sha256': _sha256(scoring_table), + 'source_files': {str(path.relative_to(Path(__file__).resolve().parents[1])): _sha256(path) + for path in sorted(source_paths)}, + } + if task == 'DVC': + provenance['metric_model'] = bundle['manifest']['metric_models']['roberta_large'] + summary = _json_value({ + 'format_version': 1, 'task': task, 'status': 'complete', + 'coverage': {'expected_rows': len(dataset.data), 'scored_rows': len(rows), + 'subset': len(rows) != len(dataset.data)}, + 'primary_metric': _primary(task, metrics), 'native_metrics': metrics, + 'annotation_diagnostics': bundle['manifest']['tasks'][task].get('annotation_diagnostics', {}), + 'provenance': provenance, + }) + temporary = output / 'summary.json.tmp' + temporary.write_text(json.dumps(summary, indent=2, allow_nan=False) + '\n') + temporary.replace(output / 'summary.json') + return summary + except Exception: + (output / 'summary.json.tmp').unlink(missing_ok=True) + (output / 'scoring-error.log').write_text(traceback.format_exc()) + raise ValueError('Local scoring failed; inspect private output/scoring-error.log') from None + + +def main(argv=None): + # CLI subprocesses can inherit Jupyter's inline backend. Leave importing + # callers' kernels untouched, but always use a headless backend here. + os.environ['MPLBACKEND'] = 'Agg' + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--task', required=True) + parser.add_argument('--predictions', required=True, type=Path) + parser.add_argument('--assets-dir', required=True, type=Path) + parser.add_argument('--output-dir', required=True, type=Path) + parser.add_argument('--allow-subset', action='store_true') + args = parser.parse_args(argv) + try: + summary = score_predictions(args.task, args.predictions, args.assets_dir, + args.output_dir, args.allow_subset) + except Exception: + parser.exit(2, 'Local scoring failed: check local assets, native identities, coverage, and private output/scoring-error.log if created.\n') + print(json.dumps({'task': summary['task'], 'coverage': summary['coverage'], + 'primary_metric': summary['primary_metric'], + 'annotation_diagnostics': summary['annotation_diagnostics']})) + + +if __name__ == '__main__': + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py new file mode 100644 index 000000000..f76321dfd --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py @@ -0,0 +1,630 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Prepare authorized, private VANTAGE sources in place without network access. + +Run ``python -m vlmeval.scoring_assets --assets-dir ROOT --asset-id REV``. +DVC additionally requires --bert-score-model RELDIR --bert-score-revision REV. +Only canonical tables and scoring-assets.json are written; source labels and media +are never changed or copied. This does not grant redistribution permission. +""" + +import argparse +import ast +import hashlib +import json +import math +import re +import subprocess +from pathlib import Path + +import pandas as pd + +TASKS = ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DGrounding', '2DPointing', 'SOT', 'Astro2D') +VIDEO_TASKS = TASKS[:4] +SOURCE_DIRS = { + 'VQA': 'VQA', + 'EventVerification': 'event_verification_subset/filtered', + 'Temporal': 'Temporal', + 'DVC': 'Dense Video Caption', + '2DGrounding': 'Spatial/2d_referring_expressions/refdrone', + '2DPointing': 'datasets/Metropolis2DPointing', + 'SOT': 'tracking/MetropolisSOT_benchmark_8f', + 'Astro2D': '2dbbox', +} +VQA_FILES = ( + 'metrics_spatial_wo_ss.json', 'Metropolis_VQA_Verification_Final_ITS_Data.json', + 'VANTAGE_VQA_Verification_Final_ITS_Data.json', 'metrics_spatial_ss.json', + 'metrics_temporal_filtered_ss.json', 'metrics_temporal_wo_ss.json', +) +EVENT_FILES = ( + 'metropolis_event_verification/test_annotation.json', + 'tailgating/tailgating_building_r/test_annotation.json', + 'tailgating/tailgating_courtyard/test_annotation.json', + 'warehouse_near_miss/test_annotations.json', +) + + +def _require(condition, message): + if not condition: + raise ValueError(message) + + +def _text(value): + return isinstance(value, str) and bool(value.strip()) + + +def _vqa_option(value): + option = value.strip().upper() if isinstance(value, str) else '' + _require(option in tuple('ABCD'), 'VQA source is missing a valid ground-truth option') + return option + + +def _local_path(root, path, directory=False): + root = Path(root).resolve(strict=True) + path = Path(path) + resolved = (root / path).resolve(strict=True) if not path.is_absolute() else path.resolve(strict=True) + _require(resolved.is_relative_to(root), 'Asset path escapes the local bundle') + _require(resolved.is_dir() if directory else resolved.is_file(), 'Required local asset has the wrong file type') + return resolved + + +def _json(path): + with Path(path).open(encoding='utf-8') as stream: + return json.load(stream) + + +def _records(path, key=None): + raw = _json(path) + if key and isinstance(raw, dict): + raw = raw.get(key) + _require(isinstance(raw, list) and len(raw) > 0, 'Required annotation must contain a nonempty record list') + _require(all(isinstance(row, dict) for row in raw), 'Annotation contains a non-object record') + return raw + + +def _number(value): + _require(not isinstance(value, bool), 'Boolean is not a valid numeric label') + try: + result = float(value) + except (TypeError, ValueError) as exc: + raise ValueError('Missing or malformed numeric ground-truth label') from exc + _require(math.isfinite(result), 'Label contains a nonfinite number') + return result + + +def _timestamp(value): + _require(value is not None and str(value).strip() != '', 'Missing timestamp label') + parts = str(value).strip().split(':') + _require(1 <= len(parts) <= 3, 'Malformed timestamp label') + seconds = 0.0 + for part in parts: + seconds = seconds * 60 + _number(part) + _require(seconds >= 0, 'Negative timestamp label') + return seconds + + +def _span(start, end): + start, end = _timestamp(start), _timestamp(end) + _require(end > start, 'Label interval must have positive duration') + return start, end + + +def _bbox(box): + _require(isinstance(box, (list, tuple)) and len(box) == 4, 'Missing or malformed bounding-box label') + x1, y1, x2, y2 = map(_number, box) + _require(x2 > x1 and y2 > y1, 'Bounding-box label must have positive area') + + +def _events(events): + _require(isinstance(events, list) and len(events) > 0, 'DVC requires a nonempty ground-truth event list') + for event in events: + _require(isinstance(event, dict), 'Malformed DVC ground-truth event') + start = event.get('start', event.get('start_time')) + end = event.get('end', event.get('end_time')) + _span(start, end) + caption = event.get('caption') or event.get('event_caption') or event.get('description') + _require(_text(caption), 'DVC event is missing its ground-truth caption') + + +def _validate_table(frame, task): + required = {'index', 'video', 'question', 'answer'} + _require(required.issubset(frame.columns) and len(frame) > 0, f'{task} table is empty or missing required columns') + for field in required: + _require(frame[field].notna().all() and frame[field].map(lambda value: str(value).strip() != '').all(), + f'{task} table contains a missing {field}') + _require(not frame['index'].map(str).duplicated().any(), f'{task} table contains duplicate indexes') + if task == 'VQA': + _require(frame['answer'].isin(list('ABCD')).all(), 'VQA requires explicit A-D ground-truth answers') + elif task == 'EventVerification': + _require(frame['answer'].map(lambda value: str(value).strip().lower() in ('yes', 'no')).all(), + 'EventVerification requires explicit yes/no ground-truth answers') + elif task == 'Temporal': + _require('duration' in frame, 'Temporal table requires real video duration metadata') + for _, row in frame.iterrows(): + answer = json.loads(row['answer']) + _span(answer.get('start'), answer.get('end')) + _require(_number(row['duration']) > 0, 'Temporal video duration must be positive') + elif task == 'DVC': + _require(not frame['video'].duplicated().any(), 'DVC requires one table row per video') + for answer in frame['answer']: + _events(json.loads(answer)) + else: + raise ValueError(f'Unknown local video task: {task}') + + +def validate_local_video_inputs(local_data_file, local_video_root, task): + """Validate a canonical labeled table before a native constructor loads it. + + No cache or remote source is consulted. Explicit paths must both exist; local + media references are relative to the supplied root and may not escape it. + """ + _require(local_data_file is not None and local_video_root is not None, + 'local_data_file and local_video_root must be supplied together') + data_file = Path(local_data_file).expanduser().resolve(strict=True) + root = Path(local_video_root).expanduser().resolve(strict=True) + _require(root.is_dir() and data_file.is_file(), 'Local video inputs must be a table and a directory') + _require(data_file.suffix.lower() == '.tsv', 'Canonical local video data must be a TSV table') + frame = pd.read_csv(data_file, sep='\t', keep_default_na=False) + _validate_table(frame, task) + for video in frame['video'].unique(): + media = str(video) + if task != 'EventVerification' and not media.endswith('.mp4'): + media += '.mp4' + _require(not Path(media).is_absolute(), 'Local video table must contain relative media identities') + _local_path(root, media) + return {'data_file': str(data_file), 'root': str(root)} + + +def validate_bert_score_model(model_dir): + """Require a complete local roberta-large checkpoint, never a Hub model ID.""" + _require(model_dir is not None, 'Local DVC scoring requires bert_score_model pointing to local roberta-large assets') + path = Path(model_dir).expanduser().resolve(strict=True) + _require(path.is_dir(), 'BERTScore model must be a local directory') + config = _json(path / 'config.json') + _require(config.get('model_type') == 'roberta' and config.get('num_hidden_layers') == 24, + 'Local DVC requires the roberta-large architecture') + _local_path(path, 'vocab.json') + _local_path(path, 'merges.txt') + weights = next((path / name for name in ('model.safetensors', 'pytorch_model.bin') if (path / name).is_file()), None) + if weights is None: + index = next((path / name for name in ('model.safetensors.index.json', 'pytorch_model.bin.index.json') + if (path / name).is_file()), None) + _require(index is not None, 'Local BERTScore model weights are missing') + weight_map = _json(index).get('weight_map') + _require(isinstance(weight_map, dict) and weight_map, 'Malformed local model weight index') + for shard in set(weight_map.values()): + _local_path(path, shard) + else: + _local_path(path, weights) + return path + + +def _annotation_paths(root, task): + source = _local_path(root, SOURCE_DIRS[task], directory=True) + if task in ('VQA', 'Temporal'): + annotations = _local_path(root, source / 'data_jsons/annotations', directory=True) + if task == 'VQA': + paths = [annotations / name for name in VQA_FILES if (annotations / name).is_file()] + required = set(VQA_FILES) - {'Metropolis_VQA_Verification_Final_ITS_Data.json', 'VANTAGE_VQA_Verification_Final_ITS_Data.json'} + _require(required.issubset({p.name for p in paths}) and len(paths) == 5, + 'VQA requires four metrics files and exactly one Verification ITS annotation') + else: + paths = sorted(annotations.glob('*.json')) + elif task == 'EventVerification': + paths = [source / name for name in EVENT_FILES] + elif task == 'DVC': + paths = sorted((source / 'events').glob('*_events.json')) + elif task == '2DGrounding': + paths = [source / 'refdrone_test_llava.json'] + elif task == '2DPointing': + paths = [source / 'Metropolis2DPointing.tsv'] + elif task == 'SOT': + directories = [p for p in sorted(source.iterdir()) if p.is_dir()] + _require(bool(directories), 'SOT has no sequence directories') + paths = [p / 'gt.json' for p in directories] + else: + from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset + paths = [] + for layout in Astro2DDetectionDataset._LAYOUT.values(): + sequence = source / layout['data_root'].rsplit('/', 1)[1] + labels = _local_path(root, sequence / 'labels', directory=True) + paths.extend(sorted(labels.glob('*.txt'))) + _require(bool(paths), f'{task} has no original ground-truth annotations') + return [_local_path(root, path) for path in paths] + + +def validate_task_sources(assets_dir, task, dataset_kwargs=None): + """Validate original labels and media; return required annotation fingerprints. + + The original filtered source layout is authoritative, not a prediction table. + Supplied manifest constructor paths must resolve to these same original + sources, so an alternate bundle-local label tree cannot evade validation. + """ + _require(task in TASKS, 'Unknown scoring asset task') + root = Path(assets_dir).resolve(strict=True) + source = _local_path(root, SOURCE_DIRS[task], directory=True) + paths = _annotation_paths(root, task) + if dataset_kwargs is not None: + expected = {} + if task in VIDEO_TASKS: + expected['local_video_root'] = source if task == 'EventVerification' else source / 'videos' + _local_path(root, dataset_kwargs.get('local_data_file', '')) + elif task == '2DGrounding': + expected = {'data_root': source, 'annotation_file': paths[0]} + elif task == '2DPointing': + expected = {'data_root': source} + elif task == 'SOT': + expected = {'prepared_data_dir': source} + else: + from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset + roots = dataset_kwargs.get('data_roots') + _require(isinstance(roots, dict) and set(roots) == set(Astro2DDetectionDataset._LAYOUT), + 'ASTRO manifest must identify exactly the registered original child sequences') + for name, layout in Astro2DDetectionDataset._LAYOUT.items(): + actual = _local_path(root, roots[name], directory=True) + _require(actual == source / layout['data_root'].rsplit('/', 1)[1], + 'ASTRO manifest path does not identify its original sequence') + for name, original in expected.items(): + _require(name in dataset_kwargs, f'{task} manifest is missing {name}') + actual = _local_path(root, dataset_kwargs[name], directory=original.is_dir()) + _require(actual == original, f'{task} manifest path does not identify its original source') + if task in ('VQA', 'Temporal'): + for path in paths: + for item in _records(path): + video = item.get('q_uid', item.get('vid')) if task == 'VQA' else item.get('vid') + _require(_text(video) and _text(item.get('question')), f'{task} annotation lacks video ID or question') + video = video.removesuffix('.json') if task == 'VQA' else video + filename = video if video.endswith('.mp4') else video + '.mp4' + _local_path(root, source / 'videos' / filename) + if task == 'VQA': + _vqa_option(item.get('gt_option')) + options = item.get('options') + _require(isinstance(options, list) and 2 <= len(options) <= 4 and all(_text(v) for v in options), + 'VQA source requires two to four explicit choices') + _require(_vqa_option(item.get('gt_option')) in tuple('ABCD')[:len(options)], + 'VQA source ground truth does not identify an available choice') + else: + answer = item.get('answer') + _require(isinstance(answer, str), 'Temporal source is missing ground-truth timestamps') + match = re.match(r'<([\d.]+)>\s*<([\d.]+)>', answer) + _require(match is not None, 'Malformed Temporal source timestamps; no default interval is permitted') + _span(*match.groups()) + elif task == 'EventVerification': + from .dataset.metropolis_event_verification import MetropolisEventVerification + for path in paths: + for item in _records(path, 'bcq'): + _require(item.get('id') is not None and str(item['id']).strip() != '', 'Event source lacks original ID') + _require(_text(item.get('question')), 'Event source lacks question') + _require(str(item.get('answer', '')).strip().lower() in ('yes', 'no'), 'Event source lacks yes/no ground truth') + video = item.get('video', item.get('video_id')) + _require(_text(video), 'Event source lacks video ID') + _local_path(root, path.parent / video) + items = MetropolisEventVerification._load_items(source) + identities = [(item.get('source_corpus', ''), str(item['id'])) for item in items] + _require(len(set(identities)) == len(identities), 'Duplicate original EventVerification IDs within a corpus') + elif task == 'DVC': + from .dataset.metropolis_dvc import MetropolisDVC + adapter = MetropolisDVC(test_mode=True) + resolved = [] + for path in paths: + raw = _json(path) + _events(raw.get('events') if isinstance(raw, dict) else raw) + video_id = path.name.removesuffix('_events.json') + videos = source / 'videos' + exact = videos / (video_id + '.mp4') + with_ext = videos / video_id + if not exact.is_file() and not (video_id.endswith('.mp4') and with_ext.is_file()): + matches = list(videos.glob(video_id.replace('[', '[[]') + '*.mp4')) + _require(len(matches) == 1, 'DVC source video ID has missing or ambiguous media') + stem = adapter._resolve_video_stem(videos, video_id) + _require(stem is not None, 'DVC source video cannot be resolved') + _local_path(root, videos / (stem + '.mp4')) + resolved.append(stem) + _require(len(set(resolved)) == len(resolved), 'DVC has multiple annotations for the same video') + elif task == '2DGrounding': + for item in _records(paths[0]): + _require(_text(item.get('media')), 'Grounding source lacks image identity') + _local_path(root, source / item['media']) + meta = item.get('_meta') or {} + _require(_text(meta.get('sentence')), 'Grounding source lacks referring expression') + _require(_number(meta.get('image_width')) > 0 and _number(meta.get('image_height')) > 0, + 'Grounding source lacks real image dimensions') + turns = [turn for turn in item.get('conversations', []) if turn.get('from') == 'gpt'] + _require(len(turns) == 1, 'Grounding source lacks an unambiguous ground-truth response') + boxes = json.loads(turns[0].get('value', '')) + _require(isinstance(boxes, list) and bool(boxes), 'Grounding source lacks bounding-box labels') + for box in boxes: + _bbox(box) + elif task == '2DPointing': + frame = pd.read_csv(paths[0], sep='\t', keep_default_na=False) + _require({'index', 'question', 'answer', 'target_point'}.issubset(frame.columns) and len(frame) > 0, + 'Pointing source lacks original IDs, answers, or target points') + _require(not frame['index'].map(str).duplicated().any(), 'Pointing source contains duplicate IDs') + _require(frame['answer'].isin(list('ABCD')).all(), 'Pointing source lacks A-D ground truth') + _require(frame['target_point'].map(lambda value: str(value).strip() != '').all(), 'Pointing source lacks target-point labels') + for value in frame['target_point']: + try: + point = ast.literal_eval(str(value)) + except (SyntaxError, ValueError) as exc: + raise ValueError('Malformed Pointing target-point ground truth') from exc + _require(isinstance(point, (list, tuple)) and len(point) == 2, 'Pointing target must have two coordinates') + for coordinate in point: + _number(coordinate) + from .smp import toliststr + _require('image_path' in frame, 'Pointing source lacks local image identities') + for _, row in frame.iterrows(): + images = toliststr(row['image_path']) + _require(len(images) == 1 and _text(images[0]), 'Pointing source requires one local image identity') + _local_path(root, source / images[0]) + elif task == 'SOT': + metadata = {} + labels = set() + metadata_path = (dataset_kwargs or {}).get('metadata_path') + if metadata_path: + metadata_path = _local_path(root, metadata_path) + for line in metadata_path.read_text(encoding='utf-8').splitlines(): + if line.strip(): + entry = json.loads(line) + _require(_text(entry.get('seq_id')) and entry['seq_id'] not in metadata, 'Malformed or duplicate SOT metadata identity') + metadata[entry['seq_id']] = entry + for path in paths: + meta = _json(path) + if 'frame_ids' in meta: + ids = meta['frame_ids'] + boxes = meta.get('gt_bboxes') + _require(_text(meta.get('label')), 'SOT source lacks original sequence label') + _require(isinstance(boxes, dict), 'SOT source lacks per-frame ground truth') + _require(isinstance(ids, list) and len(ids) >= 2 and all(type(i) is int for i in ids) + and len(set(ids)) == len(ids), 'Malformed SOT frame identities') + expected = {str(i) for i in ids[1:]} + _require(expected.issubset(boxes) and set(boxes).issubset({str(i) for i in ids}), + 'SOT ground truth must explicitly label every evaluation frame, including absent objects') + else: + count = meta.get('nframes') + _require(type(count) is int and count >= 2, 'Malformed compact SOT frame count') + ids = list(range(count)) + boxes = meta.get('bboxes') + _require(isinstance(boxes, dict), 'Compact SOT lacks frame labels') + expected = {f'frame_{i}' for i in ids[1:]} + _require(set(boxes) == expected, 'Compact SOT must explicitly label every evaluation frame') + entry = metadata.get(path.parent.name, {}) + init_bbox = meta.get('init_bbox') or entry.get('init_bbox') + _bbox(init_bbox) + if 'frame_ids' in meta and str(ids[0]) in boxes: + _require(boxes[str(ids[0])] == init_bbox, 'SOT initialization box disagrees with its frame ground truth') + label = meta.get('label') if 'frame_ids' in meta else entry.get('label') or path.parent.name.replace('__', '/') + _require(_text(label) and label not in labels, 'SOT sequence labels must be present and unique') + labels.add(label) + for box in boxes.values(): + if box is not None: + _bbox(box) + for index in range(len(ids)): + _local_path(root, path.parent / 'frames' / f'f{index:02d}.png') + _local_path(root, path.parent / 'frames/crop.png') + if metadata_path: + paths.append(metadata_path) + else: + from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset, parse_kitti_label + for layout in Astro2DDetectionDataset._LAYOUT.values(): + sequence = source / layout['data_root'].rsplit('/', 1)[1] + images_dir = _local_path(root, sequence / 'images', directory=True) + images = [p for p in sorted(images_dir.iterdir()) if p.suffix.lower() in ('.jpg', '.jpeg', '.png', '.bmp', '.tiff')] + _require(bool(images), 'ASTRO sequence has no images') + stems = [p.stem for p in images] + _require(len(set(stems)) == len(stems), 'ASTRO image identities have ambiguous label stems') + labels = {p.stem: p for p in paths if p.parent == sequence / 'labels'} + _require(set(stems) == set(labels), 'ASTRO requires an independent label file for every image, and no unmatched labels') + for image in images: + _local_path(root, image) + label = labels[image.stem] + lines = [line for line in label.read_text(encoding='utf-8').splitlines() if line.strip()] + for line in lines: + fields = line.split() + _require(len(fields) >= 15, 'Malformed KITTI ground-truth label') + # Preserve the native KITTI policy, including finite inverted + # boxes. The scorer clamps their area to zero; deleting or + # repairing them would change its ground-truth denominator. + for value in fields[4:8]: + _number(value) + _require(len(parse_kitti_label(str(label))) == len(lines), 'Native ASTRO parser skipped a malformed label') + return paths + + +def _duration(video): + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', str(video)], + check=True, capture_output=True, text=True, + ) + duration = _number(result.stdout.strip()) + _require(duration > 0, 'Video has no usable duration metadata') + return duration + + +def _sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def _snapshot(frame, root, target): + frame = frame.copy() + for column in ('image_path', 'video_path'): + if column in frame: + frame[column] = frame[column].map( + lambda value: str(Path(value).relative_to(root)) + if _text(value) and Path(value).is_absolute() and Path(value).is_relative_to(root) else value + ) + frame.to_csv(target, sep='\t', index=False) + + +def prepare_assets(assets_dir, asset_id, tasks=None, bert_score_model=None, bert_score_revision=None, + sot_metadata_path=None): + """Build native canonical tables and a fingerprinted local scoring manifest.""" + from .dataset import ( + MetropolisVQA, MetropolisTemporal, MetropolisEventVerification, MetropolisDVC, + Metropolis2DGroundingDataset, VANTAGE_2DPointing, VANTAGE_SOT, Astro2DDetectionDataset, + ) + root = Path(assets_dir).expanduser().resolve(strict=True) + _require(root.is_dir() and _text(asset_id), 'Preparation requires a local source directory and nonempty asset ID') + selected = list(tasks or TASKS) + _require(bool(selected) and len(set(selected)) == len(selected) and set(selected).issubset(TASKS), 'Invalid or duplicate preparation tasks') + manifest = {'format_version': 1, 'asset_id': asset_id, 'tasks': {}, 'files': {}} + if 'DVC' in selected: + _require(_text(bert_score_revision), 'DVC model requires an explicit immutable revision identifier') + _require(bert_score_model is not None and not Path(bert_score_model).is_absolute(), 'BERTScore model path must be relative to the bundle') + model = _local_path(root, bert_score_model, directory=True) + validate_bert_score_model(model) + model_files = {} + for path in sorted(model.rglob('*')): + if path.is_file() and '.cache' not in path.relative_to(model).parts: + _local_path(root, path) + model_files[path.relative_to(model).as_posix()] = _sha256(path) + manifest['metric_models'] = {'roberta_large': { + 'path': model.relative_to(root).as_posix(), 'revision': bert_score_revision, + 'files': model_files, + }} + # Validate every requested original source before writing generated tables. + sot_kwargs = {} + if sot_metadata_path: + _require(not Path(sot_metadata_path).is_absolute(), 'SOT metadata path must be relative to the bundle') + sot_kwargs['metadata_path'] = _local_path(root, sot_metadata_path).relative_to(root).as_posix() + originals = { + task: validate_task_sources(root, task, {'prepared_data_dir': SOURCE_DIRS[task], **sot_kwargs} if task == 'SOT' else None) + for task in selected + } + for task in selected: + source_diagnostics = {} + source = root / SOURCE_DIRS[task] + table = root / f'VANTAGE_{task}.tsv' + _require(not table.is_symlink() and (not table.exists() or table.stat().st_nlink == 1), + 'Refusing to overwrite a linked canonical output table') + kwargs = {} + if task in ('VQA', 'Temporal'): + adapter = (MetropolisVQA if task == 'VQA' else MetropolisTemporal)(test_mode=True) + for mapping_dir in (source / 'data_jsons/category_mapping', source / 'mappings'): + if mapping_dir.is_dir(): + for mapping in mapping_dir.glob('*.json'): + _local_path(root, mapping) + _require(isinstance(_json(mapping), dict), 'Malformed native category mapping') + adapter.category_mapping.update(adapter._load_category_mapping(str(mapping_dir))) + rows = [] + durations = {} + for path in originals[task]: + for item in _records(path): + record = {**item, 'gt_option': _vqa_option(item.get('gt_option'))} if task == 'VQA' else item + row = adapter._process_annotation_item(record) + _require(row is not None, f'Native {task} converter rejected a validated source record') + if task == 'Temporal' and 'duration' not in item: + filename = row['video'] if row['video'].endswith('.mp4') else row['video'] + '.mp4' + video = source / 'videos' / filename + if video not in durations: + durations[video] = _duration(video) + row['duration'] = durations[video] + rows.append(row) + rows.sort(key=lambda row: (row['video'], row.get('qid', ''))) + for index, row in enumerate(rows): + row['index'] = index + frame = pd.DataFrame(rows) + elif task == 'EventVerification': + frame = pd.DataFrame(MetropolisEventVerification._load_items(source)) + if 'index' not in frame: + frame['index'] = range(len(frame)) + frame['index'] = frame['index'].astype(str) + elif task == 'DVC': + adapter = MetropolisDVC(test_mode=True) + adapter._generate_tsv_from_events(source, table) + frame = pd.read_csv(table, sep='\t', keep_default_na=False) + _require(len(frame) == len(originals[task]), 'Native DVC conversion dropped original annotations') + elif task == '2DGrounding': + kwargs = {'data_root': SOURCE_DIRS[task], 'annotation_file': originals[task][0].relative_to(root).as_posix()} + dataset = Metropolis2DGroundingDataset(dataset='VANTAGE_2DGrounding', model_family='cr', + data_root=str(source), annotation_file=str(originals[task][0])) + frame = dataset.data + _require(len(frame) == len(_records(originals[task][0])), 'Native Grounding loader dropped original annotations') + elif task == '2DPointing': + kwargs = {'data_root': SOURCE_DIRS[task]} + dataset = VANTAGE_2DPointing(data_root=str(source), model_family='cr') + frame = dataset.data + _require(len(frame) == len(pd.read_csv(originals[task][0], sep='\t')), 'Native Pointing loader dropped original annotations') + elif task == 'SOT': + kwargs = {'prepared_data_dir': SOURCE_DIRS[task], **sot_kwargs} + native_kwargs = {'prepared_data_dir': str(source), 'model_family': 'cr'} + if sot_metadata_path: + native_kwargs['metadata_path'] = str(_local_path(root, sot_metadata_path)) + frame = VANTAGE_SOT(**native_kwargs).data + _require(len(frame) == sum(path.name == 'gt.json' for path in originals[task]), + 'Native SOT loader dropped original sequences') + else: + from .dataset.metropolis2d.astro_2d_dataset import compute_bbox_area + roots = {} + frames = [] + offset = 0 + for name, layout in Astro2DDetectionDataset._LAYOUT.items(): + sequence = source / layout['data_root'].rsplit('/', 1)[1] + roots[name] = sequence.relative_to(root).as_posix() + child = Astro2DDetectionDataset(dataset=name, data_root=str(sequence), model_family='cr') + non_positive = sum( + obj['label'] == 'person' and compute_bbox_area(obj['bbox']) == 0 + for objects in child._gt_cache.values() for obj in objects + ) + if non_positive: + source_diagnostics[name] = { + 'non_positive_area_person_ground_truth': non_positive, + 'retained_by_native_min_area': non_positive if child.min_bbox_area <= 0 else 0, + 'native_min_bbox_area': child.min_bbox_area, + 'policy': 'Original labels unchanged; native filtering and denominators preserved', + } + child_frame = child.data.copy() + child_frame['original_index'] = child_frame['index'] + child_frame['SUB_DATASET'] = name + child_frame['index'] = range(offset, offset + len(child_frame)) + offset += len(child_frame) + frames.append(child_frame) + frame = pd.concat(frames, ignore_index=True) + kwargs = {'data_roots': roots} + if task in VIDEO_TASKS: + _validate_table(frame, task) + frame.to_csv(table, sep='\t', index=False) + video_root = source if task == 'EventVerification' else source / 'videos' + validate_local_video_inputs(table, video_root, task) + kwargs = {'local_data_file': table.name, 'local_video_root': video_root.relative_to(root).as_posix()} + else: + _snapshot(frame, root, table) + manifest['tasks'][task] = {'rows': len(frame), 'dataset_kwargs': kwargs} + if source_diagnostics: + manifest['tasks'][task]['annotation_diagnostics'] = source_diagnostics + for path in originals[task] + [table]: + manifest['files'][path.relative_to(root).as_posix()] = _sha256(path) + # Category mappings influence native category-level metrics and must also + # be fingerprinted when used by the video converters. + if task in ('VQA', 'Temporal'): + for mapping_dir in (source / 'data_jsons/category_mapping', source / 'mappings'): + if mapping_dir.is_dir(): + for path in sorted(mapping_dir.glob('*.json')): + path = _local_path(root, path) + manifest['files'][path.relative_to(root).as_posix()] = _sha256(path) + output = root / 'scoring-assets.json' + _require(not output.is_symlink() and (not output.exists() or output.stat().st_nlink == 1), + 'Refusing to overwrite a linked scoring manifest') + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n', encoding='utf-8') + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--assets-dir', required=True) + parser.add_argument('--asset-id', required=True) + parser.add_argument('--tasks', nargs='+', choices=TASKS) + parser.add_argument('--bert-score-model', help='Relative local roberta-large model directory inside the bundle') + parser.add_argument('--bert-score-revision', help='Immutable source revision of the local model assets') + parser.add_argument('--sot-metadata-path', help='Optional relative original SOT metadata JSONL inside the bundle') + args = parser.parse_args(argv) + manifest = prepare_assets(args.assets_dir, args.asset_id, args.tasks, args.bert_score_model, + args.bert_score_revision, args.sot_metadata_path) + print(json.dumps({'asset_id': manifest['asset_id'], 'rows': {task: item['rows'] for task, item in manifest['tasks'].items()}}, sort_keys=True)) + + +if __name__ == '__main__': + main() From 3039a35232bb3c405624f00c9bd5e303938721ab Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Wed, 28 Oct 2026 11:50:30 -0500 Subject: [PATCH 2/6] Fix native interval scoring and bound evaluation workers Signed-off-by: Jon Patterson --- .../reasoner/vlmevalkit/launchable/README.md | 12 ++++-- .../vlmevalkit/launchable/launchable.py | 13 ++++--- .../vlmevalkit/launchable/test_launchable.py | 10 +++++ .../launchable/test_local_scoring.py | 39 ++++++++++++++++++- .../vlmevalkit/vlmeval/local_scoring.py | 7 ++-- 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 91fdba159..5aab9e6c8 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -69,11 +69,11 @@ Serving uses `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921 ### Validation status and limits -- The runtime code passed 20 native-scoring/artifact regressions in the packaged evaluator on native Linux x86-64. The committed fixtures are synthetic and use temporary directories; no benchmark labels, media, or executed notebook outputs are included. -- A private five-question Brev notebook smoke exercised Nano inference and native local scoring. Saved predictions were also rescored in a CPU evaluator with networking disabled. This demonstrates the subset workflow, not a full benchmark or published-score reproduction. -- DVC's genuine pinned local metric model was exercised offline with controlled captions, not model-generated benchmark answers. No DVC model-quality claim is made. +- The runtime code passed 23 native-scoring/artifact regressions on native Linux x86-64. Fixtures are synthetic and use temporary directories; no benchmark labels, media, or executed notebook outputs are included. +- A private five-question Brev notebook smoke exercised Nano inference and native local scoring. Saved predictions were also rescored in a CPU evaluator with networking disabled. +- Private full-coverage inference and native local scoring have now been exercised for all eight tasks with manually staged authorized assets. DVC used its genuine pinned local metric model offline on saved Nano outputs. This is recovered-run evidence, not a fresh one-click launch or published-score reproduction; detailed results remain private. - Preparation preserves original labels, native filtering, and ground-truth denominators. Annotation anomalies are recorded in private local diagnostics rather than repaired or silently omitted. -- Full eight-task model evaluation, immutable one-click cold launch, end-to-end BYOM, and named reference-score reproduction remain unverified. Scoring assets, detailed operational evidence, and benchmark results must be supplied and retained separately. +- Immutable one-click cold launch, end-to-end BYOM, and named reference-score reproduction remain unverified. Scoring assets, detailed operational evidence, and benchmark results must be supplied and retained separately. ## Run and score @@ -103,6 +103,8 @@ All output directories must be fresh. Direct scoring requires full task coverage `--mode all` always performs native inference first, then the same local scorer used for existing outputs. It never substitutes inference-only behavior or runs a remote scoring service. `--dry-run` composes configuration but does not prove inference or media compatibility. +The controller defaults to **one concurrent API/video worker** and records `api_nproc` in `run.json`. Video decoding consumes host RAM independently of GPU VRAM; increase `--api-nproc N` only after measuring available host memory. This changes scheduling, not model generation or metric parameters. + ## Metric semantics and provenance | Task | Native primary metric | Unit / aggregation | @@ -118,6 +120,8 @@ All output directories must be fresh. Direct scoring requires full task coverage Each private `summary.json` includes native metrics, explicit units/aggregation, expected/scored row counts, asset ID and annotation hashes, prediction snapshot hash, scorer-source hashes, and DVC metric-model fingerprints. There is **no invented cross-task overall VANTAGE score** or unverified leaderboard-parity claim. DVC requires a real local model; remote endpoints and dummy semantic scores are not allowed in local scoring. Missing or silently skipped examples fail scoring. Native failures leave a private `scoring-error.log`, not a success summary. +Finite zero-duration and reversed predicted intervals follow each native parser and IoU implementation. The wrapper does not drop them, repair timestamps, or replace them with full-duration answers. Unparseable responses, nonfinite timestamps, missing events/captions, or skipped rows still fail rather than producing a misleading score. + `run.json` records controller state and the exact scoring-asset manifest hash. Generated native configuration and `/opt/evaluation-packages.txt` identify runtime inputs. A BYOM endpoint's revision cannot be inferred from its display name; record its deployment revision separately. Treat predictions, native diagnostics, and summaries as private until reviewed. Nothing is uploaded automatically. ## BYOM, stop, and restart diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py index cc52c2cdf..d4fb4dfe4 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -125,7 +125,9 @@ def _write_state(work, state): (work / "run.json").write_text(json.dumps(state, indent=2) + "\n") -def run(selected, work, mode, smoke, dry_run, assets_dir): +def run(selected, work, mode, smoke, dry_run, assets_dir, api_nproc=1): + if api_nproc < 1: + raise ValueError("--api-nproc must be positive") if smoke and selected != ["VANTAGE_VQA"]: raise ValueError("--smoke requires --benchmarks VANTAGE_VQA") if work.exists(): @@ -146,11 +148,11 @@ def run(selected, work, mode, smoke, dry_run, assets_dir): "cosmos_commit": os.environ.get("COSMOS_COMMIT", "unknown"), "model": os.environ.get("COSMOS_MODEL", "${COSMOS_MODEL}"), "benchmarks": selected, "mode": mode, "smoke": smoke, "dry_run": dry_run, + "api_nproc": api_nproc, "scoring_assets": {"asset_id": bundle["manifest"]["asset_id"], "manifest_sha256": bundle["manifest_sha256"]}, "work_dirs": {}, "predictions": {}, "scores": {}} _write_state(work, state) - entries = manifest() try: for index, bench in enumerate(selected): config_path = configs / f"{bench}.json" @@ -164,10 +166,9 @@ def run(selected, work, mode, smoke, dry_run, assets_dir): config_path.write_text(json.dumps(cfg, indent=2) + "\n") out = work / f"bench_{index:02d}" state["work_dirs"][bench] = out.name - flags = entries[f"metropolis/{bench}"]["run"] command = [sys.executable, str(ROOT / "run.py"), "--config", str(config_path), "--mode", "infer", "--work-dir", str(out), - "--api-nproc", str(flags["api_nproc"]), "--verbose"] + "--api-nproc", str(api_nproc), "--verbose"] if dry_run: print(" ".join(command)) continue @@ -262,6 +263,8 @@ def main(): for child in (prep, runner): child.add_argument("--benchmarks", default="VANTAGE_VQA") runner.add_argument("--mode", choices=["infer", "all"], default="all") + runner.add_argument("--api-nproc", type=int, default=1, + help="Concurrent API/video workers; increase only for available host RAM") runner.add_argument("--smoke", action="store_true", help="Five seeded VQA samples, not a full benchmark") runner.add_argument("--dry-run", action="store_true") runner.add_argument("--work-dir", type=Path, required=True) @@ -276,7 +279,7 @@ def main(): prepare(benchmarks(args.benchmarks), args.assets_dir.resolve()) elif args.command == "run": run(benchmarks(args.benchmarks), args.work_dir.resolve(), args.mode, - args.smoke, args.dry_run, args.assets_dir.resolve()) + args.smoke, args.dry_run, args.assets_dir.resolve(), args.api_nproc) elif args.command == "score": score(args.work_dir.resolve(), args.assets_dir.resolve(), args.score_dir) else: diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py index 89e0e233f..ebc19c161 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -16,6 +16,16 @@ def test_existing_output_is_rejected_before_network_or_overwrite(self): with self.assertRaisesRegex(ValueError, "fresh --work-dir"): launchable.run(["VANTAGE_VQA"], Path(temp), "infer", True, False, Path(temp) / "assets") + def test_nonpositive_worker_count_is_rejected_before_assets_or_output(self): + for count in (0, -1): + with self.subTest(count=count), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + output = root / "fresh" + with self.assertRaisesRegex(ValueError, "positive"): + launchable.run(["VANTAGE_VQA"], output, "infer", False, False, + root / "missing-assets", api_nproc=count) + self.assertFalse(output.exists()) + def test_predictions_reject_empty_and_failure_markers(self): for value in (" ", "VIDEO_LOAD_FAILED", "prefix VIDEO_LOAD_FAILED", "Failed to obtain answer via API.", "null"): diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py index c69e4ff1e..4de0776f0 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py @@ -17,7 +17,7 @@ from PIL import Image from vlmeval.local_scoring import ( - build_local_dataset, dataset_config, load_assets, score_predictions, + _validate_predictions, build_local_dataset, dataset_config, load_assets, score_predictions, ) from vlmeval.scoring_assets import EVENT_FILES, SOURCE_DIRS, VQA_FILES, prepare_assets @@ -371,6 +371,43 @@ def test_asset_hash_and_escape_and_existing_output_are_rejected(self): with self.assertRaisesRegex(ValueError, 'escapes'): load_assets(self.assets, ['2DGrounding']) + def test_temporal_zero_interval_reduces_score_without_dropping_row(self): + _temporal(self.assets) + _prepare(self.assets, ['Temporal']) + dataset = build_local_dataset(self.assets, 'Temporal') + values = [ + json.dumps({'start': '0', 'end': '0'} if position == 0 else {'start': '1', 'end': '3'}) + for position, (_, row) in enumerate(dataset.data.iterrows()) + ] + summary = self.score('Temporal', self.predictions('Temporal', values)) + self.assertEqual(summary['primary_metric']['value'], 0.75) + self.assertEqual(summary['native_metrics']['overall']['count'], 3) + self.assertEqual(summary['coverage'], {'expected_rows': 3, 'scored_rows': 3, 'subset': False}) + + def test_dvc_degenerate_intervals_retain_native_zero_iou_without_fallback(self): + from vlmeval.dataset.metropolis_dvc import MetropolisDVC, _iou + + rows = pd.DataFrame({'prediction': [ + json.dumps([{'start': start, 'end': end, 'caption': 'A square moves.'}]) + for start, end in [('0', '0'), ('3', '1'), ('1', '3')] + ]}) + _validate_predictions('DVC', MetropolisDVC, rows) + overlaps = [] + for text in rows['prediction']: + event, = MetropolisDVC.parse_events_from_json(text) + interval = [MetropolisDVC.parse_timestamp(event[key]) for key in ('start', 'end')] + overlaps.append(_iou(interval, [1, 3])) + self.assertEqual(overlaps, [0.0, 0.0, 1.0]) + self.assertEqual(sum(overlaps) / len(overlaps), 1 / 3) + for event in ( + {'start': 'nan', 'end': '3', 'caption': 'A square moves.'}, + {'start': '1', 'caption': 'A square moves.'}, + {'start': '1', 'end': '3', 'caption': ''}, + ): + with self.subTest(event=event), self.assertRaisesRegex(ValueError, 'silent fallback'): + _validate_predictions('DVC', MetropolisDVC, + pd.DataFrame({'prediction': [json.dumps([event])]})) + def test_dvc_without_genuine_local_model_is_rejected(self): from vlmeval.scoring_assets import validate_bert_score_model empty_model = self.assets / 'model' diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py index 999fe7c70..e717739be 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py @@ -332,7 +332,8 @@ def _validate_predictions(task, dataset, rows): raise ValueError('DVC event is missing timestamps') start = dataset.parse_timestamp(event.get('start') or event.get('start_time', '0')) end = dataset.parse_timestamp(event.get('end') or event.get('end_time', '0')) - if (not math.isfinite(start) or not math.isfinite(end) or end <= start + # Native IoU retains finite zero/reversed spans as zero overlap. + if (not math.isfinite(start) or not math.isfinite(end) or not (event.get('caption') or event.get('description') or '').strip()): raise ValueError('DVC contains an invalid or skipped event') elif task == 'Temporal': @@ -350,8 +351,8 @@ def _validate_predictions(task, dataset, rows): native_stamps = stamps if native_stamps != stamps: raise ValueError('Temporal prediction would trigger a silent full-duration fallback') - if not all(math.isfinite(value) for value in stamps) or stamps[1] <= stamps[0]: - raise ValueError('Invalid temporal prediction interval') + if not all(math.isfinite(value) for value in stamps): + raise ValueError('Nonfinite temporal prediction interval') elif task == 'SOT': from .dataset.vantage_sot import parse_sot_response parsed = parse_sot_response(text, int(row['n_frames']), model_family='cr') From b0861ce448a648b2eb59c41a9358980bbb5df31c Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Thu, 29 Oct 2026 20:00:54 -0500 Subject: [PATCH 3/6] Replace Brev VANTAGE workflow with pinned MVBench evaluation Signed-off-by: Jon Patterson --- .../cosmos3/reasoner/vlmevalkit/.dockerignore | 5 + .../vlmevalkit/EXTERNAL_CONFIGURATION.md | 73 +- .../cosmos3/reasoner/vlmevalkit/README.md | 18 +- .../reasoner/vlmevalkit/launchable/Dockerfile | 2 +- .../launchable/Evaluate_Cosmos3.ipynb | 139 ++-- .../reasoner/vlmevalkit/launchable/README.md | 152 ++--- .../vlmevalkit/launchable/compose.yaml | 12 +- .../vlmevalkit/launchable/launchable.py | 352 +++++----- .../reasoner/vlmevalkit/launchable/setup.sh | 42 +- .../vlmevalkit/launchable/test_launchable.py | 176 +++-- .../launchable/test_local_scoring.py | 425 ------------ .../vlmevalkit/launchable/test_mvbench.py | 368 ++++++++++ .../vlmevalkit/vlmeval/dataset/__init__.py | 2 +- .../dataset/metropolis2d/astro_2d_dataset.py | 30 - .../vlmeval/dataset/metropolis_dvc.py | 69 +- .../dataset/metropolis_event_verification.py | 10 - .../vlmeval/dataset/metropolis_temporal.py | 16 +- .../vlmeval/dataset/metropolis_vqa.py | 16 +- .../vlmevalkit/vlmeval/dataset/mvbench.py | 126 +++- .../vlmevalkit/vlmeval/local_scoring.py | 526 --------------- .../vlmevalkit/vlmeval/mvbench_local.py | 525 +++++++++++++++ .../vlmevalkit/vlmeval/scoring_assets.py | 630 ------------------ 22 files changed, 1595 insertions(+), 2119 deletions(-) mode change 100644 => 100755 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py delete mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py delete mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py delete mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore index 0013b61a8..1969035f7 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore @@ -14,6 +14,11 @@ launchable/state **/*.key **/scoring-assets.json **/scoring-assets +**/.mvbench +**/mvbench-data +**/*.tsv +**/*.xlsx +**/*.pkl **/results **/env.sh cosmos_eval_out diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md index 138263b8d..6a8d77271 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md @@ -41,31 +41,54 @@ remain available. Missing or nonexistent grounding annotation files produce a clear `ValueError`. Omni3D retains its existing private configuration and can be pointed at another authorized dataset through `raw_image_path` and `json_path`. -## Brev Nano and fully local VANTAGE scoring +## Brev Nano and public MVBench evaluation -The [Brev launchable](launchable/README.md) uses this Framework-native kit, the -pinned Nano revision, and a caller-supplied, labeled `scoring-assets.json` bundle. -Prepare authorized sources offline with `python -m vlmeval.scoring_assets`; that -step writes canonical tables and a fingerprinted manifest, not original labels -or media. Keep the bundle and runtime state outside every source checkout. -During evaluation Compose mounts the completed bundle read-only; local scoring -does not use an API judge, dataset downloader, credential mount, or external -submission workflow. +The [Brev launchable](launchable/README.md) uses the Framework-native kit, +the pinned Nano revision, and `OpenGVLab/MVBench` at +`230a2d4fac8900333c61754641c7a13e069ac9c6`. It provides public download, +coverage validation, native inference, and local exact-match scoring for 20 +tasks and 4,000 annotated multiple-choice questions. No private annotation +bundle, private S3, GPU, or model endpoint is required for score-only execution. -For `MetropolisVQA`, `MetropolisTemporal`, `MetropolisEventVerification`, and -`MetropolisDVC`, provide `local_data_file` and `local_video_root` together to -bypass cache and remote-source discovery. The local table must include genuine -ground truth and bundle-relative media identities; missing labels or media fail. -`Astro2DBenchDataset` accepts a `data_roots` mapping covering all registered child -datasets. Existing explicit local grounding, pointing, and SOT constructor paths -are reused. Local DVC scoring requires a fingerprinted, revisioned local -roberta-large checkpoint and never consults the remote BERTScore endpoint. +Set `LAUNCHABLE_STATE_DIR` to an absolute writable host directory outside the +checkout. `MVBENCH_DATA_DIR` defaults to its `mvbench` child and is mounted +read-write at `/mvbench`; cache and results use separate mounts. The notebook +downloads the selected public scope itself. Approved media terms restrict use +to academic research; public access is not commercial-use or redistribution +permission. Full coverage requires manually acquired authorized NTU RGB+D +media under `video/nturgbd`; removed videos are never fetched from historical +revisions or mirrors. Missing media produces a coverage error, not an incomplete +full score. Default smoke is five fixed Egocentric Navigation rows. -Use `python -m vlmeval.local_scoring` for existing native predictions without a -model endpoint or GPU. Prediction identity columns must match trusted rows; -prediction-supplied answers and paths cannot replace ground truth. Subsets -require an explicit opt-in and are reported as subsets. There is no aggregate -cross-task score or claim of official parity. Dataset distribution approval and -a named reference result remain release gates. Original labels and native -filtering are preserved; annotation anomalies are recorded in private local -diagnostics rather than repaired or silently omitted. +Native `run.py` configuration uses the local dataset class directly: + +```json +{ + "data": { + "MVBench": { + "class": "MVBenchLocal", + "dataset": "MVBench", + "data_root": "/mvbench", + "nframe": 16, + "smoke": false + } + } +} +``` + +The controller composes this with `CosmosReason2` and the existing generation +recipe, defaulting to one API worker. Temporal sampling and prompts are native +MVBench, with sampled-image input and isolated clip/frame caches. The starting +16-frame target and scale/center-crop-224 frame preprocessing follow the original +recipe; Cosmos image encoding/normalization and answer parsing still differ from VideoChat2. +Native count-based accuracy (fraction), task macro, coverage and provenance +remain explicit. No LLM judge or published-score claim is involved. + +Use `launchable.py score --scope smoke|full` with either an existing controller +`--work-dir` or native `--predictions` plus a fresh `--score-dir`. Exact selected +identities and complete coverage are required. Trusted answers come from pinned +annotations, never prediction-supplied fields. Score-only skips endpoint checks +and model construction. BYOM uses `COSMOS_API_BASE`, `COSMOS_MODEL`, optional +environment-only `COSMOS_API_KEY`, and an explicit `COSMOS_MODEL_REVISION`; +setup then skips local serving. This launchable does not implement a +customer-data evaluation agent or alter other model deployment tracks. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md index a6bd89ffa..934333e4b 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# TAR and VANTAGE Benchmarks for Cosmos3 Model +# Cosmos3 evaluation: MVBench launchable and native benchmark recipes This directory is an **NVIDIA-customized [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)** maintained in Cosmos Framework. It contains reference settings for NVIDIA @@ -14,11 +14,17 @@ datasets publicly reproducible or establish official score parity. For user-hosted endpoints and local dataset inputs, see [Deployment and dataset configuration](EXTERNAL_CONFIGURATION.md). -For a Nano-only Brev notebook and fully local VANTAGE scoring, use the -[Brev launchable](launchable/README.md). It requires an authorized labeled bundle -prepared outside the checkout and mounted read-only; it has no external scoring -submission workflow. Data redistribution approval and a named reference result -remain release gates. See that guide for validation status and limits. +For the Nano-only Brev notebook, use the +[MVBench launchable](launchable/README.md): 20 tasks, 4,000 public annotated +multiple-choice questions, pinned public downloads, native inference, and local +exact-match scoring without an API judge. Its default smoke is five Egocentric +Navigation questions, not a full score; full evaluation is explicit opt-in. +The approved dataset revision requires manual authorized acquisition of 320 NTU +RGB+D videos and restricts video use to academic research. Public access does +not establish commercial rights, redistribution permission, or published-score +parity. Data and state stay outside the checkout. This is not a customer-data +evaluation-agent implementation. Existing native recipes below are independent +of this launchable and remain unchanged. The kit ships, for each benchmark, reference `run.py --config` settings — split into a shared dataset layer (`cosmos_eval/data/`) and a per-family model layer diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile index a9729aef7..bbd99db88 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile @@ -16,6 +16,6 @@ RUN pip install --no-cache-dir --no-deps -e . \ ARG COSMOS_COMMIT=unknown ENV COSMOS_COMMIT=${COSMOS_COMMIT} PYTHONUNBUFFERED=1 HF_HOME=/cache/huggingface \ AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null \ - AWS_EC2_METADATA_DISABLED=true SCORING_ASSETS_DIR=/scoring-assets LMUData=/cache/lmudata + AWS_EC2_METADATA_DISABLED=true MVBENCH_DATA_DIR=/mvbench LMUData=/cache/lmudata WORKDIR /opt/vlmevalkit/launchable CMD ["python", "jupyter_config.py"] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index 533ed9eb4..6d3774e61 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -4,24 +4,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Evaluate Cosmos3-Nano locally\n", + "# Evaluate Cosmos3-Nano on MVBench\n", "\n", - "**Validate labeled assets → run five VQA predictions → score locally.** Full eight-task inference is opt-in. Existing predictions can be scored without a model endpoint or GPU.\n", + "**Download public inputs → validate coverage → infer → score locally.** The default is the fixed five-question Egocentric Navigation smoke, not a full score. Set `RUN_FULL = True` to select full evaluation directly; it does not rerun smoke first.\n", "\n", - "This notebook uses Cosmos Framework's native VLMEvalKit datasets and metrics. It does not upload outputs or prepare an external scoring submission. A labeled, fingerprinted scoring bundle is required. Dataset access does **not** grant permission to redistribute labels or media. Public release remains gated on approved data licensing/distribution and a named reference result.\n", + "[MVBench](https://arxiv.org/abs/2311.17005) contains 4,000 annotated multiple-choice questions across 20 video-understanding tasks. This notebook uses the native VLMEvalKit MVBench task definitions, temporal bounds, prompts, and exact-match evaluator. There is no LLM judge or automatic upload. The Nano serving revision is `e59a53c25979a090fa8706c9acc0c254a6e89b92`.\n", "\n", - "The serving configuration targets the reasoner in `nvidia/Cosmos3-Nano` at `e59a53c25979a090fa8706c9acc0c254a6e89b92`." + "**Research-only media terms:** public access does not establish commercial rights or redistribution permission. The card mentions 320 NTU videos, but this pinned revision's manual list contains 200 RGB AVI files for its 200 Fine-grained Pose questions. Obtain authorized access from https://rose1.ntu.edu.sg/dataset/actionRecognition/ and use the pinned video/MVBench_videos_ntu.txt list. Removed media is never downloaded from historical revisions or mirrors. No published-score reproduction or commercial-rights claim is made. This is not a customer-data evaluation-agent implementation.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1 · Configure\n", + "## 1 · Choose one execution scope\n", "\n", - "`SCORING_ASSETS_DIR` is mounted read-only by Compose. Prepare the bundle outside every source checkout before setup; it must contain `scoring-assets.json`, canonical tables, original labels, and media. DVC also requires its pinned local roberta-large metric model. `LAUNCHABLE_STATE_DIR` must likewise be outside the checkout and separate from the bundle. See the README for preparation. No AWS credentials are mounted.\n", + "The external host `MVBENCH_DATA_DIR` is writable at `/mvbench`; caches and results are separate external mounts. No AWS credentials, Docker socket, labels, media, or model weights are embedded in this image.\n", "\n", - "For existing predictions, set `SCORE_ONLY = True`, `PREDICTIONS_FILE` to a native TSV/XLSX/JSON table, and `TASK` appropriately. Native identity columns must be retained. Set `ALLOW_SUBSET = True` only when intentionally scoring a partial dataset; coverage is recorded explicitly." + "For **score-only**, choose `SCORE_SCOPE` explicitly (`smoke` or `full`) and set exactly one of `PREDICTIONS_FILE` (native TSV/XLSX/JSON) or `EXISTING_WORK` (a completed controller inference directory). No endpoint check, GPU, model construction, or private S3 access is used. The selected scope must match the predictions exactly; arbitrary partial scores are not accepted.\n", + "\n", + "For full runs, stage authorized NTU files under `/mvbench/video/nturgbd` using Jupyter's authenticated file browser/terminal before preparation. The download cell retrieves only currently approved public files. Preparation reports missing media and refuses to run an incomplete full benchmark.\n" ] }, { @@ -40,34 +42,43 @@ "\n", "SCORE_ONLY = os.environ.get('EVAL_SCORE_ONLY') == '1'\n", "PREDICTIONS_FILE = None\n", - "TASK = 'VANTAGE_VQA'\n", - "ALLOW_SUBSET = False\n", - "RUN_FULL = False\n", - "ASSETS = Path(os.environ.get('SCORING_ASSETS_DIR', '/scoring-assets'))\n", + "EXISTING_WORK = None\n", + "SCORE_SCOPE = 'smoke' # Explicitly change to 'full' for 4,000 existing predictions.\n", + "RUN_FULL = False # Full inference is opt-in; selecting it skips the smoke.\n", + "DOWNLOAD_PUBLIC = True\n", + "NFRAME = 16 # Reference frame count; not a model-score parity claim.\n", + "API_NPROC = 1\n", + "DATA = Path(os.environ.get('MVBENCH_DATA_DIR', '/mvbench'))\n", + "SCOPE = SCORE_SCOPE if SCORE_ONLY else ('full' if RUN_FULL else 'smoke')\n", "STAMP = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')\n", - "SMOKE_WORK = Path('/results') / f'local-vqa-smoke-{STAMP}'\n", - "FULL_WORK = Path('/results') / f'local-full-{STAMP}'\n", - "SCORE_DIR = Path('/results') / f'local-score-only-{STAMP}'\n", - "CLI = [sys.executable, str(Path('launchable.py').resolve())]\n", + "WORK = Path('/results') / f'mvbench-{SCOPE}-{STAMP}'\n", + "SCORE_DIR = Path('/results') / f'mvbench-score-{SCOPE}-{STAMP}'\n", + "CLI = [sys.executable, '-u', str(Path('/opt/vlmevalkit/launchable/launchable.py'))]\n", "\n", "def invoke(command):\n", - " result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n", - " print(result.stdout, end='')\n", - " result.check_returncode()\n", - "\n", - "if SCORE_ONLY and not PREDICTIONS_FILE:\n", - " raise ValueError('Set PREDICTIONS_FILE to an existing native prediction table before score-only execution.')\n", - "print('Existing-prediction scoring' if SCORE_ONLY else 'Five-question inference and local scoring')\n", - "print('Assets:', ASSETS)" + " # Stream instead of buffering an entire long-running inference job.\n", + " with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,\n", + " text=True, bufsize=1) as process:\n", + " for line in process.stdout:\n", + " print(line, end='', flush=True)\n", + " if process.wait():\n", + " raise subprocess.CalledProcessError(process.returncode, command)\n", + "\n", + "if SCOPE not in {'smoke', 'full'}:\n", + " raise ValueError('Choose explicit smoke or full scope.')\n", + "if SCORE_ONLY and (PREDICTIONS_FILE is None) == (EXISTING_WORK is None):\n", + " raise ValueError('Set exactly one of PREDICTIONS_FILE or EXISTING_WORK for score-only execution.')\n", + "print('Score existing predictions' if SCORE_ONLY else 'Generate and score predictions', SCOPE)\n", + "print('Data:', DATA, '| frames:', NFRAME, '| concurrent API workers:', API_NPROC)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2 · Validate local labels and media\n", + "## 2 · Download the chosen public scope\n", "\n", - "Missing labels, altered annotation fingerprints, or unavailable media fail before contacting the model. Public inference-only inputs cannot substitute for ground truth." + "The downloader pins `OpenGVLab/MVBench` to `230a2d4fac8900333c61754641c7a13e069ac9c6`, fetches all 20 official annotation JSONs and licensing/NTU notes, and safely extracts selected current public archives, including the June-2024 `data0613` overlay where applicable. Smoke needs only the approximately 40 MB `vlnqa.zip` archive. Full download is substantially larger and **cannot supply the manual NTU videos**. Existing approved files are reused. Set `DOWNLOAD_PUBLIC = False` for offline scoring with already staged inputs.\n" ] }, { @@ -76,17 +87,19 @@ "metadata": {}, "outputs": [], "source": [ - "selected = TASK if SCORE_ONLY else ('all' if RUN_FULL else 'VANTAGE_VQA')\n", - "invoke(CLI + ['prepare', '--assets-dir', str(ASSETS), '--benchmarks', selected])" + "if DOWNLOAD_PUBLIC:\n", + " invoke(CLI + ['download', '--data-root', str(DATA), '--scope', SCOPE])\n", + "else:\n", + " print('Using already staged public annotations and media.')\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 3 · Connect only when generating predictions\n", + "## 3 · Prepare trusted annotations and require complete selected coverage\n", "\n", - "For BYOM, set the complete `COSMOS_API_BASE` chat-completions URL and `COSMOS_MODEL` in the evaluator environment before running the notebook. Keep any API key in the environment, not notebook source. Score-only mode skips this step entirely." + "Smoke selects exactly stable indices 3400–3404: five of 4,000 rows, one of 20 tasks. Full requires all 4,000 rows and all 20 tasks; no missing clips or questions are silently dropped. Missing full NTU media is an actionable prerequisite failure, not a partial full score. A structured coverage report is retained under `/mvbench/.mvbench/`.\n" ] }, { @@ -95,17 +108,19 @@ "metadata": {}, "outputs": [], "source": [ - "if not SCORE_ONLY:\n", - " invoke(CLI + ['check', '--wait', '1800'])" + "scope_flag = ['--smoke'] if SCOPE == 'smoke' else []\n", + "invoke(CLI + ['prepare', '--data-root', str(DATA)] + scope_flag)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 4 · Generate and score, or score existing predictions\n", + "## 4 · Check the endpoint only for inference\n", "\n", - "The default smoke runs five deterministically selected VQA examples and then the native local evaluator. It is a **subset**, not a full benchmark or published-score reproduction. Score-only invokes the native scoring CLI directly and never constructs an inference model." + "BYOM can be selected before host setup with `COSMOS_API_BASE` (complete `/v1/chat/completions` URL), exact `COSMOS_MODEL`, optional `COSMOS_API_KEY`, and `COSMOS_MODEL_REVISION`. Keep keys in the environment, not this notebook. The endpoint must accept sampled-image messages; fixed frame count is meaningful because no re-encoded video is sent. Generation uses the existing CosmosReason2 recipe: deterministic temperature 0, seed 1, no thinking, maximum 8,192 output tokens.\n", + "\n", + "The default 16 frames and bicubic scale/center-crop to 224 follow the reference recipe. Frames are sent as sampled images; Cosmos owns downstream image encoding/normalization, so VideoChat2 tensor normalization is not applied and model/parser differences remain. There is no paper-score parity claim.\n" ] }, { @@ -114,27 +129,19 @@ "metadata": {}, "outputs": [], "source": [ - "if SCORE_ONLY:\n", - " command = [sys.executable, '-m', 'vlmeval.local_scoring', '--task', TASK,\n", - " '--predictions', str(PREDICTIONS_FILE), '--assets-dir', str(ASSETS),\n", - " '--output-dir', str(SCORE_DIR)]\n", - " if ALLOW_SUBSET:\n", - " command.append('--allow-subset')\n", - " invoke(command)\n", - " SUMMARY = SCORE_DIR / 'summary.json'\n", + "if not SCORE_ONLY:\n", + " invoke(CLI + ['check', '--wait', '1800'])\n", "else:\n", - " invoke(CLI + ['run', '--assets-dir', str(ASSETS), '--benchmarks', 'VANTAGE_VQA',\n", - " '--mode', 'all', '--smoke', '--work-dir', str(SMOKE_WORK)])\n", - " SUMMARY = SMOKE_WORK / 'local-scores' / 'bench_00' / 'summary.json'" + " print('Score-only: endpoint check and model inference skipped.')\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 5 · Inspect the local score and provenance\n", + "## 5 · Native inference and local scoring, or score existing outputs\n", "\n", - "Each summary records the task's native metric, explicit units, aggregation rule, row coverage, asset and prediction hashes, and scorer-source hashes. DVC includes metric-model fingerprints. There is no invented cross-task overall score or claim of official leaderboard parity. All result files are private and can contain answer-bearing diagnostics; review before sharing." + "Progress streams here and into the private inference `run.log`. Fresh directories prevent stale prediction reuse. Inference and scoring reject wrong/missing/duplicate identities and blank or transport-failure outputs; the scorer reconstructs trusted identities and labels from pinned annotations. Malformed but nonempty model answers receive native wrong-answer semantics, not fabricated successes. Native exact matching runs locally without an API judge.\n" ] }, { @@ -143,18 +150,26 @@ "metadata": {}, "outputs": [], "source": [ - "summary = json.loads(SUMMARY.read_text())\n", - "print(json.dumps({key: summary[key] for key in ('task', 'status', 'coverage', 'primary_metric')}, indent=2))\n", - "display(FileLink(str(SUMMARY), url_prefix='/files'))" + "if SCORE_ONLY:\n", + " source = (['--work-dir', str(EXISTING_WORK)] if EXISTING_WORK is not None\n", + " else ['--predictions', str(PREDICTIONS_FILE)])\n", + " invoke(CLI + ['score', '--data-root', str(DATA), '--scope', SCOPE,\n", + " '--score-dir', str(SCORE_DIR), '--nframe', str(NFRAME)] + source)\n", + " SUMMARY = SCORE_DIR / 'summary.json'\n", + "else:\n", + " invoke(CLI + ['run', '--data-root', str(DATA), '--mode', 'all',\n", + " '--nframe', str(NFRAME), '--api-nproc', str(API_NPROC),\n", + " '--work-dir', str(WORK)] + scope_flag)\n", + " SUMMARY = WORK / 'scores' / 'summary.json'\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 6 · Optional full eight-task evaluation\n", + "## 6 · Inspect metrics, coverage, and provenance\n", "\n", - "Only after every requested task validates, set `RUN_FULL = True` in configuration and rerun from validation. This generates all eight tasks and scores them locally in a fresh directory. It can take substantially longer than the smoke. The labeled bundle must cover every task, including the real local DVC metric model. Astro preserves original labels and native filtering and denominators, with annotation diagnostics recorded privately. A full run does not by itself establish official parity: native per-task results require an agreed release protocol and named reference result." + "The primary metric is count-based multiple-choice accuracy as a **fraction**, with per-task metrics and the task-macro average reported separately. Smoke is prominently marked as a subset. Native scorer version/source, dataset revision and manifest, prediction hashes, selected IDs, frame protocol, generation parameters, and model/source revisions are recorded in private artifacts. A BYOM display name alone does not establish its checkpoint revision. No reference/published-score parity is claimed until independently established with an agreed protocol.\n" ] }, { @@ -163,24 +178,24 @@ "metadata": {}, "outputs": [], "source": [ - "if RUN_FULL and not SCORE_ONLY:\n", - " invoke(CLI + ['run', '--assets-dir', str(ASSETS), '--benchmarks', 'all',\n", - " '--mode', 'all', '--work-dir', str(FULL_WORK)])\n", - " invoke(CLI + ['results', '--work-dir', str(FULL_WORK)])\n", - " display(FileLink(str(FULL_WORK / 'run.json'), url_prefix='/files'))\n", - "else:\n", - " print('Full eight-task inference disabled.')" + "summary = json.loads(SUMMARY.read_text())\n", + "print(json.dumps({key: summary[key] for key in\n", + " ('status', 'coverage', 'primary_metric', 'per_task', 'provenance')}, indent=2))\n", + "display(FileLink(str(SUMMARY), url_prefix='/files'))\n", + "if not SCORE_ONLY:\n", + " invoke(CLI + ['results', '--work-dir', str(WORK)])\n", + " display(FileLink(str(WORK / 'run.json'), url_prefix='/files'))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Stop and retain results\n", + "## Stop and retain private results\n", "\n", - "From the launchable directory, `docker compose --profile local stop cosmos-endpoint` stops serving without deleting state. To restart an approved inference run, use `bash setup.sh` (or `bash setup.sh --brev` on Brev). `bash setup.sh --score-only` starts only the notebook for existing-prediction scoring. Stopping a container does not stop billing for the underlying Brev instance; stop that instance separately when finished.\n", + "Container lifecycle commands belong in the **host** terminal, not this notebook: `docker compose --profile local stop` preserves state. `bash setup.sh --score-only` starts only authenticated Jupyter for existing-prediction scoring. There is no Docker socket in the evaluator and the local model API has no host port. Stopping containers does not stop underlying Brev VM billing; stop that instance separately after retaining authorized results.\n", "\n", - "Clear notebook outputs before sharing. Do not upload restricted labels, media, credentials, or prediction artifacts to the PR." + "Clear outputs before sharing. Do not commit restricted media, labels, credentials, or answer-bearing artifacts. Native Linux x86-64 is the supported runtime; code changes alone do not demonstrate notebook/hardware execution or full benchmark reproduction.\n" ] } ], @@ -192,7 +207,7 @@ }, "language_info": { "name": "python", - "version": "3.11" + "version": "3.11.14" } }, "nbformat": 4, diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 5aab9e6c8..04ec8a700 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -1,146 +1,102 @@ -# Cosmos3 VANTAGE evaluation on Brev +# Cosmos3-Nano MVBench evaluation on Brev -Jupyter and CLI orchestration for native VLMEvalKit inference **and fully local scoring**. Use a labeled, fingerprinted local asset bundle; score existing predictions without a model endpoint or GPU. There is no external scoring submission, upload, or separate public harness. +Authenticated Jupyter orchestration for **public download → preparation → native inference → local scoring → results**, entirely inside the notebook after host startup. Existing native predictions can also be scored on CPU without a model endpoint, GPU, private S3, API judge, or external scoring submission. This is a benchmark launchable, **not a customer-data evaluation-agent implementation**. -**Release gates:** Dataset access does not grant permission to redistribute labels or media. Public inference-only inputs are insufficient for local scoring. Do not advertise published-score reproducibility until the labeled bundle is approved for distribution and a named reference result and release protocol are agreed. No private data, credentials, or benchmark outputs belong in this repository or PR. +[MVBench](https://arxiv.org/abs/2311.17005) has **20 tasks and 4,000 annotated multiple-choice questions**. The paper's [Ask-Anything repository](https://github.com/OpenGVLab/Ask-Anything/tree/main/video_chat2) supplies its original evaluation recipe. This launchable pins public [OpenGVLab/MVBench](https://huggingface.co/datasets/OpenGVLab/MVBench/tree/230a2d4fac8900333c61754641c7a13e069ac9c6) data to `230a2d4fac8900333c61754641c7a13e069ac9c6` and reuses native VLMEvalKit MVBench task definitions, temporal bounds, sampling and prompt construction, plus the native exact-match scorer. No labeled private annotation bundle is required. -This work targets the [16B BF16 Cosmos3-Nano checkpoint](https://huggingface.co/nvidia/Cosmos3-Nano/blob/e59a53c25979a090fa8706c9acc0c254a6e89b92/README.md), `nvidia/Cosmos3-Nano` at `e59a53c25979a090fa8706c9acc0c254a6e89b92`, and evaluates its reasoner. Other models and deployment tracks are outside this launchable's scope. +**Licensing and full-data prerequisite:** read the pinned dataset card and individual source terms before downloading. Its video terms are academic-research-only; public access does not imply commercial-use or redistribution rights. The card mentions 320 NTU RGB+D videos, but this pinned revision's exact manual list contains **200 RGB AVI files**, matching its 200 Fine-grained Pose questions. Request the exact [pinned file list](https://huggingface.co/datasets/OpenGVLab/MVBench/blob/230a2d4fac8900333c61754641c7a13e069ac9c6/video/MVBench_videos_ntu.txt) through [NTU/ROSE Lab's account and release-agreement process](https://rose1.ntu.edu.sg/dataset/actionRecognition/); only RGB is needed. Supply authorized media under `video/nturgbd` in the external data root. The downloader never recovers removed media from old revisions or mirrors. Full preparation fails with an actionable coverage report while required media is missing; an incomplete download is not a full benchmark score. -## Supply the scoring bundle +## Start authenticated Jupyter -Set absolute **host** paths outside every source checkout before setup: +Use native Linux x86-64, Docker Engine/Compose v2, and NVIDIA Container Toolkit for local Nano serving. The maintained source is [NVIDIA/cosmos-framework](https://github.com/NVIDIA/cosmos-framework), not `NVIDIA/cosmos`. Select the approved branch/commit containing this launchable and retain its exact revision. Disable Brev's managed Jupyter because Compose provides its own authenticated service. -```bash -export SCORING_ASSETS_DIR=/absolute/path/to/authorized-scoring-bundle -export LAUNCHABLE_STATE_DIR=/absolute/path/to/private-runtime-state -``` - -The bundle contains `scoring-assets.json`, original labels, canonical tables, and media. DVC additionally requires the pinned local roberta-large checkpoint. Compose mounts the bundle read-only at `/scoring-assets`; frame caches use `/cache/lmudata`, and results use `/results`. Do not mount AWS profiles, home directories, or credential files. No shared NVIDIA profile, NGC key, dataset-service credential, or API judge is required at evaluation time. - -Preparation requires a writable authorized source directory; execution requires the completed bundle mounted **read-only**, never a source checkout, credential store, or writable cache/results directory. Setup requires both exported paths and rejects checkout-local paths. Keep these exports in the host shell for later Compose lifecycle commands. - -Each requested task must be complete. Missing labels/media or modified fingerprints fail before inference. A VQA-only bundle is sufficient for the five-question smoke; full evaluation requires all eight tasks. Internal access and release approval are separate requirements. - -### Maintainers: prepare authorized sources - -Preparation is an offline conversion, not a dataset downloader. Keep the original filtered release layout, including `VQA`, `Temporal`, `Dense Video Caption`, `Spatial`, `2dbbox`, `datasets/Metropolis2DPointing`, `event_verification_subset`, and `tracking`. It does not copy media or rewrite original labels. Never put the source tree inside the code checkout. - -From the maintained Framework kit root (`.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit`), build the CPU evaluator: - -```bash -docker build --platform linux/amd64 -t cosmos3-vantage-eval:local -f launchable/Dockerfile . -``` - -For DVC, stage the genuine public metric model first. This download needs network access but no model-serving endpoint. It does not download benchmark data: +From the Framework checkout root: ```bash -docker run --rm --platform linux/amd64 \ - -v "$SCORING_ASSETS_DIR:/scoring-assets" --entrypoint python cosmos3-vantage-eval:local \ - -c 'from huggingface_hub import snapshot_download; snapshot_download("FacebookAI/roberta-large", revision="722cf37b1afa9454edce342e7895e588b6ff1d59", local_dir="/scoring-assets/metric-models/roberta-large", allow_patterns=["config.json","merges.txt","model.safetensors","tokenizer.json","tokenizer_config.json","vocab.json"])' - -docker run --rm --platform linux/amd64 --network none \ - -v "$SCORING_ASSETS_DIR:/scoring-assets" --entrypoint python cosmos3-vantage-eval:local \ - -m vlmeval.scoring_assets --assets-dir /scoring-assets --asset-id YOUR_IMMUTABLE_DATASET_VERSION \ - --bert-score-model metric-models/roberta-large \ - --bert-score-revision 722cf37b1afa9454edce342e7895e588b6ff1d59 +export LAUNCHABLE_STATE_DIR=/absolute/path/outside/checkouts/private-mvbench-state +# Optional: defaults to "$LAUNCHABLE_STATE_DIR/mvbench". +export MVBENCH_DATA_DIR=/absolute/path/outside/checkouts/mvbench-data +bash .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh --brev ``` -For VQA only, omit the metric model and pass `--tasks VQA`. Preparation validates original ground truth before native conversion, including complete tracking-frame labels. Explicit VQA labels are trimmed/uppercased, never defaulted to A; binary and four-choice source questions are supported. Invalid/missing labels fail rather than manufacturing answers. Scorer-source fingerprints record this normalization; compare results only under an agreed reference protocol. - -Do not regenerate or modify the manifest between inference and scoring. The controller records its hash and rejects a different snapshot. DVC model files are fingerprinted separately and checked before use. +Omit `--brev` on a non-Brev host. Add **`--score-only`** to start only the CPU evaluator/Jupyter service, never a GPU endpoint. Public annotations and videos can be downloaded by the notebook itself; setup does not require preexisting dataset files or start inference. Host path/configuration validation completes before starting a paid model service. -## Start the launchable +State and data directories must be absolute, writable, outside the checkout, and separate from each other’s cache/results mounts. Compose project `cosmos3-mvbench` mounts the external data directory **read-write at `/mvbench`**, cache at `/cache`, and private results at `/results`. Keep these path exports in the host shell for subsequent lifecycle commands. No weights or dataset contents are embedded in the image/repository. No Docker socket, AWS profile, home directory, or private key is mounted. -Use Linux x86-64, Docker Engine/Compose v2, and NVIDIA Container Toolkit for local serving. The source repository is [NVIDIA/cosmos-framework](https://github.com/NVIDIA/cosmos-framework), not `NVIDIA/cosmos`. Select the approved branch/commit containing this launchable and record its exact Framework commit; upstream `main` must not be assumed to include an unpublished branch. Brev's managed Jupyter must be disabled because Compose provides it. - -From the Framework checkout root, with the authorized bundle prepared and both host paths exported: +`--brev` discovers the private `wt0` mesh IPv4 address and fails without it. An explicit private `JUPYTER_BIND_ADDRESS` takes precedence; non-Brev defaults to loopback. There is no public/wildcard fallback. Setup persists only the notebook token and non-secret bind address in restricted `.env`. Use Brev's authenticated secure port-8888 link or an SSH tunnel. From the launchable directory on the **host**, retrieve the login token with: ```bash -bash .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh --brev +docker compose exec mvbench-eval jupyter server list ``` -On a non-Brev host, omit `--brev`. For **existing-prediction scoring only**, add `--score-only`; this starts Jupyter without starting the GPU endpoint. Normal setup builds the evaluator and validates VQA assets before starting serving. +Never share that token or include it in screenshots. Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb`. Jupyter's authenticated file root is `/`, so authorized NTU files can be staged via its file browser/terminal and result links work. The local model API has **no host port**, only the private container network. Host Docker commands are for setup/stop/lifecycle, not benchmark execution. + +### Pinned Nano and BYOM -Brev access must use its authenticated/secure port-8888 link. `--brev` discovers the private `wt0` mesh IPv4 address and fails without it; non-Brev defaults to loopback. An explicit private `JUPYTER_BIND_ADDRESS` takes precedence. There is no public/wildcard fallback. Setup persists only its notebook token and non-secret bind address in permission-restricted `.env`. Retrieve the login token on the host with `docker compose exec vantage-eval jupyter server list`; never share it or capture it in screenshots. +The local service targets `nvidia/Cosmos3-Nano`, revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`, with serving image `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55`. `VLLM_IMAGE` can override the image, with the override recorded. Allow storage for weights, images, public video archives, extracted media, frames, and results. The launchable does not change Ultra/LaunchPad workflows. -Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb`. Jupyter's authenticated file root is `/`, allowing links to `/results`. The evaluator has no Docker socket. The model API has no host port. Container lifecycle commands belong in the host terminal. +For BYOM, set `COSMOS_API_BASE` to the full reachable `/v1/chat/completions` URL and `COSMOS_MODEL` to the exact served ID **before setup**; optionally supply `COSMOS_API_KEY` in the environment. Setup skips local serving. Set `COSMOS_MODEL_REVISION` to your real deployment/checkpoint revision; unknown BYOM revisions remain explicitly `unknown` rather than being attributed to the pinned Nano checkpoint. Container `localhost` is not the host. The endpoint must accept sampled-image messages and expose `/v1/models`. Keys stay in the environment, never config/notebook source. -Serving uses `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` (override with `VLLM_IMAGE`). Allow space for model weights, images, the labeled bundle, frame caches, and results. +## Execute one chosen notebook scope -### Validation status and limits +The notebook starts with `RUN_FULL = False`: exactly the first five Egocentric Navigation questions (stable native indices **3400–3404**), one task and five of 4,000 rows. It is prominently identified as a **smoke subset**, never a full result. Its public archive is approximately 40 MB (`video/vlnqa.zip`). -- The runtime code passed 23 native-scoring/artifact regressions on native Linux x86-64. Fixtures are synthetic and use temporary directories; no benchmark labels, media, or executed notebook outputs are included. -- A private five-question Brev notebook smoke exercised Nano inference and native local scoring. Saved predictions were also rescored in a CPU evaluator with networking disabled. -- Private full-coverage inference and native local scoring have now been exercised for all eight tasks with manually staged authorized assets. DVC used its genuine pinned local metric model offline on saved Nano outputs. This is recovered-run evidence, not a fresh one-click launch or published-score reproduction; detailed results remain private. -- Preparation preserves original labels, native filtering, and ground-truth denominators. Annotation anomalies are recorded in private local diagnostics rather than repaired or silently omitted. -- Immutable one-click cold launch, end-to-end BYOM, and named reference-score reproduction remain unverified. Scoring assets, detailed operational evidence, and benchmark results must be supplied and retained separately. +Set **`RUN_FULL = True`** to opt into full evaluation directly. The notebook downloads the chosen scope, prepares it, checks the endpoint only for inference, runs inference plus scoring, and displays coverage/metrics/provenance. It does **not** unnecessarily rerun smoke before full. Full preparation requires all 20 official JSON files, all 4,000 questions and required media, including manually acquired NTU. Downloading current public archives alone cannot meet that gate. Missing coverage is written to `/mvbench/.mvbench/coverage-full.json` and reported clearly; nothing is silently truncated or dropped. -## Run and score +For existing predictions, set `SCORE_ONLY = True`, choose explicit `SCORE_SCOPE = 'smoke'` or `'full'`, and provide exactly one of `PREDICTIONS_FILE` or `EXISTING_WORK`. Set `DOWNLOAD_PUBLIC = False` when using already staged inputs offline. Score-only does not check the endpoint or construct an inference model. -The notebook defaults to a deterministic five-question VQA inference-and-scoring smoke. `RUN_FULL = False` prevents Run All from launching full eight-task inference. Score-only mode requires an existing native prediction table and explicitly selected task; it skips endpoint checks and model construction. +### CLI equivalents inside Jupyter's evaluator environment -CLI equivalents inside the evaluator: +These commands can also be invoked from Jupyter's terminal; no host Docker benchmark command is necessary: ```bash -python launchable.py prepare --benchmarks VANTAGE_VQA -python launchable.py run --benchmarks VANTAGE_VQA --smoke --mode all --work-dir /results/smoke-unique +python launchable.py download --scope smoke +python launchable.py prepare --smoke +python launchable.py run --smoke --mode all --work-dir /results/smoke-unique python launchable.py results --work-dir /results/smoke-unique -# Full native inference and local scoring, only after all task assets validate. -python launchable.py prepare --benchmarks all -python launchable.py run --benchmarks all --mode all --work-dir /results/full-unique +# Metadata-only acquisition, without video downloads: +python launchable.py download --scope metadata -# Separate inference from scoring; the latter never contacts a model endpoint. -python launchable.py run --benchmarks VANTAGE_VQA --mode infer --work-dir /results/infer-unique -python launchable.py score --work-dir /results/infer-unique +# Full scope: authorized NTU files must already be placed under /mvbench/video/nturgbd. +python launchable.py download --scope full +python launchable.py prepare +python launchable.py run --mode all --work-dir /results/full-unique -# Native prediction files from another run: retain original identities/questions. -python -m vlmeval.local_scoring --task VANTAGE_VQA --predictions /results/native-predictions.tsv \ - --assets-dir /scoring-assets --output-dir /results/score-unique -``` - -All output directories must be fresh. Direct scoring requires full task coverage unless `--allow-subset` is explicitly supplied. A smoke/subset is always identified in coverage and is not a reference benchmark result. Duplicate, unknown, mismatched, blank, or failed predictions are rejected. Only verified prediction strings are transferred into trusted dataset rows: prediction-supplied labels, image dimensions, or paths cannot override ground truth. +# Separate native inference from local scoring: +python launchable.py run --smoke --mode infer --work-dir /results/infer-unique +python launchable.py score --scope smoke --work-dir /results/infer-unique -`--mode all` always performs native inference first, then the same local scorer used for existing outputs. It never substitutes inference-only behavior or runs a remote scoring service. `--dry-run` composes configuration but does not prove inference or media compatibility. +# Native predictions from another run, retaining original identity columns: +python launchable.py score --scope smoke --predictions /results/native-predictions.tsv \ + --score-dir /results/score-unique +``` -The controller defaults to **one concurrent API/video worker** and records `api_nproc` in `run.json`. Video decoding consumes host RAM independently of GPU VRAM; increase `--api-nproc N` only after measuring available host memory. This changes scheduling, not model generation or metric parameters. +`--data-root` overrides the `MVBENCH_DATA_DIR` default (`/mvbench`). `run` defaults to **full scope**, `--mode all`, `--nframe 16`, and **`--api-nproc 1`**. Full is never silently limited. All run/score output directories must be fresh. `--dry-run` still requires valid selected annotations/media and emits configuration only; it does not prove model compatibility. Score scope is mandatory and must match all expected identities exactly; there is no arbitrary partial-full option. -## Metric semantics and provenance +## Protocol, metrics, and provenance -| Task | Native primary metric | Unit / aggregation | -| --- | --- | --- | -| VQA | accuracy | Fraction; question micro | -| EventVerification | macro F1 | Fraction; class macro | -| Temporal | temporal IoU | Fraction; video macro of question means | -| DVC | SODA-c | Fraction; video macro, genuine local BERTScore | -| 2DGrounding | Acc@0.5 | Percent; expression micro | -| 2DPointing | accuracy | Fraction; question micro | -| SOT | success AUC | Fraction; sequence macro; initialization excluded | -| Astro2D | F1 | Percent; unweighted sequence macro | +The original VideoChat2 MVBench notebook uses **16 frames and resolution 224**, and computes average accuracy as total correct / total questions. This launchable starts with **16 frames** (configurable via `--nframe`), applies reference bicubic GroupScale(224) and GroupCenterCrop(224) before native frame stacking/JPEG output, and uses the existing CosmosReason2 generation recipe: temperature 0, seed 1, thinking disabled, maximum 8,192 output tokens, presence penalty unset, ten API retries, and 600-second request timeout. It preserves native MVBench temporal bounds and sampled-image prompts, avoiding lossy video re-encoding and second-stage server video sampling. Cosmos image encoding/normalization and answer parsing still differ from VideoChat2; reference-model tensor normalization is not applied to API images. This is **not a paper-score reproduction or parity claim**. Revision/media-path/temporal-bound/frame-count/preprocessing-isolated caches prevent distinct clips or protocols from sharing stale frames. -Each private `summary.json` includes native metrics, explicit units/aggregation, expected/scored row counts, asset ID and annotation hashes, prediction snapshot hash, scorer-source hashes, and DVC metric-model fingerprints. There is **no invented cross-task overall VANTAGE score** or unverified leaderboard-parity claim. DVC requires a real local model; remote endpoints and dummy semantic scores are not allowed in local scoring. Missing or silently skipped examples fail scoring. Native failures leave a private `scoring-error.log`, not a success summary. +Scoring uses native `model='exact_matching'`, never an LLM/API judge. The primary metric is **count-based question accuracy as a fraction**. Native counts, per-task metrics and task-macro accuracy are reported separately. A five-question subset cannot stand in for the full 20-task metric. Supplied predictions are checked for exact identities/row counts, duplicates, blanks, and failed transport outputs; trusted labels and identity columns come from pinned official annotations, never prediction-supplied answers/paths. Nonempty malformed model answers retain native wrong-answer semantics. Failed scoring is not a successful zero or fabricated result. -Finite zero-duration and reversed predicted intervals follow each native parser and IoU implementation. The wrapper does not drop them, repair timestamps, or replace them with full-duration answers. Unparseable responses, nonfinite timestamps, missing events/captions, or skipped rows still fail rather than producing a misleading score. +The local wrapper additionally prevents punctuation-only option tokens from receiving the upstream matcher's accidental substring credit and preserves literal model strings such as `nan` as wrong answers rather than missing values. These guards are recorded in scorer provenance; they are not a claim of byte-for-byte VideoChat2 evaluator parity. -`run.json` records controller state and the exact scoring-asset manifest hash. Generated native configuration and `/opt/evaluation-packages.txt` identify runtime inputs. A BYOM endpoint's revision cannot be inferred from its display name; record its deployment revision separately. Treat predictions, native diagnostics, and summaries as private until reviewed. Nothing is uploaded automatically. +`run.json` records scope, exact selected IDs, coverage, dataset revision/manifest hash, complete emitted native config, frame count, API concurrency, generation parameters, model/deployment revision, Framework revision, source hashes, prediction location, metric summary and state transitions. `config.json`, streamed `run.log`, native predictions, `scores/summary.json`, and `/opt/evaluation-packages.txt` provide private reproducibility artifacts. Summaries retain prediction/scorer provenance. API concurrency affects scheduling and host RAM usage, not the frame count or generation protocol. Increase it only after measuring host resources. -## BYOM, stop, and restart +No published-score reproduction, reference-result parity, immutable cold-launch reliability, or complete customer-data agent is implied by implementation. Full NTU coverage is a real acquisition prerequisite. This change's empirical hardware/notebook status must be established separately; do not carry forward evidence from a different benchmark. No benchmark labels/media, credentials, executed notebook outputs, or private result artifacts belong in the PR. -At initial deployment, set `COSMOS_API_BASE` to the full `/v1/chat/completions` URL, `COSMOS_MODEL` to its exact served ID, and `COSMOS_API_KEY` only if required. Setup skips local serving for BYOM. The endpoint must support the native image/video requests and be reachable from the evaluator; container `localhost` is not the host. Native recipe exports intentionally retain environment placeholders; the controller supplies the checked endpoint URL before inference. The API adapter reads its key from the environment, not the generated configuration. Keep credentials in the environment, never notebook source or URLs. +## Stop, restart, and maintain -From the launchable directory on the host: +From the launchable directory on the **host**: ```bash docker compose --profile local ps docker compose --profile local stop -# Recreate with the same bundle and intended mode: +# Restart only Jupyter to score retained predictions: bash setup.sh --score-only ``` -Stopping a container does not stop underlying Brev VM billing. Stop the instance separately after preserving authorized results. Do not delete state or volumes to stop a run. Clear notebook outputs before sharing. - -## Local development checks - -In the installed evaluator environment, run `python -m unittest discover -s launchable -p 'test_*.py'`. Behavioral fixtures exercise native scorers without private data or inference services; DVC additionally needs a smoke with the real metric checkpoint. The committed notebook contains no executed outputs. +Stopping containers does not stop Brev VM billing. Stop the instance separately after preserving authorized results; do not delete state/volumes merely to stop a run. Clear notebook outputs before sharing. -Platform limit: local ARM64 Docker's `linux/amd64` Jupyter kernel previously stalled even in a clean upstream image. Native Linux x86-64 is the supported runtime. CLI/native metric checks under emulation are not proof of notebook execution or a fresh Brev GPU deployment. +Maintainers should run `python -m unittest discover -s launchable -p 'test_*.py'` after integration, then exercise notebook smoke, offline CPU rescore, missing-NTU full-coverage failure, and an authorized complete full run when its prerequisites are available. Behavioral fixtures use synthetic artifacts and do not establish model/hardware results. The committed notebook is unexecuted. Native Linux x86-64 is the supported runtime; local ARM64 Docker emulation is not notebook/hardware validation. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml index e1ae93b95..59240d591 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: cosmos3-vantage +name: cosmos3-mvbench services: cosmos-endpoint: profiles: [local] @@ -20,7 +20,6 @@ services: HF_HOME: /cache/huggingface volumes: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache - - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/data:/data:ro # Deliberately no host port: only the evaluator can reach this API. command: - nvidia/Cosmos3-Nano @@ -50,7 +49,7 @@ services: timeout: 10s retries: 120 start_period: 60s - vantage-eval: + mvbench-eval: platform: linux/amd64 build: context: .. @@ -65,19 +64,20 @@ services: COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} COSMOS_API_BASE: ${COSMOS_API_BASE:-http://cosmos-endpoint:8080/v1/chat/completions} COSMOS_MODEL: ${COSMOS_MODEL:-nvidia/Cosmos3-Nano} + COSMOS_MODEL_REVISION: ${COSMOS_MODEL_REVISION:-unknown} + VLLM_IMAGE: ${VLLM_IMAGE:-vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55} COSMOS_API_KEY: ${COSMOS_API_KEY:-EMPTY} JUPYTER_TOKEN: ${JUPYTER_TOKEN:?Run bash setup.sh to generate a Jupyter token} LMUData: /cache/lmudata - SCORING_ASSETS_DIR: /scoring-assets + MVBENCH_DATA_DIR: /mvbench EVAL_SCORE_ONLY: ${EVAL_SCORE_ONLY:-0} AWS_CONFIG_FILE: /dev/null AWS_SHARED_CREDENTIALS_FILE: /dev/null AWS_EC2_METADATA_DISABLED: 'true' volumes: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache - - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/data:/data:ro - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/results:/results - - ${SCORING_ASSETS_DIR:?Set an external absolute prepared bundle directory}:/scoring-assets:ro + - ${MVBENCH_DATA_DIR:?Set an external writable MVBench data directory}:/mvbench healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8888/login', timeout=5)"] interval: 15s diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py old mode 100644 new mode 100755 index d4fb4dfe4..e0853ab61 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Notebook/CLI orchestration; inference and metrics remain native VLMEvalKit's.""" +"""Notebook/CLI orchestration for pinned MVBench and native VLMEvalKit scoring.""" from __future__ import annotations import argparse +import hashlib import json import os from pathlib import Path @@ -15,20 +16,8 @@ import urllib.request ROOT = Path(__file__).resolve().parents[1] -KIT = ROOT / "cosmos_eval" - - -def manifest(): - return json.loads((KIT / "manifest.json").read_text()) - - -def benchmarks(value): - available = [key.split("/", 1)[1] for key in manifest() if "/VANTAGE_" in key] - selected = available if value == "all" else value.split(",") - unknown = set(selected) - set(available) - if unknown or not selected: - raise ValueError(f"Unknown VANTAGE benchmark(s): {', '.join(sorted(unknown))}") - return [bench for bench in available if bench in selected] +MODEL_REVISION = "e59a53c25979a090fa8706c9acc0c254a6e89b92" +SERVING_IMAGE = "vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55" def endpoint(wait=0): @@ -66,186 +55,214 @@ def endpoint(wait=0): time.sleep(min(5, max(0, deadline - time.monotonic()))) -def prepare(selected, assets_dir): - from vlmeval.local_scoring import build_local_dataset, load_assets +def prepare(data_root, smoke=False): + from vlmeval.mvbench_local import prepare_dataset - bundle = load_assets(assets_dir, selected) - for bench in selected: - dataset = build_local_dataset(assets_dir, bench) - if dataset is None or dataset.data.empty: - raise RuntimeError(f"{bench}: no labeled dataset rows") - print(f"{bench}: {len(dataset.data)} local labeled rows ready", flush=True) - diagnostics = bundle["manifest"]["tasks"][bench.removeprefix("VANTAGE_")].get("annotation_diagnostics") - if diagnostics: - print(f"{bench}: original annotation diagnostics: {json.dumps(diagnostics, sort_keys=True)}", flush=True) + prepared = prepare_dataset(data_root, smoke=smoke) + print(json.dumps(prepared["coverage"], indent=2), flush=True) + return prepared -def run_command(command, **kwargs): +def run_command(command, log_path=None): + """Tee unbuffered child progress to Jupyter and a durable private run log.""" env = {key: value for key, value in os.environ.items() if not key.startswith("AWS_")} env.update(AWS_CONFIG_FILE="/dev/null", AWS_SHARED_CREDENTIALS_FILE="/dev/null", - AWS_EC2_METADATA_DISABLED="true", MPLBACKEND="Agg") - subprocess.run(command, cwd=ROOT, env=env, check=True, **kwargs) + AWS_EC2_METADATA_DISABLED="true", MPLBACKEND="Agg", PYTHONUNBUFFERED="1") + log = Path(log_path).open("w") if log_path is not None else None + try: + with subprocess.Popen(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, bufsize=1) as process: + for line in process.stdout: + print(line, end="", flush=True) + if log is not None: + log.write(line) + log.flush() + code = process.wait() + if code: + raise subprocess.CalledProcessError(code, command) + finally: + if log is not None: + log.close() def prediction_files(work): - return sorted(p for p in work.rglob("cosmos_*") - if p.parent.name.startswith("T") and p.suffix in {".tsv", ".xlsx", ".json"} - and p.stem in {f"cosmos_{bench}" for bench in benchmarks("all")}) + return sorted(path for path in work.rglob("cosmos_MVBench.*") + if path.parent.name.startswith("T") and path.suffix in {".tsv", ".xlsx", ".json"}) -def verify_predictions(work, bench, expected_rows=None): +def verify_predictions(path, selected_indices): + """Check complete selected identities and transport failures before native scoring.""" import pandas as pd - files = [p for p in prediction_files(work) if p.stem == f"cosmos_{bench}"] - if len(files) != 1: - raise RuntimeError(f"{bench}: expected one prediction table, found {len(files)}") - path = files[0] + path = Path(path) + if path.is_dir(): + files = prediction_files(path) + if len(files) != 1: + raise RuntimeError(f"MVBench: expected one prediction table, found {len(files)}") + path = files[0] if path.suffix == ".tsv": - frame = pd.read_csv(path, sep="\t") + frame = pd.read_csv(path, sep="\t", keep_default_na=False, dtype={"index": str, "prediction": str}) elif path.suffix == ".json": - frame = pd.read_json(path) + frame = pd.read_json(path, dtype={"index": str}) + elif path.suffix == ".xlsx": + frame = pd.read_excel(path, keep_default_na=False, dtype={"index": str, "prediction": str}) else: - frame = pd.read_excel(path) - if "prediction" not in frame or frame.empty: - raise RuntimeError(f"{bench}: empty or invalid prediction output") - if expected_rows is not None and len(frame) != expected_rows: - raise RuntimeError(f"{bench}: expected {expected_rows} predictions, found {len(frame)}") + raise ValueError("Predictions must be a native TSV, XLSX, or JSON table") + if not {"index", "prediction"}.issubset(frame.columns) or frame.empty: + raise RuntimeError("MVBench: empty or invalid prediction output") + expected = [str(index) for index in selected_indices] + if len(frame) != len(expected): + raise RuntimeError(f"MVBench: expected {len(expected)} predictions, found {len(frame)}") + indices = frame["index"].astype(str) + if indices.duplicated().any() or set(indices) != set(expected): + raise RuntimeError("MVBench: duplicate, missing, or unexpected prediction identities") text = frame["prediction"].astype(str).str.strip().str.lower() failed = (frame["prediction"].isna() - | text.isin(("", "nan", "none", "null", "")) + | text.eq("") | text.str.contains("failed to obtain answer", regex=False) | text.str.contains("video_load_failed", regex=False)) if failed.any(): - raise RuntimeError(f"{bench}: {int(failed.sum())} failed/empty predictions; inspect the run log") - print(f"{bench}: {len(frame)} predictions verified", flush=True) + raise RuntimeError(f"MVBench: {int(failed.sum())} failed/empty predictions; inspect the run log") + print(f"MVBench: {len(frame)} predictions verified", flush=True) return path +def source_provenance(): + files = ("run.py", "vlmeval/mvbench_local.py", "vlmeval/dataset/video_dataset_config.py", + "vlmeval/dataset/mvbench.py", "vlmeval/api/cosmos_reason.py", "launchable/launchable.py") + return {"framework_revision": os.environ.get("COSMOS_COMMIT", "unknown"), + "source_sha256": {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() + for name in files if (ROOT / name).is_file()}} + + def _write_state(work, state): - (work / "run.json").write_text(json.dumps(state, indent=2) + "\n") + (work / "run.json").write_text(json.dumps(state, indent=2, default=str) + "\n") -def run(selected, work, mode, smoke, dry_run, assets_dir, api_nproc=1): - if api_nproc < 1: - raise ValueError("--api-nproc must be positive") - if smoke and selected != ["VANTAGE_VQA"]: - raise ValueError("--smoke requires --benchmarks VANTAGE_VQA") +def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): + if api_nproc < 1 or nframe < 1: + raise ValueError("--api-nproc and --nframe must be positive") + if mode not in {"infer", "all"}: + raise ValueError("--mode must be infer or all") if work.exists(): raise ValueError(f"Use a fresh --work-dir (already exists: {work}) to avoid stale results") - from vlmeval.local_scoring import dataset_config, load_assets - - # Missing labels fail before allocating inference work or contacting the model. - bundle = load_assets(assets_dir, selected) - local_configs = {bench: dataset_config(assets_dir, bench) for bench in selected} + # All media and selected identities validate before contacting a paid endpoint. + prepared = prepare(data_root, smoke) if not dry_run: - prepare(selected, assets_dir) endpoint(wait=1800) + recipe = json.loads((ROOT / "cosmos_eval/models/cosmos.json").read_text()) + model = {"class": recipe["class"], **recipe["defaults"]} + model.update(api_base=os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions"), + model=os.environ.get("COSMOS_MODEL", "nvidia/Cosmos3-Nano")) + config = {"model": {"cosmos": model}, "data": {"MVBench": { + "class": "MVBenchLocal", "dataset": "MVBench", "data_root": str(data_root), + "nframe": nframe, "smoke": smoke}}} work.mkdir(parents=True, mode=0o700) - configs = work / "_configs" - run_command([sys.executable, str(KIT / "run_all.py"), "--benchmarks", ",".join(selected), - "--export-configs", str(configs)]) - state = {"format_version": 2, "source": "local", "status": "planned" if dry_run else "running", - "cosmos_commit": os.environ.get("COSMOS_COMMIT", "unknown"), - "model": os.environ.get("COSMOS_MODEL", "${COSMOS_MODEL}"), - "benchmarks": selected, "mode": mode, "smoke": smoke, "dry_run": dry_run, - "api_nproc": api_nproc, - "scoring_assets": {"asset_id": bundle["manifest"]["asset_id"], - "manifest_sha256": bundle["manifest_sha256"]}, - "work_dirs": {}, "predictions": {}, "scores": {}} + config_path = work / "config.json" + config_path.write_text(json.dumps(config, indent=2) + "\n") + state = {"format_version": 3, "benchmark": "MVBench", "source": "mvbench-local", + "status": "planned" if dry_run else "running", "mode": mode, + "scope": "smoke" if smoke else "full", "smoke": smoke, "dry_run": dry_run, + "dataset_revision": prepared["dataset_revision"], + "manifest_sha256": prepared["manifest_sha256"], "coverage": prepared["coverage"], + "selected_indices": prepared["selected_indices"], "data_root": str(data_root), + "nframe": nframe, "api_nproc": api_nproc, "config": config, + "generation_parameters": {key: value for key, value in model.items() + if key not in {"class", "api_base", "model"}}, + "model": model["model"], + "model_revision": os.environ.get( + "COSMOS_MODEL_REVISION", + MODEL_REVISION if model["api_base"] == "http://cosmos-endpoint:8080/v1/chat/completions" + and model["model"] == "nvidia/Cosmos3-Nano" else "unknown"), + "serving_image": (os.environ.get("VLLM_IMAGE", SERVING_IMAGE) + if model["api_base"] == "http://cosmos-endpoint:8080/v1/chat/completions" + else os.environ.get("COSMOS_SERVING_IMAGE", "unknown")), + "provenance": source_provenance(), "predictions": None, "scores": None} _write_state(work, state) + command = [sys.executable, "-u", str(ROOT / "run.py"), "--config", str(config_path), + "--mode", "infer", "--work-dir", str(work / "native"), + "--api-nproc", str(api_nproc), "--verbose"] + if dry_run: + print(" ".join(command), flush=True) + return state try: - for index, bench in enumerate(selected): - config_path = configs / f"{bench}.json" - cfg = json.loads(config_path.read_text()) - # Recipe exports retain placeholders; supply the endpoint already checked above. - cfg["model"]["cosmos"]["api_base"] = os.environ.get( - "COSMOS_API_BASE", cfg["model"]["cosmos"]["api_base"]) - cfg["data"] = {bench: local_configs[bench]} - if smoke: - cfg["data"][bench].update(limit=5, random_state=0) - config_path.write_text(json.dumps(cfg, indent=2) + "\n") - out = work / f"bench_{index:02d}" - state["work_dirs"][bench] = out.name - command = [sys.executable, str(ROOT / "run.py"), "--config", str(config_path), - "--mode", "infer", "--work-dir", str(out), - "--api-nproc", str(api_nproc), "--verbose"] - if dry_run: - print(" ".join(command)) - continue - out.mkdir(mode=0o700) - with (out / "run.log").open("w") as log: - run_command(command, stdout=log, stderr=subprocess.STDOUT) - path = verify_predictions(out, bench, expected_rows=5 if smoke else None) - state["predictions"][bench] = str(path.relative_to(work)) - _write_state(work, state) - if not dry_run: - state["status"] = "inferred" - _write_state(work, state) - if mode == "all": - score(work, assets_dir) - else: - results(work) - except Exception: + run_command(command, log_path=work / "run.log") + path = verify_predictions(work / "native", prepared["selected_indices"]) + state.update(status="inferred", predictions=str(path.relative_to(work))) + _write_state(work, state) + if mode == "all": + score(data_root, state["scope"], work=work) + else: + results(work) + except BaseException: state = json.loads((work / "run.json").read_text()) state["status"] = "failed" _write_state(work, state) raise + return json.loads((work / "run.json").read_text()) -def score(work, assets_dir, score_dir=None): - from vlmeval.local_scoring import load_assets, score_predictions +def score(data_root, scope, work=None, predictions=None, score_dir=None, nframe=16): + from vlmeval.mvbench_local import score_predictions - state = json.loads((work / "run.json").read_text()) - if state.get("source") != "local": - raise ValueError("Use the native score-only CLI for predictions from older or other inference runs") - if state.get("dry_run") or state.get("status") not in {"inferred", "complete"}: - raise ValueError("Only successful inference runs can be scored") - selected = state["benchmarks"] - bundle = load_assets(assets_dir, selected) - if state.get("scoring_assets", {}).get("manifest_sha256") != bundle["manifest_sha256"]: - raise ValueError("Scoring assets changed since inference; use the original fingerprinted bundle") - score_dir = Path(score_dir) if score_dir is not None else work / "local-scores" + if scope not in {"smoke", "full"}: + raise ValueError("Scoring requires explicit smoke or full scope") + if (work is None) == (predictions is None): + raise ValueError("Supply exactly one of --work-dir or --predictions") + if nframe < 1: + raise ValueError("--nframe must be positive") + state = None + if work is not None: + state = json.loads((work / "run.json").read_text()) + if (state.get("source") != "mvbench-local" or state.get("dry_run") + or state.get("status") not in {"inferred", "complete"}): + raise ValueError("Only successful MVBench inference runs can be scored") + if scope != state["scope"]: + raise ValueError("Scoring scope differs from inference scope") + nframe = state["nframe"] + predictions = work / state["predictions"] + score_dir = Path(score_dir) if score_dir is not None else work / "scores" + elif score_dir is None: + raise ValueError("--score-dir is required for existing native prediction files") + score_dir = Path(score_dir) if score_dir.exists(): raise ValueError("Scoring output already exists; choose a fresh --score-dir") - predictions = {bench: verify_predictions(work / state["work_dirs"][bench], bench, - expected_rows=5 if state.get("smoke") else None) - for bench in selected} - score_dir.mkdir(parents=True, mode=0o700) - state["status"] = "scoring" - state["scores"] = {} - _write_state(work, state) + prepared = prepare(data_root, smoke=scope == "smoke") + if state is not None and any(state[key] != prepared[key] for key in + ("dataset_revision", "manifest_sha256", "selected_indices")): + raise ValueError("Dataset identities or annotations changed since inference") + predictions = verify_predictions(predictions, prepared["selected_indices"]) + if state is not None: + state["status"] = "scoring" + _write_state(work, state) try: - for index, bench in enumerate(selected): - output = score_dir / f"bench_{index:02d}" - summary = score_predictions(bench, predictions[bench], assets_dir, output, - allow_subset=bool(state.get("smoke"))) - state["scores"][bench] = {"summary": str(output / "summary.json"), - "primary_metric": summary["primary_metric"], - "coverage": summary["coverage"]} + summary = score_predictions(predictions, data_root, score_dir, + smoke=scope == "smoke", nframe=nframe) + if state is not None: + state.update(status="complete", scores={"summary": str(score_dir / "summary.json"), + "primary_metric": summary["primary_metric"], + "coverage": summary["coverage"]}) + _write_state(work, state) + results(work) + else: + print(json.dumps(summary, indent=2, default=str), flush=True) + return summary + except BaseException: + if state is not None: + state["status"] = "failed" _write_state(work, state) - state["status"] = "complete" - _write_state(work, state) - except Exception: - state["status"] = "failed" - _write_state(work, state) raise - results(work) def results(work): state = json.loads((work / "run.json").read_text()) - print(f"Local run: {state.get('status', 'unknown')}" + - (" (five-question smoke subset)" if state.get("smoke") else "")) - for bench in state["benchmarks"]: - result = state.get("scores", {}).get(bench) - if result: - metric, coverage = result["primary_metric"], result["coverage"] - print(f"{bench}: {metric['name']}={metric['value']} {metric['unit']}; " - f"{coverage['scored_rows']}/{coverage['expected_rows']} rows") - print(f" Local score: {result['summary']}") - else: - print(f"{bench}: predictions only; local scoring has not completed") - print("Per-task native metrics only; no external scoring submission or unverified leaderboard parity.") + print(f"MVBench: {state['status']} ({state['scope']})", flush=True) + print(json.dumps({"coverage": state["coverage"], "scores": state.get("scores"), + "dataset_revision": state["dataset_revision"], + "model_revision": state["model_revision"], "provenance": state["provenance"]}, + indent=2), flush=True) + print("Smoke is not a full benchmark. No published-score or paper-parity claim.", flush=True) + return state def main(): @@ -254,34 +271,45 @@ def main(): sub = parser.add_subparsers(dest="command", required=True) check = sub.add_parser("check", help="Wait for /v1/models and verify the requested model") check.add_argument("--wait", type=int, default=1800) - prep = sub.add_parser("prepare", help="Validate explicitly supplied local labeled scoring assets") - runner = sub.add_parser("run", help="Run native inference and optionally local scoring") - scorer = sub.add_parser("score", help="Score existing predictions locally, without a model endpoint") - for child in (prep, runner, scorer): - child.add_argument("--assets-dir", type=Path, - default=Path(os.environ.get("SCORING_ASSETS_DIR", "/scoring-assets"))) + download = sub.add_parser("download", help="Download approved pinned public inputs, never removed NTU media") + download.add_argument("--scope", choices=["metadata", "smoke", "full"], default="smoke") + prep = sub.add_parser("prepare", help="Validate pinned annotations and complete selected media coverage") + runner = sub.add_parser("run", help="Run native inference and optionally native exact-match scoring") + scorer = sub.add_parser("score", help="Score existing predictions without a GPU or model endpoint") + for child in (download, prep, runner, scorer): + child.add_argument("--data-root", type=Path, + default=Path(os.environ.get("MVBENCH_DATA_DIR", "/mvbench"))) for child in (prep, runner): - child.add_argument("--benchmarks", default="VANTAGE_VQA") + child.add_argument("--smoke", action="store_true", help="Fixed first five Egocentric Navigation IDs, not full MVBench") runner.add_argument("--mode", choices=["infer", "all"], default="all") - runner.add_argument("--api-nproc", type=int, default=1, - help="Concurrent API/video workers; increase only for available host RAM") - runner.add_argument("--smoke", action="store_true", help="Five seeded VQA samples, not a full benchmark") + runner.add_argument("--api-nproc", type=int, default=1) + runner.add_argument("--nframe", type=int, default=16) runner.add_argument("--dry-run", action="store_true") runner.add_argument("--work-dir", type=Path, required=True) - scorer.add_argument("--work-dir", type=Path, required=True) + sources = scorer.add_mutually_exclusive_group(required=True) + sources.add_argument("--work-dir", type=Path) + sources.add_argument("--predictions", type=Path) + scorer.add_argument("--scope", choices=["smoke", "full"], required=True) scorer.add_argument("--score-dir", type=Path) - report = sub.add_parser("results", help="Show verified local metric summaries") + scorer.add_argument("--nframe", type=int, default=16) + report = sub.add_parser("results", help="Show metric summaries, coverage, and provenance") report.add_argument("--work-dir", type=Path, required=True) args = parser.parse_args() if args.command == "check": endpoint(args.wait) + elif args.command == "download": + from vlmeval.mvbench_local import fetch_dataset + fetch_dataset(args.data_root.resolve(), scope=args.scope) elif args.command == "prepare": - prepare(benchmarks(args.benchmarks), args.assets_dir.resolve()) + prepare(args.data_root.resolve(), args.smoke) elif args.command == "run": - run(benchmarks(args.benchmarks), args.work_dir.resolve(), args.mode, - args.smoke, args.dry_run, args.assets_dir.resolve(), args.api_nproc) + run(args.work_dir.resolve(), args.mode, args.smoke, args.dry_run, + args.data_root.resolve(), args.api_nproc, args.nframe) elif args.command == "score": - score(args.work_dir.resolve(), args.assets_dir.resolve(), args.score_dir) + score(args.data_root.resolve(), args.scope, + work=args.work_dir.resolve() if args.work_dir else None, + predictions=args.predictions.resolve() if args.predictions else None, + score_dir=args.score_dir.resolve() if args.score_dir else None, nframe=args.nframe) else: results(args.work_dir.resolve()) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh index 20686600a..9bc6a3c3d 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -11,22 +11,26 @@ docker info >/dev/null export COSMOS_COMMIT COSMOS_COMMIT=$(git rev-parse HEAD) : "${LAUNCHABLE_STATE_DIR:?Set an absolute state directory outside the Framework checkout}" -: "${SCORING_ASSETS_DIR:?Set an absolute authorized scoring bundle directory outside the Framework checkout}" -export LAUNCHABLE_STATE_DIR SCORING_ASSETS_DIR -python3 - "$LAUNCHABLE_STATE_DIR" "$SCORING_ASSETS_DIR" "$(git rev-parse --show-toplevel)" <<'PY' +export MVBENCH_DATA_DIR="${MVBENCH_DATA_DIR:-$LAUNCHABLE_STATE_DIR/mvbench}" +export LAUNCHABLE_STATE_DIR +python3 - "$LAUNCHABLE_STATE_DIR" "$MVBENCH_DATA_DIR" "$(git rev-parse --show-toplevel)" <<'PY' +import os from pathlib import Path import sys checkout = Path(sys.argv[3]).resolve() -state, assets = (Path(value) for value in sys.argv[1:3]) -if any(not path.is_absolute() or path.resolve().is_relative_to(checkout) for path in (state, assets)): - raise SystemExit("State and scoring assets must use absolute paths outside the Framework checkout") -if not assets.is_dir(): - raise SystemExit("Scoring bundle directory is absent; prepare authorized sources first") -if any(assets.resolve().is_relative_to((state / name).resolve()) for name in ("cache", "results")): - raise SystemExit("Scoring assets must not be nested in writable cache/results mounts") +state, data = (Path(value) for value in sys.argv[1:3]) +for path in (state, data): + if not path.is_absolute() or path.resolve().is_relative_to(checkout): + raise SystemExit("State and MVBench data must use absolute paths outside the Framework checkout") +if any(data.resolve().is_relative_to((state / name).resolve()) + or (state / name).resolve().is_relative_to(data.resolve()) for name in ("cache", "results")): + raise SystemExit("MVBench data must be separate from writable cache/results mounts") +for path in (state / "cache", state / "results", data): + path.mkdir(parents=True, exist_ok=True) + if not os.access(path, os.W_OK): + raise SystemExit(f"Runtime directory must be writable: {path}") PY -mkdir -p "$LAUNCHABLE_STATE_DIR"/{cache,data,results} export EVAL_SCORE_ONLY=0 for argument in "$@"; do if [[ "$argument" == "--score-only" ]]; then EVAL_SCORE_ONLY=1; fi @@ -72,20 +76,20 @@ print(address) PY ) chmod 600 .env -docker compose build vantage-eval -if [[ "$EVAL_SCORE_ONLY" == "0" ]]; then - # Reject missing/incomplete labels before starting a paid model service. - docker compose run --rm --no-deps vantage-eval python launchable.py prepare -fi +docker compose config --quiet +docker compose build mvbench-eval if [[ "$EVAL_SCORE_ONLY" == "0" && -z "${COSMOS_API_BASE:-}" ]]; then nvidia-smi + export COSMOS_MODEL_REVISION="${COSMOS_MODEL_REVISION:-e59a53c25979a090fa8706c9acc0c254a6e89b92}" docker compose --profile local pull cosmos-endpoint docker compose --profile local up -d else # Existing-prediction scoring and BYOM never start a local model/GPU. - docker compose up -d vantage-eval + export COSMOS_MODEL_REVISION="${COSMOS_MODEL_REVISION:-unknown}" + docker compose up -d mvbench-eval fi printf 'Jupyter is starting on %s:8888. Open its authenticated secure link or SSH tunnel.\n' "${JUPYTER_BIND_ADDRESS:-127.0.0.1}" printf '%s\n' \ - 'Retrieve the notebook login token with: docker compose exec vantage-eval jupyter server list' \ - 'Use the notebook to score existing predictions or run native inference and local scoring.' + 'Retrieve the notebook login token with: docker compose exec mvbench-eval jupyter server list' \ + 'Open Evaluate_Cosmos3.ipynb to download public MVBench data, prepare, infer, and score.' \ + 'Full evaluation also requires manually acquired authorized NTU RGB+D media; see the notebook.' diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py index ebc19c161..4793b5fa3 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -1,67 +1,165 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Artifact safety regressions; no GPU, credentials, or network required.""" +"""Controller behavior regressions; synthetic artifacts, no GPU or network.""" import contextlib import io +import json from pathlib import Path +import subprocess +import sys import tempfile +import types import unittest +from unittest.mock import patch import launchable +def prepared(indices=(3400, 3401, 3402, 3403, 3404)): + return {"dataset_revision": "pinned-test-revision", "manifest_sha256": "trusted-hash", + "selected_indices": list(indices), "coverage": { + "expected_rows": 4000, "selected_rows": len(indices), + "available_rows": len(indices), "expected_tasks": 20, + "selected_tasks": 1, "subset": len(indices) != 4000}} + + +def table(work, indices, values=None, stamp="T20261029"): + output = work / "cosmos" / stamp + output.mkdir(parents=True) + path = output / "cosmos_MVBench.tsv" + values = values if values is not None else ["A"] * len(indices) + path.write_text("index\tprediction\n" + "".join( + f"{index}\t{value}\n" for index, value in zip(indices, values))) + return path + + class ArtifactTests(unittest.TestCase): - def test_existing_output_is_rejected_before_network_or_overwrite(self): - with tempfile.TemporaryDirectory() as temp: + def test_existing_output_rejected_before_network_or_overwrite(self): + with tempfile.TemporaryDirectory() as temp, patch.object(launchable, "prepare") as prep: with self.assertRaisesRegex(ValueError, "fresh --work-dir"): - launchable.run(["VANTAGE_VQA"], Path(temp), "infer", True, False, Path(temp) / "assets") + launchable.run(Path(temp), "infer", True, False, Path(temp) / "data") + prep.assert_not_called() - def test_nonpositive_worker_count_is_rejected_before_assets_or_output(self): - for count in (0, -1): - with self.subTest(count=count), tempfile.TemporaryDirectory() as temp: - root = Path(temp) - output = root / "fresh" - with self.assertRaisesRegex(ValueError, "positive"): - launchable.run(["VANTAGE_VQA"], output, "infer", False, False, - root / "missing-assets", api_nproc=count) + def test_nonpositive_parameters_rejected_before_output(self): + for workers, frames in ((0, 16), (-1, 16), (1, 0), (1, -1)): + with self.subTest(workers=workers, frames=frames), tempfile.TemporaryDirectory() as temp: + output = Path(temp) / "fresh" + with patch.object(launchable, "prepare") as prep: + with self.assertRaisesRegex(ValueError, "positive"): + launchable.run(output, "infer", False, False, Path(temp) / "missing", + api_nproc=workers, nframe=frames) + prep.assert_not_called() self.assertFalse(output.exists()) + def test_coverage_failure_precedes_endpoint_and_output(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) / "fresh" + with patch.object(launchable, "prepare", side_effect=ValueError("missing NTU")), \ + patch.object(launchable, "endpoint") as endpoint: + with self.assertRaisesRegex(ValueError, "missing NTU"): + launchable.run(work, "all", False, False, Path(temp) / "data") + endpoint.assert_not_called() + self.assertFalse(work.exists()) + def test_predictions_reject_empty_and_failure_markers(self): for value in (" ", "VIDEO_LOAD_FAILED", "prefix VIDEO_LOAD_FAILED", - "Failed to obtain answer via API.", "null"): + "Failed to obtain answer via API."): with self.subTest(value=value), tempfile.TemporaryDirectory() as temp: - work = Path(temp) - output = work / "cosmos" / "T20261023" - output.mkdir(parents=True) - (output / "cosmos_VANTAGE_VQA.tsv").write_text( - "index\tprediction\n0\t" + value + "\n") + path = table(Path(temp), [3400], [value]) with self.assertRaisesRegex(RuntimeError, "failed/empty"): - launchable.verify_predictions(work, "VANTAGE_VQA") - - def test_smoke_requires_exactly_five_predictions(self): - for count in (4, 5, 6): - with self.subTest(count=count), tempfile.TemporaryDirectory() as temp: - work = Path(temp) - output = work / "cosmos" / "T20261023" - output.mkdir(parents=True) - (output / "cosmos_VANTAGE_VQA.tsv").write_text( - "index\tprediction\n" + "".join(f"{i}\tA\n" for i in range(count))) - if count == 5: - with contextlib.redirect_stdout(io.StringIO()): - launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) - else: - with self.assertRaisesRegex(RuntimeError, "expected 5 predictions"): - launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) + launchable.verify_predictions(path, [3400]) + + def test_selected_identities_exact_even_when_row_count_matches(self): + for indices in ([3400, 3400], [3400, 9999], [3400], [3400, 3401, 3402]): + with self.subTest(indices=indices), tempfile.TemporaryDirectory() as temp: + path = table(Path(temp), indices) + with self.assertRaisesRegex(RuntimeError, "identities|expected 2 predictions"): + launchable.verify_predictions(path, [3400, 3401]) + def test_multiple_dated_prediction_tables_are_ambiguous(self): with tempfile.TemporaryDirectory() as temp: work = Path(temp) - for stamp in ("T20261023", "T20261024"): - output = work / "cosmos" / stamp - output.mkdir(parents=True) - (output / "cosmos_VANTAGE_VQA.tsv").write_text("index\tprediction\n0\tA\n") + table(work, [3400], stamp="T20261028") + table(work, [3400], stamp="T20261029") with self.assertRaisesRegex(RuntimeError, "expected one prediction table, found 2"): - launchable.verify_predictions(work, "VANTAGE_VQA") + launchable.verify_predictions(work, [3400]) + + def test_inference_failure_marks_state_and_keeps_artifacts(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + work = root / "fresh" + def fail(command, log_path=None): + Path(log_path).write_text("private diagnostic\n") + raise subprocess.CalledProcessError(3, command) + with patch.object(launchable, "prepare", return_value=prepared()), \ + patch.object(launchable, "endpoint"), \ + patch.object(launchable, "run_command", side_effect=fail): + with self.assertRaises(subprocess.CalledProcessError): + launchable.run(work, "all", True, False, root / "data") + self.assertEqual(json.loads((work / "run.json").read_text())["status"], "failed") + self.assertEqual((work / "run.log").read_text(), "private diagnostic\n") + self.assertTrue((work / "config.json").is_file()) + + + def test_score_output_and_scope_guards(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + native = types.ModuleType("vlmeval.mvbench_local") + native.score_predictions = unittest.mock.Mock() + with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}): + with self.assertRaisesRegex(ValueError, "explicit"): + launchable.score(root, "subset", predictions=root / "predictions.tsv", score_dir=root / "new") + with self.assertRaisesRegex(ValueError, "fresh --score-dir"): + launchable.score(root, "smoke", predictions=root / "predictions.tsv", score_dir=root) + native.score_predictions.assert_not_called() + + def test_changed_annotations_or_scope_fail_before_native_scoring(self): + for change in ("scope", "manifest_sha256", "selected_indices"): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + selection = prepared() + state = {**selection, "source": "mvbench-local", "status": "inferred", + "scope": "smoke", "nframe": 16, "predictions": "native.tsv"} + if change == "scope": + state["scope"] = "full" + elif change == "manifest_sha256": + state[change] = "different-hash" + else: + state[change] = [9999] + (root / "run.json").write_text(json.dumps(state)) + native = types.ModuleType("vlmeval.mvbench_local") + native.score_predictions = unittest.mock.Mock() + with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}), \ + patch.object(launchable, "prepare", return_value=selection): + with self.assertRaisesRegex(ValueError, "scope|changed"): + launchable.score(root / "data", "smoke", work=root) + native.score_predictions.assert_not_called() + self.assertFalse((root / "scores").exists()) + + def test_failed_prediction_never_enters_native_scorer(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path = table(root / "native", [3400], ["VIDEO_LOAD_FAILED"]) + native = types.ModuleType("vlmeval.mvbench_local") + native.score_predictions = unittest.mock.Mock() + with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}), \ + patch.object(launchable, "prepare", return_value=prepared([3400])): + with self.assertRaisesRegex(RuntimeError, "failed/empty"): + launchable.score(root / "data", "smoke", predictions=path, score_dir=root / "scores") + native.score_predictions.assert_not_called() + self.assertFalse((root / "scores").exists()) + + def test_streamed_child_output_and_failure_log_retained(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "run.log" + captured = io.StringIO() + with contextlib.redirect_stdout(captured): + with self.assertRaises(subprocess.CalledProcessError): + launchable.run_command([sys.executable, "-u", "-c", + "print('first', flush=True); raise SystemExit(7)"], log_path=path) + self.assertEqual(captured.getvalue(), "first\n") + self.assertEqual(path.read_text(), "first\n") if __name__ == "__main__": diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py deleted file mode 100644 index 4de0776f0..000000000 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_local_scoring.py +++ /dev/null @@ -1,425 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Synthetic, nonsecret local bundles exercising real native scoring behavior. - -Run from the installed VLMEvalKit environment. No GPU, network, credentials, -model endpoint, or fabricated DVC semantic scorer is used. -""" -import contextlib -import hashlib -import io -import json -from pathlib import Path -import tempfile -import unittest - -import pandas as pd -from PIL import Image - -from vlmeval.local_scoring import ( - _validate_predictions, build_local_dataset, dataset_config, load_assets, score_predictions, -) -from vlmeval.scoring_assets import EVENT_FILES, SOURCE_DIRS, VQA_FILES, prepare_assets - - -def _json(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value)) - - -def _image(path, size=(100, 100)): - path.parent.mkdir(parents=True, exist_ok=True) - Image.new('RGB', size, color='white').save(path) - - -def _video(path): - # Only file identity is used by these scoring-only fixtures. No decoder or - # inference is involved; duration is explicitly supplied by annotations. - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(b'synthetic scoring fixture: media is not decoded') - - -def _prepare(root, tasks): - with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): - return prepare_assets(root, 'synthetic-nonsecret-v1', tasks=tasks) - - -def _grounding(root, count=2): - source = root / SOURCE_DIRS['2DGrounding'] - _image(source / 'images/example.png') - rows = [] - for index in range(count): - rows.append({ - 'media': 'images/example.png', - 'conversations': [{'from': 'human', 'value': 'Locate the square.'}, - {'from': 'gpt', 'value': '[[10, 10, 50, 50]]'}], - '_meta': {'sentence': f'square {index}', 'image_width': 100, 'image_height': 100}, - }) - _json(source / 'refdrone_test_llava.json', rows) - - -def _sot(root): - source = root / SOURCE_DIRS['SOT'] - for sequence in ('first', 'second'): - directory = source / sequence - _json(directory / 'gt.json', { - 'frame_ids': [0, 10, 20], 'label': f'camera/{sequence}', - 'init_bbox': [100, 100, 500, 500], - 'gt_bboxes': {'0': [100, 100, 500, 500], '10': [100, 100, 500, 500], '20': None}, - }) - for name in ('f00.png', 'f01.png', 'f02.png', 'crop.png'): - _image(directory / 'frames' / name) - - -def _astro(root): - from vlmeval.dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset - for child, layout in Astro2DDetectionDataset._LAYOUT.items(): - source = root / SOURCE_DIRS['Astro2D'] / layout['data_root'].rsplit('/', 1)[1] - # Unequal child sizes make micro-vs-sequence-macro mistakes observable. - count = 2 if child.endswith('_C0065') else 1 - for index in range(count): - _image(source / 'images' / f'{index}.png') - labels = source / 'labels' / f'{index}.txt' - labels.parent.mkdir(parents=True, exist_ok=True) - labels.write_text('Pedestrian 0 0 0 10 10 50 50 0 0 0 0 0 0 0\n') - - -def _events(root): - source = root / SOURCE_DIRS['EventVerification'] - for index, relative in enumerate(EVENT_FILES): - annotation = source / relative - _video(annotation.parent / 'clip.mp4') - _json(annotation, {'bcq': [{'id': f'item-{index}', 'video': 'clip.mp4', - 'question': 'Is the square visible?', 'answer': 'yes', - 'system_prompt': 'Answer yes or no.'}]}) - - -def _vqa(root): - source = root / SOURCE_DIRS['VQA'] - files = [name for name in VQA_FILES if not name.startswith('VANTAGE_')] - for index, name in enumerate(files): - video = f'clip-{index}' - _video(source / 'videos' / (video + '.mp4')) - _json(source / 'data_jsons/annotations' / name, [{ - 'q_uid': video, 'question_id': f'q-{index}', 'question': 'Choose the square.', - 'options': ['square', 'circle', 'triangle', 'star'], 'gt_option': 'A', - }]) - - -def _temporal(root): - source = root / SOURCE_DIRS['Temporal'] - rows = [] - for video, question, answer in ( - ('first', 'q1', '<1><3>'), ('first', 'q2', '<1><3>'), ('second', 'q3', '<1><3>')): - _video(source / 'videos' / (video + '.mp4')) - rows.append({'vid': video, 'question_id': question, 'question': question, - 'answer': answer, 'duration': 10}) - _json(source / 'data_jsons/annotations/examples.json', rows) - - -def _pointing(root): - source = root / SOURCE_DIRS['2DPointing'] - _image(source / 'images/example.png') - pd.DataFrame([ - {'index': index, 'question_id': f'point-{index}', 'question': 'Choose the marked point.', - 'answer': answer, 'target_point': '[20,30]', 'image_path': 'images/example.png', - 'A': 'upper left', 'B': 'upper right', 'C': 'lower left', 'D': 'lower right'} - for index, answer in ((10, 'A'), (11, 'B')) - ]).to_csv(source / 'Metropolis2DPointing.tsv', sep='\t', index=False) - - -class LocalScoringTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - self.assets = self.root / 'assets' - self.assets.mkdir() - - def predictions(self, task, values, name='predictions.tsv'): - table = build_local_dataset(self.assets, task).data.copy() - table['prediction'] = values - path = self.root / name - table.to_csv(path, sep='\t', index=False) - return path - - def score(self, task, path, name='result', **kwargs): - return score_predictions(task, path, self.assets, self.root / name, **kwargs) - - def test_grounding_ignores_injected_gt_paths_and_reports_native_percent(self): - _grounding(self.assets) - _prepare(self.assets, ['2DGrounding']) - path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) - table = pd.read_csv(path, sep='\t') - table['gt_bboxes'] = '[[70,70,90,90]]' - table['image_path'] = '/untrusted/does-not-exist.png' - table['image_width'] = 98765 - table['image_height'] = 98765 - table.to_csv(path, sep='\t', index=False) - diagnostic = io.StringIO() - with contextlib.redirect_stdout(diagnostic), contextlib.redirect_stderr(diagnostic): - summary = self.score('2DGrounding', path) - self.assertEqual(diagnostic.getvalue(), '') - self.assertEqual(summary['primary_metric']['value'], 100.0) - self.assertEqual(summary['primary_metric']['unit'], 'percent') - self.assertEqual(summary['native_metrics']['Mean_IoU'], 100.0) - scored = pd.read_csv(self.root / 'result/predictions_VANTAGE_2DGrounding.tsv', sep='\t') - self.assertTrue(scored['image_path'].map(lambda value: Path(value).is_file()).all()) - self.assertEqual(summary['provenance']['predictions_sha256'], hashlib.sha256(path.read_bytes()).hexdigest()) - self.assertEqual(json.loads((self.root / 'result/summary.json').read_text()), summary) - - def test_partial_requires_explicit_subset_and_never_claims_full_coverage(self): - _grounding(self.assets) - _prepare(self.assets, ['2DGrounding']) - path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) - table = pd.read_csv(path, sep='\t').iloc[:1] - table.to_csv(path, sep='\t', index=False) - with self.assertRaisesRegex(ValueError, 'Full local dataset coverage'): - self.score('2DGrounding', path) - self.assertFalse((self.root / 'result').exists()) - summary = self.score('2DGrounding', path, allow_subset=True) - self.assertEqual(summary['coverage'], {'expected_rows': 2, 'scored_rows': 1, 'subset': True}) - - def test_duplicate_unknown_and_mismatched_identity_fail_before_output(self): - _grounding(self.assets) - _prepare(self.assets, ['2DGrounding']) - path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) - original = pd.read_csv(path, sep='\t') - for kind in ('duplicate', 'unknown', 'mismatch', 'missing_identity'): - with self.subTest(kind=kind): - table = original.copy() - if kind == 'duplicate': - table.loc[1, 'index'] = table.loc[0, 'index'] - elif kind == 'unknown': - table.loc[1, 'index'] = 999 - elif kind == 'mismatch': - table.loc[1, 'sentence'] = 'another example' - else: - table = table.drop(columns=['sentence']) - table.to_csv(path, sep='\t', index=False) - with self.assertRaises(ValueError): - self.score('2DGrounding', path) - self.assertFalse((self.root / 'result').exists()) - - def test_native_sot_uses_integer_cache_and_absent_frame_semantics(self): - _sot(self.assets) - _prepare(self.assets, ['SOT']) - predictions = [json.dumps({'frame_1': [100, 100, 500, 500], 'frame_2': None}), - json.dumps({'frame_1': None, 'frame_2': None})] - path = self.predictions('SOT', predictions) - summary = self.score('SOT', path) - # Native threshold grid includes zero: a missed visible frame has AUC - # 1/21, not exactly zero. The absent frame is excluded, not rewarded. - self.assertAlmostEqual(summary['native_metrics']['camera/first'], 1.0) - self.assertAlmostEqual(summary['native_metrics']['camera/second'], 1 / 21) - self.assertAlmostEqual(summary['primary_metric']['value'], (1 + 1 / 21) / 2) - self.assertEqual(summary['primary_metric']['unit'], 'fraction') - - def test_sot_missing_gt_frame_is_rejected_even_with_updated_fingerprint(self): - _sot(self.assets) - _prepare(self.assets, ['SOT']) - path = self.assets / SOURCE_DIRS['SOT'] / 'first/gt.json' - annotation = json.loads(path.read_text()) - del annotation['gt_bboxes']['20'] - path.write_text(json.dumps(annotation)) - manifest_path = self.assets / 'scoring-assets.json' - manifest = json.loads(manifest_path.read_text()) - manifest['files'][path.relative_to(self.assets).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() - manifest_path.write_text(json.dumps(manifest)) - with self.assertRaises(ValueError): - load_assets(self.assets, ['SOT']) - - def test_astro_distinct_roots_and_unweighted_native_sequence_macro(self): - _astro(self.assets) - _prepare(self.assets, ['Astro2D']) - config = dataset_config(self.assets, 'Astro2D') - self.assertEqual(len(set(config['data_roots'].values())), 3) - dataset = build_local_dataset(self.assets, 'Astro2D') - values = ['[{"bbox_2d":[100,100,500,500],"label":"person"}]' - if row['SUB_DATASET'].endswith('_C0065') else '[]' - for _, row in dataset.data.iterrows()] - path = self.predictions('Astro2D', values) - # Including VANTAGE_Astro2D in a parent directory must not corrupt the - # native child file paths (ConcatDataset's historical string trap). - summary = self.score('Astro2D', path, name='VANTAGE_Astro2D-output') - self.assertAlmostEqual(summary['primary_metric']['value'], 100 / 3) - self.assertEqual(summary['primary_metric']['aggregation'], 'unweighted_sequence_macro') - self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:f1'], 100) - self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_IVAKPI05:f1'], 0) - - def test_astro_inverted_source_boxes_remain_in_native_denominator(self): - from vlmeval.dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset - _astro(self.assets) - layout = Astro2DDetectionDataset._LAYOUT['VANTAGE_Astro2D_C0065'] - label = self.assets / SOURCE_DIRS['Astro2D'] / layout['data_root'].rsplit('/', 1)[1] / 'labels/0.txt' - label.write_text(label.read_text() + 'Person 0 0 0 50 10 10 50 0 0 0 0 0 0 0\n') - _prepare(self.assets, ['Astro2D']) - count = len(build_local_dataset(self.assets, 'Astro2D').data) - path = self.predictions('Astro2D', ['[{"bbox_2d":[100,100,500,500],"label":"person"}]'] * count) - summary = self.score('Astro2D', path) - self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:total_gt_objects'], 3) - self.assertEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:false_negatives'], 1) - self.assertAlmostEqual(summary['native_metrics']['VANTAGE_Astro2D_C0065:f1'], 80) - self.assertAlmostEqual(summary['primary_metric']['value'], (80 + 100 + 100) / 3) - - def test_event_invalid_predictions_are_not_silently_removed(self): - _events(self.assets) - _prepare(self.assets, ['EventVerification']) - path = self.predictions('EventVerification', ['yes', 'yes', 'yes', 'unparseable']) - with self.assertRaisesRegex(ValueError, 'silent fallback'): - self.score('EventVerification', path) - self.assertFalse((self.root / 'result').exists()) - path = self.predictions('EventVerification', ['yes'] * 4) - summary = self.score('EventVerification', path) - self.assertEqual(summary['native_metrics']['Valid Predictions'], 4) - self.assertEqual(summary['primary_metric']['value'], 1) - - def test_vqa_uses_local_ground_truth_not_prediction_answer(self): - _vqa(self.assets) - _prepare(self.assets, ['VQA']) - dataset = build_local_dataset(self.assets, 'VQA') - path = self.predictions('VQA', ['B'] * len(dataset.data)) - table = pd.read_csv(path, sep='\t') - table['answer'] = 'B' - table.to_csv(path, sep='\t', index=False) - summary = self.score('VQA', path) - self.assertEqual(summary['primary_metric']['value'], 0) - - def test_vqa_binary_explicit_label_is_normalized_without_defaulting(self): - _vqa(self.assets) - source = self.assets / SOURCE_DIRS['VQA'] / 'data_jsons/annotations' / VQA_FILES[0] - records = json.loads(source.read_text()) - records[0].update(options=['square', 'circle'], gt_option=' b ') - _json(source, records) - _prepare(self.assets, ['VQA']) - count = len(build_local_dataset(self.assets, 'VQA').data) - summary = self.score('VQA', self.predictions('VQA', ['A'] * count)) - self.assertEqual(summary['primary_metric']['value'], (count - 1) / count) - - def test_vqa_reused_fallback_qid_cannot_hide_wrong_question(self): - _vqa(self.assets) - source = self.assets / SOURCE_DIRS['VQA'] - _video(source / 'videos/shared.mp4') - _json(source / 'data_jsons/annotations' / VQA_FILES[0], [ - {'q_uid': 'shared.mp4', 'question': question, - 'options': ['square', 'circle', 'triangle', 'star'], 'gt_option': answer} - for question, answer in [('Choose the square.', 'A'), ('Choose the circle.', 'B')] - ]) - _prepare(self.assets, ['VQA']) - count = len(build_local_dataset(self.assets, 'VQA').data) - path = self.predictions('VQA', ['A'] * count) - table = pd.read_csv(path, sep='\t') - shared = table.index[table['video'] == 'shared.mp4'] - self.assertEqual(table.loc[shared, 'qid'].nunique(), 1) - table.loc[shared, 'question'] = table.loc[shared, 'question'].tolist()[::-1] - table.to_csv(path, sep='\t', index=False) - with self.assertRaisesRegex(ValueError, 'stable identity'): - self.score('VQA', path) - self.assertFalse((self.root / 'result').exists()) - - def test_temporal_preserves_video_macro_not_question_micro(self): - _temporal(self.assets) - _prepare(self.assets, ['Temporal']) - dataset = build_local_dataset(self.assets, 'Temporal') - values = [json.dumps({'start': '1', 'end': '3'}) if row['video'] == 'first' - else json.dumps({'start': '5', 'end': '7'}) for _, row in dataset.data.iterrows()] - path = self.predictions('Temporal', values) - summary = self.score('Temporal', path) - self.assertEqual(summary['primary_metric']['value'], 0.5) - self.assertEqual(summary['native_metrics']['overall']['count'], 3) - - def test_temporal_malformed_prediction_cannot_receive_full_duration_default(self): - _temporal(self.assets) - _prepare(self.assets, ['Temporal']) - path = self.predictions('Temporal', ['no parseable interval'] * 3) - with self.assertRaisesRegex(ValueError, 'silent fallback'): - self.score('Temporal', path) - self.assertFalse((self.root / 'result').exists()) - - def test_pointing_uses_native_accuracy_and_rejects_textual_id_alias(self): - _pointing(self.assets) - _prepare(self.assets, ['2DPointing']) - path = self.predictions('2DPointing', ['A', 'D']) - table = pd.read_csv(path, sep='\t') - table['answer'] = ['A', 'D'] - table.to_csv(path, sep='\t', index=False) - summary = self.score('2DPointing', path) - self.assertEqual(summary['primary_metric']['value'], 0.5) - self.assertEqual(summary['primary_metric']['unit'], 'fraction') - table['index'] = ['010', '11'] - table.to_csv(path, sep='\t', index=False) - with self.assertRaisesRegex(ValueError, 'identity does not match'): - self.score('2DPointing', path, name='alias') - - def test_asset_hash_and_escape_and_existing_output_are_rejected(self): - _grounding(self.assets) - _prepare(self.assets, ['2DGrounding']) - path = self.predictions('2DGrounding', ['[[100,100,500,500]]'] * 2) - (self.root / 'existing').mkdir() - with self.assertRaisesRegex(ValueError, 'fresh output'): - self.score('2DGrounding', path, name='existing') - manifest_path = self.assets / 'scoring-assets.json' - original = manifest_path.read_text() - manifest = json.loads(original) - annotation = self.assets / SOURCE_DIRS['2DGrounding'] / 'refdrone_test_llava.json' - annotation.write_text(annotation.read_text() + '\n') - with self.assertRaisesRegex(ValueError, 'fingerprint mismatch'): - load_assets(self.assets, ['2DGrounding']) - manifest['tasks']['2DGrounding']['dataset_kwargs']['data_root'] = '../' - manifest['files'][annotation.relative_to(self.assets).as_posix()] = hashlib.sha256(annotation.read_bytes()).hexdigest() - manifest_path.write_text(json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, 'escapes'): - load_assets(self.assets, ['2DGrounding']) - - def test_temporal_zero_interval_reduces_score_without_dropping_row(self): - _temporal(self.assets) - _prepare(self.assets, ['Temporal']) - dataset = build_local_dataset(self.assets, 'Temporal') - values = [ - json.dumps({'start': '0', 'end': '0'} if position == 0 else {'start': '1', 'end': '3'}) - for position, (_, row) in enumerate(dataset.data.iterrows()) - ] - summary = self.score('Temporal', self.predictions('Temporal', values)) - self.assertEqual(summary['primary_metric']['value'], 0.75) - self.assertEqual(summary['native_metrics']['overall']['count'], 3) - self.assertEqual(summary['coverage'], {'expected_rows': 3, 'scored_rows': 3, 'subset': False}) - - def test_dvc_degenerate_intervals_retain_native_zero_iou_without_fallback(self): - from vlmeval.dataset.metropolis_dvc import MetropolisDVC, _iou - - rows = pd.DataFrame({'prediction': [ - json.dumps([{'start': start, 'end': end, 'caption': 'A square moves.'}]) - for start, end in [('0', '0'), ('3', '1'), ('1', '3')] - ]}) - _validate_predictions('DVC', MetropolisDVC, rows) - overlaps = [] - for text in rows['prediction']: - event, = MetropolisDVC.parse_events_from_json(text) - interval = [MetropolisDVC.parse_timestamp(event[key]) for key in ('start', 'end')] - overlaps.append(_iou(interval, [1, 3])) - self.assertEqual(overlaps, [0.0, 0.0, 1.0]) - self.assertEqual(sum(overlaps) / len(overlaps), 1 / 3) - for event in ( - {'start': 'nan', 'end': '3', 'caption': 'A square moves.'}, - {'start': '1', 'caption': 'A square moves.'}, - {'start': '1', 'end': '3', 'caption': ''}, - ): - with self.subTest(event=event), self.assertRaisesRegex(ValueError, 'silent fallback'): - _validate_predictions('DVC', MetropolisDVC, - pd.DataFrame({'prediction': [json.dumps([event])]})) - - def test_dvc_without_genuine_local_model_is_rejected(self): - from vlmeval.scoring_assets import validate_bert_score_model - empty_model = self.assets / 'model' - empty_model.mkdir() - with self.assertRaises((ValueError, FileNotFoundError)): - validate_bert_score_model(empty_model) - # Preparation must not create a manifest or silently use a remote or - # dummy semantic scorer when a local checkpoint was not provisioned. - with self.assertRaises(ValueError): - _prepare(self.assets, ['DVC']) - self.assertFalse((self.assets / 'scoring-assets.json').exists()) - - -if __name__ == '__main__': - unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py new file mode 100644 index 000000000..4c31c4373 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py @@ -0,0 +1,368 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic consumer-boundary tests; no network, GPU, or private media. + +The fixture trust root is patched to hashes of synthetic annotations, never to +bypass authentication. Counts, native task ordering, preparation, image reading, +native exact matching, and extraction all execute their real implementations. +""" +import contextlib +import io +import json +from pathlib import Path +import stat +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +import pandas as pd +from PIL import Image + +from vlmeval import mvbench_local as local +from vlmeval.dataset.mvbench import MVBench +from vlmeval.smp import get_intermediate_file_path + + +def _write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding='utf-8') + + +class MVBenchTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.data = self.root / 'data' + self.hashes = {} + self.sources = {} + for task, (name, prefix, data_type, bound) in MVBench.TASKS.items(): + rows = [] + for index in range(200): + row = { + 'video': 'fixture' if data_type == 'frame' else 'fixture.mp4', + 'question': f'Choose the shape in synthetic {task} question {index}.', + 'candidates': ['red square', 'blue circle', 'green triangle'], + 'answer': ['red square', 'blue circle', 'green triangle'][index % 3], + } + if bound: + row.update(start=0, end=2) + rows.append(row) + self.sources[name] = rows + path = self.data / 'json' / name + _write_json(path, rows) + self.hashes[f'json/{name}'] = local._sha256(path) + for name in ('README.md', 'video/MVBench_videos_ntu.txt'): + path = self.data / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('Synthetic public-input fixture, no actual dataset media.\n') + self.hashes[name] = local._sha256(path) + self.trust = patch.dict(local.METADATA_SHA256, self.hashes, clear=True) + self.trust.start() + self.addCleanup(self.trust.stop) + self._media(local.SMOKE_TASK) + + def _media(self, task): + name, prefix, data_type, bound = MVBench.TASKS[task] + path = self.data / prefix.replace('your_data_path', 'video') / self.sources[name][0]['video'] + if data_type == 'frame': + path.mkdir(parents=True, exist_ok=True) + for index in range(1, 10): + color = (255, 0, 0) if index <= 3 else (0, 0, 255) + Image.new('RGB', (320, 240), color).save(path / f'{index:05d}.jpg') + else: + path.parent.mkdir(parents=True, exist_ok=True) + # Scoring/coverage fixtures need identity and nonempty media, not a + # decoder. The prompt test below uses real numbered JPEG inputs. + path.write_bytes(b'Synthetic scoring-only media identity; not decoded.') + return path + + def _repin(self, name): + path = self.data / 'json' / name + _write_json(path, self.sources[name]) + local.METADATA_SHA256[f'json/{name}'] = local._sha256(path) + + def _table(self): + prepared = local.prepare_dataset(self.data, smoke=True) + table = pd.read_csv(prepared['data_file'], sep='\t') + table['prediction'] = ['(A)', '(B)', '(C)', '(A)', '(B)'] + return table + + def _predictions(self, table=None, name='predictions.tsv'): + path = self.root / name + (self._table() if table is None else table).to_csv(path, sep='\t', index=False) + return path + + def _score(self, predictions=None, name='results'): + path = self._predictions() if predictions is None else predictions + with contextlib.redirect_stdout(io.StringIO()): + return local.score_predictions(path, self.data, self.root / name, smoke=True) + + def test_smoke_is_fixed_subset_and_never_a_full_score(self): + prepared = local.prepare_dataset(self.data, smoke=True) + self.assertEqual(prepared['selected_indices'], list(range(3400, 3405))) + self.assertEqual(prepared['coverage']['expected_rows'], 4000) + self.assertEqual(prepared['coverage']['available_rows'], 200) + self.assertEqual(prepared['coverage']['selected_rows'], 5) + self.assertEqual(prepared['coverage']['selected_tasks'], ['Egocentric Navigation']) + self.assertTrue(prepared['coverage']['subset']) + with self.assertRaises(local.MVBenchCoverageError) as caught: + local.prepare_dataset(self.data) + self.assertIn('4000', str(caught.exception)) + self.assertIn('video/nturgbd', str(caught.exception)) + self.assertIn('Fine-grained Pose', caught.exception.report['missing_media_by_task']) + self.assertTrue((self.data / '.mvbench/coverage-full.json').is_file()) + summary = self._score() + self.assertEqual(summary['status'], 'smoke_only') + self.assertEqual(summary['primary_metric']['value'], 1.0) + self.assertEqual(summary['primary_metric']['total'], 5) + self.assertFalse(summary['provenance']['paper_parity']) + self.assertEqual(json.loads((self.root / 'results/summary.json').read_text()), summary) + + def test_full_requires_manual_ntu_and_does_not_drop_its_rows(self): + for task in MVBench.TASKS: + if task != 'Fine-grained Pose': + self._media(task) + with self.assertRaises(local.MVBenchCoverageError) as caught: + local.prepare_dataset(self.data) + self.assertEqual(caught.exception.report['coverage']['available_rows'], 3800) + self.assertEqual(set(caught.exception.report['missing_media_by_task']), {'Fine-grained Pose'}) + self._media('Fine-grained Pose') + full = local.prepare_dataset(self.data) + self.assertEqual(full['coverage']['available_rows'], 4000) + self.assertEqual(full['coverage']['selected_rows'], 4000) + self.assertEqual(len(full['coverage']['selected_tasks']), 20) + self.assertFalse(full['coverage']['subset']) + + def test_source_metadata_and_prebuilt_tsv_cannot_replace_ground_truth(self): + prepared = local.prepare_dataset(self.data, smoke=True) + canonical = Path(prepared['data_file']) + canonical.write_text('index\tanswer\n3400\tforged\n') + repaired = local.prepare_dataset(self.data, smoke=True) + self.assertEqual(len(pd.read_csv(repaired['data_file'], sep='\t')), 5) + source = self.data / 'json/egocentric_navigation.json' + source.write_text(source.read_text().replace('red square', 'forged square')) + with self.assertRaisesRegex(ValueError, 'checksum mismatch'): + local.prepare_dataset(self.data, smoke=True) + + def test_media_content_mutation_changes_manifest_not_just_path(self): + first = local.prepare_dataset(self.data, smoke=True) + video = self.data / next(iter(first['media_sha256'])) + video.write_bytes(b'Changed synthetic media at the same path.') + second = local.prepare_dataset(self.data, smoke=True) + self.assertNotEqual(first['media_manifest_sha256'], second['media_manifest_sha256']) + self.assertNotEqual(first['manifest_sha256'], second['manifest_sha256']) + self.assertEqual(first['annotation_manifest_sha256'], second['annotation_manifest_sha256']) + + def test_missing_bad_answer_and_changed_official_row_count_are_rejected(self): + name = 'egocentric_navigation.json' + for answer in (None, 'not a candidate'): + with self.subTest(answer=answer): + self.sources[name][0]['answer'] = answer + self._repin(name) + with self.assertRaisesRegex(ValueError, 'Answer must identify'): + local.prepare_dataset(self.data, smoke=True) + self.sources[name][0]['answer'] = 'red square' + self.sources[name].pop() + self._repin(name) + with self.assertRaisesRegex(ValueError, 'exactly 200'): + local.prepare_dataset(self.data, smoke=True) + + def test_empty_and_missing_media_cannot_enter_scoring(self): + video = self._media(local.SMOKE_TASK) + for remove in (False, True): + with self.subTest(remove=remove): + if remove: + video.unlink() + else: + video.write_bytes(b'') + with self.assertRaises(local.MVBenchCoverageError): + local.prepare_dataset(self.data, smoke=True) + + def test_legitimate_one_and_five_candidate_questions_are_not_rewritten(self): + name = 'egocentric_navigation.json' + self.sources[name][0].update(candidates=['red square'], answer='red square') + self.sources[name][1].update( + candidates=['red square', 'blue circle', 'green triangle', 'yellow star', 'purple pentagon'], + answer='purple pentagon', + ) + self._repin(name) + table = self._table() + table.loc[1, 'prediction'] = '(E)' + summary = self._score(self._predictions(table)) + self.assertEqual(summary['primary_metric']['correct'], 5) + + def test_prediction_identity_coverage_is_exact_not_just_row_count(self): + original = self._table() + cases = { + 'missing': original.iloc[:-1].copy(), + 'duplicate': pd.concat([original.iloc[:-1], original.iloc[:1]], ignore_index=True), + 'extra': pd.concat([original, original.iloc[:1].assign(index=99999)], ignore_index=True), + 'substituted': original.assign(index=[3400, 3401, 3402, 3403, 99999]), + } + for name, table in cases.items(): + with self.subTest(name=name): + with self.assertRaisesRegex(ValueError, 'coverage must match'): + self._score(self._predictions(table, name + '.tsv'), name) + self.assertFalse((self.root / name / 'summary.json').exists()) + reordered = original.iloc[::-1] + summary = self._score(self._predictions(reordered, 'reordered.tsv')) + self.assertEqual(summary['primary_metric']['correct'], 5) + + def test_tampered_identity_labels_candidates_and_prompt_are_rejected(self): + marker = self.root / 'executed' + changes = { + 'answer': 'forged answer', 'video': '../elsewhere.mp4', + 'task_type': 'Action Sequence', 'question': 'forged question', + 'candidates': f'__import__("pathlib").Path({str(marker)!r}).touch()', + } + for column, value in changes.items(): + with self.subTest(column=column): + table = self._table() + table.loc[0, column] = value + with self.assertRaisesRegex(ValueError, 'Untrusted'): + self._score(self._predictions(table, f'{column}.tsv'), column) + self.assertFalse(marker.exists()) + # Only index/prediction is also valid; all answer identities are rebuilt. + summary = self._score(self._predictions(self._table()[['index', 'prediction']], 'minimal.tsv')) + self.assertEqual(summary['primary_metric']['correct'], 5) + + def test_failure_rows_never_receive_truthy_minus_one_credit(self): + for prediction in ('', ' ', 'Failed to obtain answer via API.', 'VIDEO_LOAD_FAILED', + 'Failed to obtain answer: RuntimeError synthetic failure'): + with self.subTest(prediction=prediction): + table = self._table() + table.loc[0, 'prediction'] = prediction + with self.assertRaisesRegex(ValueError, 'Missing or failed prediction'): + self._score(self._predictions(table), 'failed') + self.assertFalse((self.root / 'failed/summary.json').exists()) + + def test_existing_score_output_is_preserved(self): + output = self.root / 'existing' + output.mkdir() + sentinel = output / 'summary.json' + sentinel.write_text('previous result') + with self.assertRaisesRegex(ValueError, 'fresh output'): + local.score_predictions(self._predictions(), self.data, output, smoke=True) + self.assertEqual(sentinel.read_text(), 'previous result') + + def test_nonempty_malformed_predictions_are_native_wrong_answers_offline(self): + table = self._table() + # These strings must not become pandas NA or numeric scalars in native + # scoring. Native model=None deterministically gives unknown option Z. + table['prediction'] = ['nan', '1', '.', '(', ')'] + with patch('vlmeval.dataset.mvbench.build_judge', side_effect=AssertionError('network judge forbidden')): + summary = self._score(self._predictions(table)) + self.assertEqual(summary['primary_metric']['correct'], 0) + self.assertEqual(summary['primary_metric']['total'], 5) + self.assertFalse(summary['provenance']['network_judge']) + scores = pd.read_csv(self.root / 'results/scored_predictions.tsv', sep='\t') + self.assertEqual(scores['score'].tolist(), [0] * 5) + self.assertEqual(scores['index'].tolist(), list(range(3400, 3405))) + + def test_native_micro_accuracy_is_not_task_macro_for_unequal_sizes(self): + table = pd.DataFrame([ + {'index': index, 'task_type': 'large' if index < 3 else 'small', + 'question': 'Choose a shape.', 'answer': 'red square', + 'candidates': repr(['red square', 'blue circle']), + 'prediction': '(A)' if index < 3 else '(B)'} + for index in range(4) + ]) + native_file = str(self.root / 'unequal.tsv') + table.to_csv(native_file, sep='\t', index=False) + with contextlib.redirect_stdout(io.StringIO()): + rating = MVBench.evaluate(native_file, trusted_data=table, model='exact_matching') + summary = local._summarize_rating(rating) + self.assertEqual(summary['primary_metric']['value'], 0.75) + self.assertEqual(summary['task_macro_accuracy'], 0.5) + self.assertEqual(summary['per_task']['large']['total'], 3) + self.assertTrue(Path(get_intermediate_file_path(native_file, '_score')).exists()) + + def test_native_image_prompt_respects_clip_bounds_cache_and_reference_crop(self): + frames = self._media('Episodic Reasoning') + dataset = local.build_dataset(self.data, nframe=4, smoke=True) + line = dict(dataset.data.iloc[0]) + line.update(prefix=str(frames.parent.relative_to(self.data)), video=frames.name, + data_type='frame', bound=True, start=0, end=1) + first = dataset.build_prompt(line, video_llm=True) + first_images = [part['value'] for part in first if part['type'] == 'image'] + self.assertEqual(len(first_images), 4) + self.assertFalse(any(part['type'] == 'video' for part in first)) + self.assertIn('Question: Choose the shape', first[-3]['value']) + self.assertEqual(first[-1]['value'], 'Best option:(') + for path in first_images: + with Image.open(path) as image: + self.assertEqual(image.size, (224, 224)) + line.update(start=1, end=2) + second = dataset.build_prompt(line, video_llm=True) + second_images = [part['value'] for part in second if part['type'] == 'image'] + self.assertTrue(set(first_images).isdisjoint(second_images)) + with Image.open(first_images[-1]) as left, Image.open(second_images[-1]) as right: + self.assertNotEqual(left.getpixel((100, 100)), right.getpixel((100, 100))) + self.assertEqual(dataset.save_video_into_images(line), second_images) + key = dataset.clip_cache_key(line) + dataset.nframe = 8 + self.assertNotEqual(dataset.clip_cache_key(line), key) + dataset.nframe = 4 + # Mutate a numbered source frame, not the path or clip bounds. + Image.new('RGB', (320, 240), (0, 255, 0)).save(frames / '00005.jpg') + self.assertNotEqual(dataset.clip_cache_key(line), key) + changed_images = dataset.save_video_into_images(line) + self.assertTrue(set(changed_images).isdisjoint(second_images)) + alternate = frames.parent / 'different-source' / frames.name + alternate.mkdir(parents=True) + for frame in frames.glob('*.jpg'): + (alternate / frame.name).write_bytes(frame.read_bytes()) + line['prefix'] += '/different-source' + self.assertNotEqual(dataset.clip_cache_key(line), dataset.clip_cache_key({ + **line, 'prefix': str(frames.parent.relative_to(self.data)), + })) + + def test_extraction_validates_every_member_before_writing(self): + for bad_name in ('../escape.txt', '/absolute.txt', 'C:/drive.txt', 'folder\\escape.txt'): + with self.subTest(name=bad_name): + archive = self.root / 'unsafe.zip' + with zipfile.ZipFile(archive, 'w') as source: + source.writestr('innocent.txt', 'must not be extracted either') + source.writestr(bad_name, 'unsafe') + destination = self.root / 'extract' + with self.assertRaisesRegex(ValueError, 'Unsafe relative path'): + local._safe_extract(archive, destination) + self.assertFalse((destination / 'innocent.txt').exists()) + self.assertFalse((self.root / 'escape.txt').exists()) + + def test_zip_symlinks_and_preexisting_escape_links_are_rejected(self): + archive = self.root / 'symlink.zip' + link = zipfile.ZipInfo('link') + link.create_system = 3 + link.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(archive, 'w') as source: + source.writestr(link, '../escape') + with self.assertRaisesRegex(ValueError, 'member type'): + local._safe_extract(archive, self.root / 'extract') + destination = self.root / 'extract' + outside = self.root / 'outside' + outside.mkdir() + (destination / 'linked').symlink_to(outside, target_is_directory=True) + with zipfile.ZipFile(archive, 'w') as source: + source.writestr('linked/escape.txt', 'unsafe') + with self.assertRaisesRegex(ValueError, 'escapes dataset root'): + local._safe_extract(archive, destination) + self.assertFalse((outside / 'escape.txt').exists()) + + def test_june_overlay_overwrites_only_its_unwrapped_media_path(self): + destination = self.root / 'video' + target = destination / 'scene_qa/video/fixture.mp4' + target.parent.mkdir(parents=True) + target.write_bytes(b'old') + archive = self.root / 'data0613.zip' + with zipfile.ZipFile(archive, 'w') as source: + source.writestr('data0613/scene_qa/video/fixture.mp4', b'corrected') + local._safe_extract(archive, destination, overlay=True) + self.assertEqual(target.read_bytes(), b'corrected') + self.assertFalse((destination / 'data0613').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py index bcfd45442..9f8e713d5 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py @@ -125,7 +125,7 @@ from .moat import MOAT from .moviechat1k import MovieChat1k from .mssbench import MSSBenchDataset -from .mvbench import MVBench, MVBench_MP4 +from .mvbench import MVBench, MVBenchLocal, MVBench_MP4 from .mvpbench import MVPBench from .mvu_eval import MVUEval from .NPMM import NPMM diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py index 1230daf9c..f69d3941c 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis2d/astro_2d_dataset.py @@ -588,36 +588,6 @@ class Astro2DBenchDataset(concat_base): ], } - def __init__(self, dataset='VANTAGE_Astro2D', data_roots=None, **kwargs): - if data_roots is None: - super().__init__(dataset=dataset, **kwargs) - return - children = self.DATASET_SETS[dataset] - if set(data_roots) != set(children): - raise ValueError('data_roots must specify every registered ASTRO child') - self.dataset_name = dataset - self.datasets = children - self.dataset_map = {} - tables = [] - for name in children: - root = os.fspath(data_roots[name]) - if not os.path.isdir(root): - raise ValueError('ASTRO data_roots must be existing local directories') - child = Astro2DDetectionDataset( - dataset=name, data_root=root, - min_bbox_area=400 if name.endswith('_IVAKPI05') else 0, - **kwargs, - ) - self.dataset_map[name] = child - table = child.data.copy() - table['SUB_DATASET'] = name - tables.append(table) - self.data = pd.concat(tables, ignore_index=True) - self.data['original_index'] = self.data.pop('index') - self.data['index'] = np.arange(len(self.data)) - self.TYPE = next(iter(self.dataset_map.values())).TYPE - self.MODALITY = next(iter(self.dataset_map.values())).MODALITY - def evaluate(self, eval_file, **judge_kwargs): result = super().evaluate(eval_file, **judge_kwargs) # f1 (F1@0.5) first so dict iteration / Format-D parsers pick it diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py index 8b2ef8f3f..c022ebf1c 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_dvc.py @@ -113,19 +113,15 @@ class MetropolisDVC(VideoBaseDataset): def __init__(self, dataset='MetropolisDVC', pack=False, nframe=0, fps=0, total_pixels=8192 * 32 * 32, max_pixels=None, max_frames=128, test_mode=False, limit=None, - random_state=None, include_categories=None, local_data_file=None, local_video_root=None): + random_state=None, include_categories=None): self.test_mode = test_mode - self.local_data_file = local_data_file - self.local_video_root = local_video_root - if (local_data_file is None) != (local_video_root is None): - raise ValueError("local_data_file and local_video_root must be supplied together") self.limit = limit self.max_pixels = max_pixels self.max_frames = max_frames self.random_state = random_state self.include_categories = set(include_categories) if include_categories else None - if not test_mode or local_data_file is not None: + if not test_mode: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -328,10 +324,6 @@ def prepare_dataset(self, dataset_name='MetropolisDVC'): - category: category label - qid: question ID """ - if self.local_data_file is not None: - from ..scoring_assets import validate_local_video_inputs - return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'DVC') - if dataset_name == 'VANTAGE_DVC': return self._prepare_vantage_from_s3(dataset_name) @@ -487,19 +479,6 @@ def evaluate(self, eval_file, **judge_kwargs): - Temporal IoU for localization accuracy - BERTScore for caption semantic similarity """ - local_scoring = judge_kwargs.get('local_scoring', False) - local_model = judge_kwargs.get('bert_score_model') - if local_scoring: - if judge_kwargs.get('bert_score_endpoint'): - raise ValueError("Local DVC scoring does not permit a BERTScore endpoint") - from ..scoring_assets import validate_bert_score_model - local_model = str(validate_bert_score_model(local_model)) - try: - import torch - from bert_score import BERTScorer - except ImportError as exc: - raise ImportError("Local DVC scoring requires torch and bert-score") from exc - data = load(eval_file) # Convert data to SODA format @@ -555,11 +534,6 @@ def evaluate(self, eval_file, **judge_kwargs): if gt_timestamps: gts[vid] = {"timestamps": gt_timestamps, "sentences": gt_sentences} - if local_scoring and ( - not len(data) or parse_fail_count or len(preds) != len(data) or len(gts) != len(data) - ): - raise ValueError("Local DVC scoring rejects unmatched, duplicate, or unparseable scoring rows") - if parse_fail_count > 0: print(f'[MetropolisDVC] WARNING: {parse_fail_count} events failed to parse ' f'(missing or malformed start/end timestamps). ' @@ -574,27 +548,29 @@ def evaluate(self, eval_file, **judge_kwargs): print(f"\nEvaluating {len(gt_vids)} videos with SODA-c metric...") - # Explicit local scoring never consults the remote endpoint environment. - bert_endpoint = None if local_scoring else ( - judge_kwargs.get('bert_score_endpoint') or os.environ.get('BERT_SCORE_ENDPOINT') - ) + # Check for remote BERTScore endpoint (set via env var or judge_kwargs) + bert_endpoint = judge_kwargs.get('bert_score_endpoint') or os.environ.get('BERT_SCORE_ENDPOINT') + + # Initialize BERTScorer (local or remote) bert_scorer = None - use_remote = bool(bert_endpoint) - if not use_remote: + use_remote = False + + if bert_endpoint: + # Use remote BERTScore endpoint + print(f"Using remote BERTScore endpoint: {bert_endpoint}") + use_remote = True + else: + # Try local BERTScorer try: import torch from bert_score import BERTScorer - except ImportError as exc: - raise ImportError("Real DVC semantic scoring requires torch and bert-score") from exc - device = "cuda" if torch.cuda.is_available() else "cpu" - scorer_kwargs = dict(device=device) - if local_scoring: - # Native roberta-large BERTScore uses layer 17. An absolute existing - # model directory makes Transformers resolve every file locally. - scorer_kwargs['num_layers'] = 17 - bert_scorer = BERTScorer( - model_type=local_model if local_scoring else "roberta-large", **scorer_kwargs - ) + device = "cuda" if torch.cuda.is_available() else "cpu" + bert_scorer = BERTScorer(model_type="roberta-large", device=device) + print(f"BERTScorer initialized locally on {device}") + except ImportError: + print("Warning: bert_score not available, using dummy scorer (F1=0.5)") + print("Install with: pip install bert-score") + print("Or set BERT_SCORE_ENDPOINT env var for remote evaluation") def compute_bert_scores_remote(candidates: List[str], references: List[str]) -> List[float]: """Call remote BERTScore endpoint.""" @@ -643,7 +619,8 @@ def compute_bert_scores_remote(candidates: List[str], references: List[str]) -> _, _, F1 = bert_scorer.score(pred_sentences, refs) score_matrix[gi, :] = F1.cpu().numpy() else: - raise RuntimeError("DVC semantic scorer was not initialized") + # Dummy scores if BERTScore unavailable + score_matrix = np.ones_like(iou_matrix) * 0.5 # SODA-c: optimal matching using IoU × BERTScore combined_matrix = iou_matrix * score_matrix diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py index 1b5d25345..14abc2b42 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_event_verification.py @@ -60,8 +60,6 @@ def __init__( max_pixels=None, max_frames=None, system_prompt_option='merged', - local_data_file=None, - local_video_root=None, ): """ MetropolisEventVerification dataset for event verification in videos. @@ -71,10 +69,6 @@ def __init__( self.max_pixels = max_pixels self.max_frames = max_frames self.system_prompt_option = system_prompt_option - self.local_data_file = local_data_file - self.local_video_root = local_video_root - if (local_data_file is None) != (local_video_root is None): - raise ValueError("local_data_file and local_video_root must be supplied together") super().__init__( dataset=dataset, nframe=nframe, @@ -87,10 +81,6 @@ def supported_datasets(cls): return list(cls._S3_PATHS.keys()) def prepare_dataset(self, dataset_name='MetropolisEventVerification'): - if self.local_data_file is not None: - from ..scoring_assets import validate_local_video_inputs - return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'EventVerification') - from s3fs import S3FileSystem bucket, prefix = self._S3_PATHS[dataset_name] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py index d8f2740cc..0a3d2185a 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_temporal.py @@ -119,12 +119,8 @@ class MetropolisTemporal(VideoBaseDataset): # Question generation prefix (must match metropolis-benchmarks/code/run_qwen3_temporal_api_pt.py) QUESTION_PREFIX = """Localize a series of activity events in the video, output the start and end timestamp for each event. Provide the result in json format with 'mm:ss.ff' format for time depiction for this event. Use keywords 'start' and 'end' in the json output.""" - def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, random_state=None, include_categories=None, local_data_file=None, local_video_root=None): + def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, random_state=None, include_categories=None): self.test_mode = test_mode - self.local_data_file = local_data_file - self.local_video_root = local_video_root - if (local_data_file is None) != (local_video_root is None): - raise ValueError("local_data_file and local_video_root must be supplied together") self.category_mapping = {} self.limit = limit # Store limit parameter self.verbose = verbose # Enable debug logging @@ -138,10 +134,8 @@ def __init__(self, dataset='MetropolisTemporal', pack=False, nframe=0, fps=0, to self.S3_BUCKET = bucket self.S3_PREFIX = prefix self.S3_PROFILE = profile - if not test_mode or local_data_file is not None: + if not test_mode: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) - if local_data_file is not None and 'category' in self.data: - self.category_mapping = dict(zip(self.data['video'], self.data['category'])) # Track original size original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -218,10 +212,6 @@ def prepare_dataset(self, dataset_name='MetropolisTemporal'): """ Prepare the dataset by loading from S3 or local cache. """ - if self.local_data_file is not None: - from ..scoring_assets import validate_local_video_inputs - return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'Temporal') - def check_integrity(pth): data_file = osp.join(pth, f'{dataset_name}.tsv') if not osp.exists(data_file): @@ -974,8 +964,6 @@ def evaluate(self, eval_file, **judge_kwargs): def _get_category(self, video_id: str) -> str: """Get category for a video based on its ID.""" - if self.local_data_file is not None and video_id in self.category_mapping: - return self.category_mapping[video_id] if not self.category_mapping: return "Other" diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py index e31b1fe65..a2caafd8c 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/metropolis_vqa.py @@ -90,12 +90,8 @@ class MetropolisVQA(VideoBaseDataset): def __init__(self, dataset='MetropolisVQA', pack=False, nframe=0, fps=1.0, total_pixels=None, max_pixels=None, max_frames=None, test_mode=False, limit=None, verbose=False, - random_state=None, include_categories=None, local_data_file=None, local_video_root=None): + random_state=None, include_categories=None): self.test_mode = test_mode - self.local_data_file = local_data_file - self.local_video_root = local_video_root - if (local_data_file is None) != (local_video_root is None): - raise ValueError("local_data_file and local_video_root must be supplied together") self.category_mapping = {} self.limit = limit self.verbose = verbose @@ -111,10 +107,8 @@ def __init__(self, dataset='MetropolisVQA', pack=False, nframe=0, fps=1.0, total self.S3_PREFIX = prefix self.S3_PROFILE = profile - if not test_mode or local_data_file is not None: + if not test_mode: super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps, total_pixels=total_pixels) - if local_data_file is not None and 'category' in self.data: - self.category_mapping = dict(zip(self.data['video'], self.data['category'])) # Track original size original_size = len(self.data) if hasattr(self, 'data') else 0 @@ -172,10 +166,6 @@ def prepare_dataset(self, dataset_name='MetropolisVQA'): """ Prepare the dataset by loading from S3 or local cache. """ - if self.local_data_file is not None: - from ..scoring_assets import validate_local_video_inputs - return validate_local_video_inputs(self.local_data_file, self.local_video_root, 'VQA') - def check_integrity(pth): data_file = osp.join(pth, f'{dataset_name}.tsv') if not osp.exists(data_file): @@ -597,8 +587,6 @@ def get_category(self, vid: str) -> str: def _get_category(self, video_id: str) -> str: """Get category for a video based on its ID from mapping.""" - if self.local_data_file is not None and video_id in self.category_mapping: - return self.category_mapping[video_id] if not self.category_mapping: return "Other" diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py index 44b96041a..778cec3a4 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py @@ -38,8 +38,7 @@ class MVBench(VideoBaseDataset): TYPE = 'Video-MCQ' DEFAULT_JUDGE = ['chatgpt-0125', 'gpt-4-0125'] - def __init__(self, dataset='MVBench', nframe=0, fps=-1): - self.type_data_list = { + TASKS = { 'Action Sequence': ('action_sequence.json', 'your_data_path/star/Charades_v1_480/', 'video', True), # has start & end 'Action Prediction': ('action_prediction.json', @@ -80,7 +79,10 @@ def __init__(self, dataset='MVBench', nframe=0, fps=-1): 'your_data_path/tvqa/frames_fps3_hq/', 'frame', True), # has start & end, read frame 'Counterfactual Inference': ('counterfactual_inference.json', 'your_data_path/clevrer/video_validation/', 'video', False), - } + } + + def __init__(self, dataset='MVBench', nframe=0, fps=-1): + self.type_data_list = self.TASKS super().__init__(dataset=dataset, nframe=nframe, fps=fps) @classmethod @@ -369,7 +371,7 @@ def build_prompt(self, line, video_llm): return message @classmethod - def evaluate(self, eval_file, **judge_kwargs): + def evaluate(self, eval_file, *, trusted_data=None, **judge_kwargs): assert get_file_extension(eval_file) in ['xlsx', 'json', 'tsv'], 'data file should be an supported format (xlsx/json/tsv) file' # noqa: E501 @@ -391,7 +393,9 @@ def evaluate(self, eval_file, **judge_kwargs): res = {} if not osp.exists(tmp_file) else load(tmp_file) res = {k: v for k, v in res.items() if FAIL_MSG not in v} - data = load(eval_file) + # Local scoring supplies an authenticated table directly so pandas + # cannot reinterpret literal model strings (e.g. "nan" or "1"). + data = load(eval_file) if trusted_data is None else trusted_data.copy() data_un = data[~pd.isna(data['prediction'])] for idx in data_un['index']: @@ -411,6 +415,11 @@ def evaluate(self, eval_file, **judge_kwargs): if FAIL_MSG in pred: data.loc[idx, 'score'] = -1 + elif trusted_data is not None and not any( + character.isalnum() for character in pred.lower().strip().split(' ')[0]): + # Punctuation alone is not an option: native substring + # matching otherwise grants credit for ".", "(" and ")". + data.loc[idx, 'score'] = 0 else: data.loc[idx, 'score'] = int(check_ans_with_model( pred, ans, model, @@ -432,6 +441,113 @@ def evaluate(self, eval_file, **judge_kwargs): dump(rating, tgt_file) return rating +class MVBenchLocal(MVBench): + """Pinned local MVBench, preserving native sampling and image prompts. + + Unlike the generic downloader, local preparation never switches revisions or + drops missing NTU rows. Frame caches are clip-specific rather than keyed by + video basename alone. Native video conversion is deliberately disabled: + Cosmos receives exactly nframe sampled images and the native MCQ prompt. + """ + + def __init__(self, dataset='MVBench', data_root=None, nframe=16, smoke=False, fps=-1): + from pathlib import Path + + if dataset != 'MVBench': + raise ValueError('MVBenchLocal supports dataset="MVBench" only') + if data_root is None: + raise ValueError('MVBenchLocal requires an explicit data_root') + if isinstance(nframe, bool) or not isinstance(nframe, int) or nframe <= 0: + raise ValueError('nframe must be a positive integer') + if fps > 0: + raise ValueError('MVBenchLocal uses native fixed-frame sampling, not fps sampling') + self._local_root = Path(data_root).expanduser().resolve() + self._local_smoke = bool(smoke) + super().__init__(dataset=dataset, nframe=nframe, fps=fps) + self.frame_root = str(self._local_root / '.mvbench' / 'frames') + os.makedirs(self.frame_root, exist_ok=True) + + def prepare_dataset(self, dataset_name='MVBench', **kwargs): + from vlmeval.mvbench_local import FRAME_RESOLUTION, prepare_dataset + from .utils.mvbench import GroupCenterCrop, GroupScale + + prepared = prepare_dataset(self._local_root, smoke=self._local_smoke) + self.local_provenance = prepared + self.decord_method = { + 'video': self.read_video, 'gif': self.read_gif, 'frame': self.read_frame, + } + self.frame_fps = 3 + # Reference sampling/cropping, but not VideoChat2 tensor normalization: + # the Cosmos server owns its model-specific image normalization. + self.transform = T.Compose([ + GroupScale(FRAME_RESOLUTION, interpolation=Image.BICUBIC), + GroupCenterCrop(FRAME_RESOLUTION), + Stack(), ToTorchFormatTensor(), + ]) + return prepared + + def qa_template(self, data): + import ast + + candidates = data['candidates'] + if isinstance(candidates, str): + candidates = ast.literal_eval(candidates) + if (not isinstance(candidates, list) + or not all(isinstance(option, str) for option in candidates) + or candidates.count(data['answer']) != 1): + raise ValueError('MVBench question has invalid candidates or answer') + question = f"Question: {data['question']}\nOptions:\n" + question += '\n'.join(f'({chr(ord("A") + index)}) {option}' + for index, option in enumerate(candidates)) + answer_index = candidates.index(data['answer']) + return question.rstrip(), f'({chr(ord("A") + answer_index)}) {data["answer"]}' + + def clip_cache_key(self, line): + import hashlib + from vlmeval.mvbench_local import ( + DATASET_REVISION, FRAME_PREPROCESSING, FRAME_RESOLUTION, _media_content_sha256, + ) + + identity = { + 'dataset_revision': DATASET_REVISION, + 'dataset_root': str(self._local_root), + 'media': str(os.path.join(line['prefix'], line['video'])), + 'data_type': line['data_type'], + 'bounds': [float(line['start']), float(line['end'])] if line['bound'] else None, + 'nframe': self.nframe, + 'sampling': 'native-MVBench.get_index-v1', + 'resolution': FRAME_RESOLUTION, 'preprocessing': FRAME_PREPROCESSING, + 'media_sha256': _media_content_sha256(self._local_root, line), + } + return hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + + def save_video_into_images(self, line): + key = self.clip_cache_key(line) + frame_paths = self.frame_paths(key) + if all(osp.isfile(path) and osp.getsize(path) > 0 for path in frame_paths): + return frame_paths + bound = (line['start'], line['end']) if line['bound'] else None + video_path = osp.join(self.data_root, line['prefix'], line['video']) + self.num_segments = self.nframe + images = self.decord_method[line['data_type']](video_path, bound) + return self.save_video_frames(images, key, self.nframe) + + def build_prompt(self, line, video_llm=False): + return super().build_prompt(line, video_llm=False) + + def evaluate(self, eval_file, **judge_kwargs): + from pathlib import Path + from vlmeval.mvbench_local import score_predictions + + if judge_kwargs.get('model', 'exact_matching') != 'exact_matching': + raise ValueError('MVBenchLocal only permits the network-free exact_matching judge') + path = Path(eval_file) + return score_predictions( + path, self._local_root, path.parent / f'{path.stem}_mvbench', + smoke=self._local_smoke, nframe=self.nframe, + ) + + class MVBench_MP4(VideoBaseDataset): DEFAULT_JUDGE = ['chatgpt-0125', 'gpt-4-0125'] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py deleted file mode 100644 index e717739be..000000000 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/local_scoring.py +++ /dev/null @@ -1,526 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Credential-free native scoring against a fingerprinted, private local asset bundle. - -Prediction files are not annotation sources. Only their verified identities and -prediction values cross into a new table reconstructed from the local dataset. -The output directory contains private native artifacts and must not be published. -""" -import argparse -from contextlib import contextmanager, redirect_stderr, redirect_stdout -import hashlib -import inspect -from io import BytesIO -import json -import logging -import math -import numbers -import os -from pathlib import Path -import re -import traceback - -import pandas as pd - -TASKS = ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DGrounding', - '2DPointing', 'SOT', 'Astro2D') -_CONFIG_DIR = Path(__file__).resolve().parents[1] / 'cosmos_eval/data/metropolis' -_CLASSES = { - 'VQA': 'MetropolisVQA', 'EventVerification': 'MetropolisEventVerification', - 'Temporal': 'MetropolisTemporal', 'DVC': 'MetropolisDVC', - '2DGrounding': 'Metropolis2DGroundingDataset', '2DPointing': 'VANTAGE_2DPointing', - 'SOT': 'VANTAGE_SOT', 'Astro2D': 'Astro2DBenchDataset', -} -_KWARGS = { - **{task: {'local_data_file', 'local_video_root'} - for task in ('VQA', 'EventVerification', 'Temporal', 'DVC')}, - '2DGrounding': {'data_root', 'annotation_file'}, '2DPointing': {'data_root'}, - 'SOT': {'prepared_data_dir', 'metadata_path'}, 'Astro2D': {'data_roots'}, -} -_ASTRO_CHILDREN = ('VANTAGE_Astro2D_C0065', 'VANTAGE_Astro2D_IVAKPI05', - 'VANTAGE_Astro2D_IVAKPI11') - - -def _task(task): - if task.startswith('VANTAGE_'): - task = task[len('VANTAGE_'):] - if task not in TASKS: - raise ValueError('Unknown local scoring task') - return task - - -def _sha256(path): - digest = hashlib.sha256() - with open(path, 'rb') as stream: - for block in iter(lambda: stream.read(1024 * 1024), b''): - digest.update(block) - return digest.hexdigest() - - -def _inside(root, value, directory=False): - if not isinstance(value, str) or not value or Path(value).is_absolute(): - raise ValueError('Asset paths must be nonempty bundle-relative paths') - path = (root / value).resolve() - if not path.is_relative_to(root): - raise ValueError('Asset path escapes the local bundle') - if not (path.is_dir() if directory else path.is_file()): - raise ValueError('Required local asset is absent') - return path - - -def _require_fingerprint(bundle, path): - path = Path(path).resolve() - if not path.is_relative_to(bundle['root']): - raise ValueError('Ground-truth path escapes the local bundle') - key = path.relative_to(bundle['root']).as_posix() - if key not in bundle['manifest']['files']: - raise ValueError('Required ground truth is not fingerprinted') - - -@contextmanager -def _private_native_output(): - # Native scorers print categories, sequence identities and occasionally GT. - # Do not collect these in a public log or interpolate their exceptions. - previous = logging.root.manager.disable - with open(os.devnull, 'w') as sink, redirect_stdout(sink), redirect_stderr(sink): - logging.disable(logging.CRITICAL) - try: - yield - finally: - logging.disable(previous) - - -def _validate_source(bundle, task): - from .scoring_assets import validate_local_video_inputs, validate_task_sources - config = bundle['configs'][task] - with _private_native_output(): - originals = validate_task_sources( - bundle['root'], task, - bundle['manifest']['tasks'][task]['dataset_kwargs'], - ) - for path in originals: - _require_fingerprint(bundle, path) - if task in ('VQA', 'EventVerification', 'Temporal', 'DVC'): - _require_fingerprint(bundle, config['local_data_file']) - with _private_native_output(): - validate_local_video_inputs(config['local_data_file'], config['local_video_root'], task) - elif task == '2DPointing': - table = Path(config['data_root']) / 'Metropolis2DPointing.tsv' - _require_fingerprint(bundle, table) - if not table.is_file(): - raise ValueError('Local pointing table is absent; downloading is not permitted') - elif task == '2DGrounding': - _require_fingerprint(bundle, config['annotation_file']) - elif task == 'SOT' and 'metadata_path' in config: - _require_fingerprint(bundle, config['metadata_path']) - - -def load_assets(assets_dir, tasks): - """Validate local paths, annotation hashes, and task label coverage. - - Returns bundle metadata with ``root`` (Path), ``manifest``, ``configs`` and - ``manifest_sha256``. No credentials, cache lookup, endpoint, or model is used. - """ - if isinstance(tasks, str): - tasks = [tasks] - selected = tuple(dict.fromkeys(_task(task) for task in tasks)) - if not selected: - raise ValueError('At least one local task is required') - root = Path(assets_dir).expanduser().resolve() - manifest_file = root / 'scoring-assets.json' - with open(manifest_file) as stream: - manifest = json.load(stream) - if (manifest.get('format_version') != 1 or not isinstance(manifest.get('asset_id'), str) - or not manifest['asset_id'].strip() or not isinstance(manifest.get('tasks'), dict) - or not isinstance(manifest.get('files'), dict) or not manifest['files']): - raise ValueError('Invalid local scoring asset manifest') - if set(manifest['tasks']) - set(TASKS): - raise ValueError('Unknown task in local scoring manifest') - for relative, expected in manifest['files'].items(): - path = _inside(root, relative) - if not isinstance(expected, str) or not re.fullmatch('[a-f0-9]{64}', expected) or _sha256(path) != expected: - raise ValueError('Local asset fingerprint mismatch') - configs = {} - for task in selected: - entry = manifest['tasks'].get(task) - if (not isinstance(entry, dict) or type(entry.get('rows')) is not int or entry['rows'] <= 0 - or not isinstance(entry.get('dataset_kwargs'), dict)): - raise ValueError('Requested task is absent or incomplete in local assets') - kwargs = entry['dataset_kwargs'] - required = _KWARGS[task] - ({'metadata_path'} if task == 'SOT' else set()) - if not required <= set(kwargs) or set(kwargs) - _KWARGS[task]: - raise ValueError('Manifest may supply only the native local dataset paths') - with open(_CONFIG_DIR / f'VANTAGE_{task}.json') as stream: - config = json.load(stream) - if config.get('class') != _CLASSES[task] or config.get('dataset') != f'VANTAGE_{task}': - raise ValueError('Committed native dataset config is inconsistent') - if task not in ('VQA', 'EventVerification', 'Temporal', 'DVC'): - config['model_family'] = 'cr' - for key, value in kwargs.items(): - if key == 'data_roots': - if not isinstance(value, dict) or set(value) != set(_ASTRO_CHILDREN): - raise ValueError('ASTRO requires all three registered local child roots') - config[key] = {name: str(_inside(root, path, directory=True)) for name, path in value.items()} - else: - config[key] = str(_inside(root, value, directory=key in { - 'data_root', 'local_video_root', 'prepared_data_dir'})) - configs[task] = config - bundle = {'root': root, 'manifest': manifest, 'configs': configs, - 'manifest_sha256': _sha256(manifest_file)} - for task in selected: - _validate_source(bundle, task) - if 'DVC' in selected: - model = manifest.get('metric_models', {}).get('roberta_large', {}) - if not isinstance(model, dict) or not isinstance(model.get('revision'), str) or not model['revision'].strip(): - raise ValueError('DVC requires a revisioned, genuine local roberta-large model') - bundle['bert_score_model'] = str(_inside(root, model.get('path'), directory=True)) - from .scoring_assets import validate_bert_score_model - validate_bert_score_model(bundle['bert_score_model']) - model_root = Path(bundle['bert_score_model']) - files = {path.relative_to(model_root).as_posix(): path - for path in model_root.rglob('*') - if path.is_file() and '.cache' not in path.relative_to(model_root).parts} - expected = model.get('files') - if not isinstance(expected, dict) or not expected or set(files) != set(expected): - raise ValueError('DVC metric-model files do not match the fingerprinted bundle') - for relative, path in files.items(): - if not path.resolve().is_relative_to(root) or _sha256(path) != expected[relative]: - raise ValueError('DVC metric-model fingerprint mismatch') - return bundle - - -def dataset_config(assets_dir, task): - """Return the committed native inference config with absolute local paths.""" - task = _task(task) - return load_assets(assets_dir, [task])['configs'][task] - - -def _build_dataset(bundle, task): - from . import dataset as native - config = dict(bundle['configs'][task]) - cls = getattr(native, config.pop('class')) - with _private_native_output(): - dataset = cls(**config) - expected = bundle['manifest']['tasks'][task]['rows'] - if len(dataset.data) != expected or 'index' not in dataset.data: - raise ValueError('Native dataset silently omitted required local examples') - keys = [_identity(value) for value in dataset.data['index']] - if len(set(keys)) != expected: - raise ValueError('Native dataset identities are not unique') - if task == 'SOT': - if set(dataset.data['index']) != set(dataset._gt_cache) or any( - type(key) is not int for key in dataset._gt_cache): - raise ValueError('SOT native integer identity cache is incomplete') - return dataset - - -def build_local_dataset(assets_dir, task): - """Build and validate the native dataset without constructing a model.""" - task = _task(task) - return _build_dataset(load_assets(assets_dir, [task]), task) - - -def _identity(value): - # Integral numeric indexes and their canonical decimal spelling are equal; - # distinct textual IDs (e.g. '00' and '0') are deliberately not collapsed. - if isinstance(value, bool) or value is None: - raise ValueError('Missing or invalid example identity') - if isinstance(value, numbers.Integral): - return str(int(value)) - if isinstance(value, numbers.Real): - if math.isfinite(value) and value.is_integer(): - return str(int(value)) - raise ValueError('Invalid numeric example identity') - if not isinstance(value, str) or not value or value != value.strip(): - raise ValueError('Missing or invalid example identity') - return value - - -def _stable_identity(value): - if isinstance(value, str): - if not value.strip(): - raise ValueError('Missing stable example identity') - return value - return _identity(value) - - -def _read_predictions(predictions): - path = Path(predictions).expanduser().resolve() - contents = path.read_bytes() - stream = BytesIO(contents) - if path.suffix.lower() in {'.tsv', '.csv'}: - table = pd.read_csv(stream, sep='\t' if path.suffix.lower() == '.tsv' else ',', - dtype=str, keep_default_na=False) - elif path.suffix.lower() == '.xlsx': - table = pd.read_excel(stream, dtype=object, keep_default_na=False) - elif path.suffix.lower() == '.json': - rows = json.loads(contents) - if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): - raise ValueError('Prediction JSON must be an array of row objects') - table = pd.DataFrame(rows) - else: - raise ValueError('Predictions must be TSV, CSV, XLSX, or JSON rows') - if table.empty or not {'index', 'prediction'} <= set(table): - raise ValueError('Predictions require nonempty index and prediction columns') - return table, hashlib.sha256(contents).hexdigest() - - -def _identity_columns(task, trusted): - if task == 'Astro2D': - return ['SUB_DATASET', 'original_index', 'image_filename'] - if task == 'SOT': - return ['label'] - if task == '2DGrounding': - return ['sentence'] - if task == '2DPointing': - return [column for column in ('question_id', 'question', 'A', 'B', 'C', 'D') if column in trusted] - # VQA's native fallback qid is shared by every question in one video. - # The actual question (including options) must match, not just video/qid. - return [column for column in ('video', 'qid', 'id', 'question', 'options') if column in trusted] - - -def _trusted_predictions(task, dataset, predictions, allow_subset): - trusted = dataset.data - keys = [_identity(value) for value in trusted['index']] - lookup = {key: position for position, key in enumerate(keys)} - incoming = [_identity(value) for value in predictions['index']] - if len(set(incoming)) != len(incoming): - raise ValueError('Duplicate prediction identity') - if set(incoming) - set(keys): - raise ValueError('Prediction identity does not match the local dataset') - if not allow_subset and set(incoming) != set(keys): - raise ValueError('Full local dataset coverage is required; use allow_subset explicitly') - identities = _identity_columns(task, trusted) - if not set(identities) <= set(predictions): - raise ValueError('Predictions must retain the native stable identity columns: ' + ', '.join(identities)) - rows = trusted.iloc[[lookup[key] for key in incoming]].copy().reset_index(drop=True) - for column in identities: - if any(_stable_identity(left) != _stable_identity(right) - for left, right in zip(predictions[column], rows[column])): - raise ValueError('Prediction stable identity does not match its native index') - values = list(predictions['prediction']) - if any(not isinstance(value, str) or not value.strip() or value.strip().lower() in {'null', 'nan', 'none'} - or any(marker in value for marker in ('VIDEO_LOAD_FAILED', 'Failed to obtain answer via API.')) - for value in values): - raise ValueError('Missing or failed prediction; no score was produced') - rows['prediction'] = values - # Normalize only after verified native identities. Both sides of native - # joins then survive TSV numeric inference, including textual '00' IDs. - if task in ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DPointing'): - rows['index'] = [lookup[key] for key in incoming] - dataset.data = trusted.copy() - dataset.data['index'] = list(range(len(trusted))) - if task == 'SOT': - rows['index'] = rows['index'].astype(int) - return rows - - -def _validate_predictions(task, dataset, rows): - for position, row in rows.iterrows(): - text = row['prediction'] - try: - if task == 'EventVerification': - if dataset._extract_answer(text) not in ('yes', 'no'): - raise ValueError('Unparseable event prediction would be silently skipped') - elif task == 'DVC': - events = dataset.parse_events_from_json(text) - if not events: - raise ValueError('DVC prediction has no scorable events') - for event in events: - if not (('start' in event or 'start_time' in event) - and ('end' in event or 'end_time' in event)): - raise ValueError('DVC event is missing timestamps') - start = dataset.parse_timestamp(event.get('start') or event.get('start_time', '0')) - end = dataset.parse_timestamp(event.get('end') or event.get('end_time', '0')) - # Native IoU retains finite zero/reversed spans as zero overlap. - if (not math.isfinite(start) or not math.isfinite(end) - or not (event.get('caption') or event.get('description') or '').strip()): - raise ValueError('DVC contains an invalid or skipped event') - elif task == 'Temporal': - # Check the same branch native evaluate uses, but disallow its - # silent [0,duration] fallback for malformed responses. - try: - stamps = dataset.parse_timestamps_json(text, row['duration'], strict=True) - except Exception: - stamps = dataset.parse_timestamps(text, row['duration'], strict=True) - # Native JSON's permissive fallback returns a full interval - # rather than trying angle timestamps. Reject that mismatch. - try: - native_stamps = dataset.parse_timestamps_json(text, row['duration']) - except Exception: - native_stamps = stamps - if native_stamps != stamps: - raise ValueError('Temporal prediction would trigger a silent full-duration fallback') - if not all(math.isfinite(value) for value in stamps): - raise ValueError('Nonfinite temporal prediction interval') - elif task == 'SOT': - from .dataset.vantage_sot import parse_sot_response - parsed = parse_sot_response(text, int(row['n_frames']), model_family='cr') - if not parsed and text.strip() != '{}': - raise ValueError('Unparseable SOT prediction') - # MCQ, grounding and detection keep malformed nonempty outputs in - # their native denominators (zero / empty detection), not skip them. - except Exception: - raise ValueError(f'{task} prediction at row {position} cannot be scored without silent fallback') from None - - -def _evaluate(task, dataset, rows, output, bundle): - path = output / f'predictions_VANTAGE_{task}.tsv' - rows.to_csv(path, sep='\t', index=False) - with _private_native_output(): - if task == 'Astro2D': - # Invoke children directly: ConcatDataset replaces dataset names in - # the entire path, which can corrupt user-chosen output directories. - metrics = {} - present = [] - for child in dataset.datasets: - subset = rows[rows['SUB_DATASET'] == child].copy() - if subset.empty: - continue - subset['index'] = subset.pop('original_index') - subset = subset.drop(columns=['SUB_DATASET']) - child_path = output / f'{child}.tsv' - subset.to_csv(child_path, sep='\t', index=False) - result = dataset.dataset_map[child].evaluate(str(child_path), verbose=False) - if result.get('error') or result.get('total_predictions') != len(subset): - raise ValueError('ASTRO native scorer omitted examples') - metrics.update({f'{child}:{key}': value for key, value in result.items()}) - present.append(child) - for key in ('f1', 'precision', 'recall', 'f1_0.95', 'f1_mIOU'): - metrics[key] = sum(metrics[f'{child}:{key}'] for child in present) / len(present) - else: - kwargs = {'verbose': False} - if task == 'DVC': - kwargs.update(local_scoring=True, bert_score_model=bundle['bert_score_model']) - metrics = dataset.evaluate(str(path), **kwargs) - if isinstance(metrics, pd.DataFrame): - if len(metrics) != 1: - raise ValueError('Unexpected native metric table') - metrics = metrics.iloc[0].to_dict() - if not isinstance(metrics, dict) or metrics.get('error'): - raise ValueError('Native scoring failed') - if task == 'EventVerification' and metrics.get('Valid Predictions') != len(rows): - raise ValueError('Native event scorer silently omitted predictions') - if task == 'Temporal' and metrics.get('overall', {}).get('count') != len(rows): - raise ValueError('Native temporal scorer silently omitted predictions') - if task == 'DVC' and metrics.get('overall', {}).get('count') != len(rows): - raise ValueError('Native DVC scorer silently omitted predictions') - if task == '2DGrounding' and metrics.get('total_samples') != len(rows): - raise ValueError('Native grounding scorer silently omitted predictions') - if task == 'SOT' and len(metrics) != len(rows) + 1: - raise ValueError('Native tracking scorer silently omitted predictions') - return metrics, path - - -def _primary(task, metrics): - name, unit, aggregation = { - 'VQA': ('accuracy', 'fraction', 'question_micro'), - 'EventVerification': ('macro_f1', 'fraction', 'class_macro'), - 'Temporal': ('iou', 'fraction', 'video_macro_of_question_means'), - 'DVC': ('SODA_c', 'fraction', 'video_macro'), - '2DGrounding': ('Acc@0.5', 'percent', 'expression_micro'), - '2DPointing': ('accuracy', 'fraction', 'question_micro'), - 'SOT': ('success_auc', 'fraction', 'sequence_macro'), - 'Astro2D': ('f1', 'percent', 'unweighted_sequence_macro'), - }[task] - if task in ('VQA', '2DPointing', 'SOT'): - value = metrics['Overall'] - elif task in ('Temporal', 'DVC'): - value = metrics['overall'][name] - else: - value = metrics[name] - if not isinstance(value, numbers.Real) or not math.isfinite(value): - raise ValueError('Native primary metric is not finite') - return {'name': name, 'value': float(value), 'unit': unit, 'aggregation': aggregation} - - -def _json_value(value): - if isinstance(value, dict): - return {str(key): _json_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_json_value(item) for item in value] - if hasattr(value, 'item'): - return _json_value(value.item()) - if isinstance(value, float) and not math.isfinite(value): - raise ValueError('Native metrics contain a nonfinite value') - return value - - -def score_predictions(task, predictions, assets_dir, output_dir, allow_subset=False): - """Score native predictions locally; write private artifacts and summary.json. - - Stable identity columns from the native inference table must be retained. - A subset is never labeled full coverage, even when explicitly permitted. - Existing output directories are rejected rather than reused or overwritten. - """ - task = _task(task) - output = Path(output_dir).expanduser().resolve() - if output.exists(): - raise ValueError('Local scoring requires a fresh output directory') - bundle = load_assets(assets_dir, [task]) - predictions, prediction_sha256 = _read_predictions(predictions) - dataset = _build_dataset(bundle, task) - rows = _trusted_predictions(task, dataset, predictions, allow_subset) - _validate_predictions(task, dataset, rows) - output.mkdir(mode=0o700, parents=True, exist_ok=False) - try: - metrics, scoring_table = _evaluate(task, dataset, rows, output, bundle) - source_paths = {Path(__file__).resolve(), Path(inspect.getfile(type(dataset))).resolve(), - Path(__file__).with_name('scoring_assets.py').resolve(), - _CONFIG_DIR / f'VANTAGE_{task}.json'} - if task == 'Astro2D': - source_paths.update(Path(inspect.getfile(type(child))).resolve() - for child in dataset.dataset_map.values()) - if task in ('Astro2D', '2DGrounding'): - from .dataset.metropolis2d import utils as geometry - source_paths.add(Path(inspect.getfile(geometry)).resolve()) - provenance = { - 'asset_id': bundle['manifest']['asset_id'], 'manifest_sha256': bundle['manifest_sha256'], - 'asset_files': bundle['manifest']['files'], 'predictions_sha256': prediction_sha256, - 'scoring_table_sha256': _sha256(scoring_table), - 'source_files': {str(path.relative_to(Path(__file__).resolve().parents[1])): _sha256(path) - for path in sorted(source_paths)}, - } - if task == 'DVC': - provenance['metric_model'] = bundle['manifest']['metric_models']['roberta_large'] - summary = _json_value({ - 'format_version': 1, 'task': task, 'status': 'complete', - 'coverage': {'expected_rows': len(dataset.data), 'scored_rows': len(rows), - 'subset': len(rows) != len(dataset.data)}, - 'primary_metric': _primary(task, metrics), 'native_metrics': metrics, - 'annotation_diagnostics': bundle['manifest']['tasks'][task].get('annotation_diagnostics', {}), - 'provenance': provenance, - }) - temporary = output / 'summary.json.tmp' - temporary.write_text(json.dumps(summary, indent=2, allow_nan=False) + '\n') - temporary.replace(output / 'summary.json') - return summary - except Exception: - (output / 'summary.json.tmp').unlink(missing_ok=True) - (output / 'scoring-error.log').write_text(traceback.format_exc()) - raise ValueError('Local scoring failed; inspect private output/scoring-error.log') from None - - -def main(argv=None): - # CLI subprocesses can inherit Jupyter's inline backend. Leave importing - # callers' kernels untouched, but always use a headless backend here. - os.environ['MPLBACKEND'] = 'Agg' - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--task', required=True) - parser.add_argument('--predictions', required=True, type=Path) - parser.add_argument('--assets-dir', required=True, type=Path) - parser.add_argument('--output-dir', required=True, type=Path) - parser.add_argument('--allow-subset', action='store_true') - args = parser.parse_args(argv) - try: - summary = score_predictions(args.task, args.predictions, args.assets_dir, - args.output_dir, args.allow_subset) - except Exception: - parser.exit(2, 'Local scoring failed: check local assets, native identities, coverage, and private output/scoring-error.log if created.\n') - print(json.dumps({'task': summary['task'], 'coverage': summary['coverage'], - 'primary_metric': summary['primary_metric'], - 'annotation_diagnostics': summary['annotation_diagnostics']})) - - -if __name__ == '__main__': - main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py new file mode 100644 index 000000000..09964a65a --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py @@ -0,0 +1,525 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned, local MVBench inputs and native, network-free exact-match scoring. + +Video rights are not conferred by the dataset card's MIT metadata license. +The source card restricts videos to academic research and requires respecting +upstream agreements. NTU RGB+D acquisition is manual, never an old-revision or +mirror fallback. This module deliberately does not claim paper-score parity. +""" +import ast +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import tempfile +import warnings +import zipfile + +DATASET_REPO = 'OpenGVLab/MVBench' +DATASET_REVISION = '230a2d4fac8900333c61754641c7a13e069ac9c6' +EXPECTED_ROWS = 4000 +ROWS_PER_TASK = 200 +FRAME_RESOLUTION = 224 +FRAME_PREPROCESSING = 'GroupScale(224,bicubic)+GroupCenterCrop(224)+Stack(roll=False)+ToTorchFormatTensor(div=True)+JPEG' +SMOKE_TASK = 'Egocentric Navigation' +SMOKE_ROWS = 5 +RIGHTS_NOTICE = ( + 'MVBench video copyrights belong to their original creators/platforms; ' + 'the dataset card permits academic research use only and requires compliance ' + 'with all source agreements. Review the downloaded README.md before use. ' + 'The card describes 320 NTU RGB+D videos; this pinned revision lists 200 ' + 'required RGB AVI files for Fine-grained Pose. Obtain authorized access at ' + 'https://rose1.ntu.edu.sg/dataset/actionRecognition/ and use the exact pinned ' + 'video/MVBench_videos_ntu.txt list. Place authorized media in ' + '/video/nturgbd. No removed media are downloaded.' +) +# SHA-256 of the exact public bytes at DATASET_REVISION (not a user manifest). +METADATA_SHA256 = { + 'json/action_antonym.json': '0ebbec1f540f598a82bf27f569e7377fdc5c4b6e62d53d3847a88abb785f5513', + 'json/action_count.json': '5e3389d3f8a81744096825d735d45d04b1842fb49da7f5b9a70d4c2224505117', + 'json/action_localization.json': '164290bb5f447b6cdff26eb51a2470bcf4ce31f84d092ec4337f7bd4ce558eec', + 'json/action_prediction.json': '15601d39ca8604dbb642149c097f416518f4816b0c0bec57a6202d95f8bdac79', + 'json/action_sequence.json': 'd5f8c063a5ae5a433c3fb8ff78a03acbf0513a01928b384ed90a1e044adf230e', + 'json/character_order.json': '72c20ad5749da31759c92f4ed83ed117a52f70963ce1d85d5dfd1582accdead4', + 'json/counterfactual_inference.json': '6fabbbaba68b10b48bfb714c51b87569266bc65303530e5ab9c39cab2924bdfa', + 'json/egocentric_navigation.json': '4a5f80cca3471a15efe6d7d4ded192218871992cab32fa3a3eede93ea7fb3972', + 'json/episodic_reasoning.json': '39ca717f721c274a20835835e4ca417f58d5fb2d17ffebb7042d373561c871c3', + 'json/fine_grained_action.json': '21fb61328ab02e399170a22a2a4080a05dff18e8423de709e49b9fa2180fde58', + 'json/fine_grained_pose.json': 'f0cebc0a498474c03618aac20f67810cc41197ac97534a51f930c217dc13f091', + 'json/moving_attribute.json': 'b760214e4320b2ceed4580cc0febfc8c8b084487c7efcaef712b704adb850548', + 'json/moving_count.json': 'a561538a69b2722c96d2b608af488c04db6a0d803b03b49661b8834e9425d245', + 'json/moving_direction.json': '81379646d91fb078a8b38de897f40977eeeebd946ca1c6ef0f58e4bde9d21209', + 'json/object_existence.json': '2bb290258243c7dafe57042768d491e6a239d30d0ff74d0f163a989c60f69620', + 'json/object_interaction.json': '9624cf2fef5ad4ec882edf955ce1d10c806e71e4e8c6a880fd867b224914302b', + 'json/object_shuffle.json': 'f9e1bd17a9bbab1d9f5be71411ca619cf9743ca47bb808985b202138b7928118', + 'json/scene_transition.json': 'edab8a7992851061cac170f7e50782961ec21826f99e70ee72ae73a9dafb79f9', + 'json/state_change.json': '21fa00a1e4fdbd3a0fda4d04f1b4dfd8ee62a1e3f3663fdb0f6e40799c36027f', + 'json/unexpected_action.json': '76b6f440ccb1ed007c67236c0ec7a39ee7cfbfb04197e9cc4a303b9230ca958b', + 'README.md': '0e9520eb913e0825e8cff14eb7a9821a23587312ed2ee97cb86a4b2c2335ec99', + 'video/MVBench_videos_ntu.txt': '878766d10973e1c9e030fca19abb4417f2a8182803199724ef6a9b0b64ec906e', +} +ARCHIVE_SHA256 = { + 'video/FunQA_test.zip': 'd6ef954652c2a1b48fb772d52c97da35dc7f8b54e5bdf4b43ba4f773ed726723', + 'video/Moments_in_Time_Raw.zip': '16add0d03cbb7c6b76974551879f7dc8a7ce985b89699a2231136b81d3ba9967', + 'video/clevrer.zip': '9040ad663440611bd8580fc0fc3da31bd1a4bfef877631db7b570a7d7d41de1c', + 'video/data0613.zip': 'f045863b2ccf7f32ada437d72529fea1c026fc015a7df548f703c22b71a24962', + 'video/perception.zip': '98fc197a90c875fb24f34fe89cb91e2788e1e7ac2459bf860ed62d9f07104b34', + 'video/scene_qa.zip': '26b3dc385e5a8f7b8fd9aab32a3522488fdfe7bbe06f0d98206d43647c77cce9', + 'video/ssv2_video.zip': '8493fe3762bdc205b587c4d02e016f9ccbb314b523ed0bf2c5bdbe367456d1e9', + 'video/sta.zip': '4c27cbf62bbc1b0217b5183871f298e7a76b2a498dd4482365c617f25338de9a', + 'video/star.zip': 'bdb5f192a98f78e79e51da6ae801f8ba79d1d09c0d8c19b6e63105251cb17585', + 'video/tvqa.zip': '6c5bccf237164286123a7afcc1bd7e8323487d7f2fd0dc5f43609a34064f5d73', + 'video/vlnqa.zip': '578f9edf2973560ac0d79990d9a2961ccae8edcdb1cb0be2c3bf15045f04428a', +} +NATIVE_SCORER = 'VLMEvalKit MVBench.evaluate(model=exact_matching), vendored utils/mvbench.py' + + +class MVBenchCoverageError(ValueError): + """Incomplete media coverage; ``report`` gives exact missing paths by task.""" + + def __init__(self, message, report): + super().__init__(message) + self.report = report + + +def _sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def _json_write(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + # Atomic replacement avoids half-written canonical inputs when interrupted. + with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, indent=2, allow_nan=False) + stream.write('\n') + os.replace(temporary, path) + + +def _inside(root, relative): + name = str(relative) + parts = PurePosixPath(name) + if not name or '\\' in name or '\x00' in name or ':' in name or parts.is_absolute() or '..' in parts.parts: + raise ValueError(f'Unsafe relative path: {name!r}') + target = Path(root).joinpath(*parts.parts) + if not target.resolve().is_relative_to(Path(root).resolve()): + raise ValueError(f'Path escapes dataset root: {name!r}') + return target + + +def _safe_extract(archive, destination, overlay=False): + """Extract regular ZIP members only; validate *all* names before writing. + + data0613 is the official June-2024 correction overlay. Strip that single + archive's data0613/ wrapper and overwrite base media, after base extraction. + """ + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive) as source: + members = [] + for info in source.infolist(): + _inside(destination, info.filename) + mode = info.external_attr >> 16 + if stat.S_ISLNK(mode) or (stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR)): + raise ValueError(f'Unsupported ZIP member type: {info.filename!r}') + parts = PurePosixPath(info.filename).parts + if overlay: + if not parts or parts[0] != 'data0613': + raise ValueError(f'Unexpected correction-overlay path: {info.filename!r}') + parts = parts[1:] + if not parts: + continue + target = _inside(destination, PurePosixPath(*parts)) + if target.is_symlink(): + raise ValueError(f'Refusing ZIP symlink target: {target}') + members.append((info, target)) + for info, target in members: + if info.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with source.open(info) as reader, tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as writer: + temporary = Path(writer.name) + try: + shutil.copyfileobj(reader, writer) + except BaseException: + temporary.unlink(missing_ok=True) + raise + os.replace(temporary, target) + + +def fetch_dataset(root: Path, scope='smoke'): + """Download only approved pinned public inputs, without login or tokens. + + ``metadata`` gets all official annotations and rights notices. ``smoke`` + adds vlnqa.zip. ``full`` adds the current public archives (never NTU). + Fetch is not a coverage assertion: call prepare_dataset before inference. + """ + from huggingface_hub import hf_hub_download + + if scope not in ('metadata', 'smoke', 'full'): + raise ValueError('scope must be metadata, smoke, or full') + root = Path(root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + warnings.warn(RIGHTS_NOTICE, UserWarning, stacklevel=2) + archives = [] if scope == 'metadata' else ['video/vlnqa.zip'] + if scope == 'full': + archives = [name for name in ARCHIVE_SHA256 if name != 'video/data0613.zip'] + ['video/data0613.zip'] + requested = list(METADATA_SHA256) + archives + for name in requested: + _inside(root, name) + path = Path(hf_hub_download( + repo_id=DATASET_REPO, repo_type='dataset', revision=DATASET_REVISION, + filename=name, local_dir=str(root), token=False, + )) + expected = METADATA_SHA256.get(name, ARCHIVE_SHA256.get(name)) + if _sha256(path) != expected: + raise ValueError(f'Pinned file checksum mismatch: {name}. Remove the damaged local file and fetch again.') + if name in archives: + _safe_extract(path, root / 'video', overlay=name == 'video/data0613.zip') + notice = root / '.mvbench' / 'RIGHTS_NOTICE.txt' + notice.parent.mkdir(parents=True, exist_ok=True) + notice.write_text(RIGHTS_NOTICE + '\n', encoding='utf-8') + return { + 'root': str(root), 'dataset_revision': DATASET_REVISION, 'scope': scope, + 'downloaded_files': requested, 'rights_notice': RIGHTS_NOTICE, + 'manual_ntu_required_for_full': True, + } + + +def _task_definitions(): + from vlmeval.dataset.mvbench import MVBench + return MVBench.TASKS + + +def _read_annotations(root): + """Authenticate every source byte before deriving identities or labels.""" + for name, expected in METADATA_SHA256.items(): + path = _inside(root, name) + if not path.is_file(): + raise ValueError(f'Missing pinned metadata {name}; run fetch_dataset(root, scope="metadata").') + if _sha256(path) != expected: + raise ValueError(f'Pinned metadata checksum mismatch: {name}; local annotations are not trusted.') + rows = [] + for task, (name, prefix, data_type, bound) in _task_definitions().items(): + source = json.loads((root / 'json' / name).read_text(encoding='utf-8')) + if not isinstance(source, list) or len(source) != ROWS_PER_TASK: + raise ValueError(f'{name} must contain exactly {ROWS_PER_TASK} questions') + for position, item in enumerate(source): + candidates = item.get('candidates') + if (not isinstance(candidates, list) or not 1 <= len(candidates) <= 26 + or not all(isinstance(option, str) and option.strip() for option in candidates) + or len(set(candidates)) != len(candidates)): + raise ValueError(f'Invalid candidates in {name}:{position}') + answer = item.get('answer') + if not isinstance(answer, str) or candidates.count(answer) != 1: + raise ValueError(f'Answer must identify exactly one candidate in {name}:{position}') + if not isinstance(item.get('question'), str) or not item['question'].strip(): + raise ValueError(f'Missing question in {name}:{position}') + if not isinstance(item.get('video'), str) or not item['video'].strip(): + raise ValueError(f'Missing video in {name}:{position}') + media_prefix = prefix.replace('your_data_path', 'video') + _inside(root, str(PurePosixPath(media_prefix) / item['video'])) + start, end = item.get('start'), item.get('end') + if bound and (not all(isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) + for v in (start, end)) or start < 0 or end <= start): + raise ValueError(f'Invalid temporal bounds in {name}:{position}') + rows.append({ + 'index': len(rows), 'task_type': task, 'prefix': media_prefix, + 'data_type': data_type, 'bound': bound, 'start': start, 'end': end, + 'video': item['video'], 'question': item['question'], 'answer': answer, + # repr is exclusively of authenticated, validated string lists; + # upstream evaluate's eval never receives incoming candidates. + 'candidates': repr(candidates), + }) + if len(rows) != EXPECTED_ROWS or len(_task_definitions()) != 20: + raise ValueError('MVBench must contain 4000 questions across all 20 native tasks') + return rows + + +def _media_available(root, row): + path = _inside(root, str(PurePosixPath(row['prefix']) / row['video'])) + if row['data_type'] != 'frame': + return path.is_file() and path.stat().st_size > 0 + if not path.is_dir(): + return False + frames = sorted(path.glob('*.jpg')) + return bool(frames) and all( + frame.name == f'{position:05d}.jpg' and frame.stat().st_size > 0 + for position, frame in enumerate(frames, 1) + ) + + +_MEDIA_HASH_CACHE = {} + + +def _media_content_sha256(root, row): + """Hash media bytes, memoizing only while file identities/stats are stable.""" + path = _inside(root, str(PurePosixPath(row['prefix']) / row['video'])) + files = sorted(path.glob('*.jpg')) if row['data_type'] == 'frame' else [path] + if not files: + raise ValueError(f'Media has no frames: {path}') + + def signature(): + result = [] + for file in files: + _inside(root, file.relative_to(root).as_posix()) + info = file.stat() + result.append((file.name, info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)) + return tuple(result) + + before = signature() + key = str(path) + cached = _MEDIA_HASH_CACHE.get(key) + if cached is not None and cached[0] == before: + return cached[1] + if row['data_type'] == 'frame': + digest = hashlib.sha256() + for file in files: + digest.update(file.name.encode()) + digest.update(b'\0') + digest.update(bytes.fromhex(_sha256(file))) + value = digest.hexdigest() + else: + value = _sha256(path) + if signature() != before: + raise ValueError(f'Media changed while hashing: {path}') + _MEDIA_HASH_CACHE[key] = (before, value) + return value + + +def _manifest_sha256(): + payload = {'repo': DATASET_REPO, 'revision': DATASET_REVISION, 'metadata': METADATA_SHA256, + 'archives': ARCHIVE_SHA256, 'tasks': _task_definitions(), + 'task_order': list(_task_definitions())} + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def prepare_dataset(root: Path, smoke=False): + """Rebuild canonical TSV from pinned annotations; never silently drop rows. + + Full requires all 4000 rows/media; smoke selects ONLY the first five official + Egocentric Navigation identities. Coverage always reports the full universe. + """ + import pandas as pd + + root = Path(root).expanduser().resolve() + rows = _read_annotations(root) + selected = [row for row in rows if row['task_type'] == SMOKE_TASK][:SMOKE_ROWS] if smoke else rows + available = {} + missing = {} + for row in rows: + key = (row['prefix'], row['video'], row['data_type']) + if key not in available: + available[key] = _media_available(root, row) + if not available[key]: + missing.setdefault(row['task_type'], set()).add(str(PurePosixPath(row['prefix']) / row['video'])) + coverage = { + 'expected_rows': EXPECTED_ROWS, + 'available_rows': sum(available[(r['prefix'], r['video'], r['data_type'])] for r in rows), + 'selected_rows': len(selected), 'expected_tasks': 20, + 'selected_tasks': list(dict.fromkeys(r['task_type'] for r in selected)), + 'subset': bool(smoke), + } + report = { + 'dataset_revision': DATASET_REVISION, 'coverage': coverage, + 'missing_media_by_task': {task: sorted(paths) for task, paths in missing.items()}, + 'rights_notice': RIGHTS_NOTICE, + } + scope = 'smoke' if smoke else 'full' + report_file = root / '.mvbench' / f'coverage-{scope}.json' + _json_write(report_file, report) + missing_selected = [r['index'] for r in selected if not available[(r['prefix'], r['video'], r['data_type'])]] + if missing_selected: + detail = ' Full evaluation requires all 4000 questions and all 20 tasks.' if not smoke else ' Smoke requires its fixed five navigation videos.' + raise MVBenchCoverageError( + f'MVBench media coverage incomplete: {coverage["available_rows"]}/4000 available; ' + f'{len(missing_selected)} selected rows lack usable media.{detail} ' + f'See {report_file}. {RIGHTS_NOTICE}', report, + ) + media_hashes = {} + for row in selected: + relative = str(PurePosixPath(row['prefix']) / row['video']) + if relative not in media_hashes: + media_hashes[relative] = _media_content_sha256(root, row) + annotation_manifest = _manifest_sha256() + media_manifest = hashlib.sha256(json.dumps(media_hashes, sort_keys=True).encode()).hexdigest() + manifest = hashlib.sha256(f'{annotation_manifest}:{media_manifest}'.encode()).hexdigest() + _json_write(root / '.mvbench' / f'media-manifest-{scope}.json', { + 'dataset_revision': DATASET_REVISION, 'sha256': media_manifest, 'media': media_hashes, + }) + data_file = root / '.mvbench' / f'MVBench-{scope}-{manifest[:16]}.tsv' + with tempfile.NamedTemporaryFile(mode='w', dir=data_file.parent, delete=False) as stream: + temporary = Path(stream.name) + pd.DataFrame(selected).to_csv(stream, sep='\t', index=False) + os.replace(temporary, data_file) + return { + 'root': str(root), 'data_file': str(data_file), 'dataset_revision': DATASET_REVISION, + 'manifest_sha256': manifest, 'coverage': coverage, + 'annotation_manifest_sha256': annotation_manifest, + 'media_manifest_sha256': media_manifest, 'media_sha256': media_hashes, + 'selected_indices': [row['index'] for row in selected], + } + + +def build_dataset(root: Path, nframe=16, smoke=False): + from vlmeval.dataset.mvbench import MVBenchLocal + return MVBenchLocal(data_root=str(root), nframe=nframe, smoke=smoke) + + +def _load_predictions(path): + import pandas as pd + + path = Path(path) + if path.suffix == '.tsv': + return pd.read_csv(path, sep='\t', keep_default_na=False, dtype={'prediction': str}) + if path.suffix == '.xlsx': + return pd.read_excel(path, keep_default_na=False, dtype={'prediction': str}) + if path.suffix == '.json': + return pd.DataFrame(json.loads(path.read_text(encoding='utf-8'))) + raise ValueError('Predictions must be TSV, XLSX, or JSON; pickle inputs are not accepted') + + +def _verified_predictions(incoming, trusted): + """Only strings cross the trust boundary; identities and labels are fixed.""" + if not {'index', 'prediction'}.issubset(incoming.columns): + raise ValueError('Predictions require index and prediction columns') + indices = [] + for value in incoming['index']: + try: + number = float(value) + except (TypeError, ValueError): + raise ValueError(f'Invalid prediction identity: {value!r}') from None + if isinstance(value, bool) or not math.isfinite(number) or number != int(number): + raise ValueError(f'Invalid prediction identity: {value!r}') + indices.append(int(number)) + expected = list(trusted['index']) + if len(indices) != len(set(indices)) or set(indices) != set(expected): + raise ValueError('Prediction coverage must match selected stable indices exactly: no missing, duplicate, or extra rows') + incoming = incoming.copy() + incoming['index'] = indices + incoming = incoming.set_index('index').loc[expected] + for position, row in trusted.reset_index(drop=True).iterrows(): + supplied = incoming.loc[row['index']] + for column in trusted.columns: + if column == 'index' or column not in incoming.columns: + continue + actual, wanted = supplied[column], row[column] + if column == 'candidates': + try: + actual = ast.literal_eval(actual) if isinstance(actual, str) else actual + wanted = ast.literal_eval(wanted) + except (ValueError, SyntaxError, TypeError): + raise ValueError(f'Untrusted candidates for index {row["index"]}') from None + elif column in ('start', 'end'): + if wanted is None or (isinstance(wanted, float) and math.isnan(wanted)): + if actual == '' or actual is None or (isinstance(actual, float) and math.isnan(actual)): + continue + try: + actual = float(actual) + except (ValueError, TypeError): + pass + elif column == 'bound' and isinstance(actual, str): + actual = {'True': True, 'False': False}.get(actual, actual) + if actual != wanted: + raise ValueError(f'Untrusted {column} for index {row["index"]}') + predictions = incoming['prediction'].tolist() + for index, prediction in zip(expected, predictions): + if (not isinstance(prediction, str) or not prediction.strip() + or 'failed to obtain answer' in prediction.lower() + or 'video_load_failed' in prediction.lower()): + raise ValueError(f'Missing or failed prediction for index {index}; inference must complete before scoring') + return predictions + + +def _summarize_rating(rating): + per_task = {} + for task, counts in rating.items(): + if task == 'overall': + continue + correct, total = counts[:2] + if not (0 <= correct <= total and total > 0): + raise ValueError(f'Invalid native score counts for {task}') + per_task[task] = {'correct': int(correct), 'total': int(total), 'accuracy': correct / total} + correct = sum(v['correct'] for v in per_task.values()) + total = sum(v['total'] for v in per_task.values()) + if not total or list(rating['overall'][:2]) != [correct, total]: + raise ValueError('Native overall counts disagree with task counts') + return { + 'primary_metric': {'name': 'accuracy', 'value': correct / total, 'unit': 'fraction', + 'correct': correct, 'total': total}, + 'task_macro_accuracy': sum(v['accuracy'] for v in per_task.values()) / len(per_task), + 'per_task': per_task, + } + + +def score_predictions(predictions: Path, root: Path, output_dir: Path, smoke=False, nframe=16): + """Native MVBench exact matching on authenticated labels, with no judge. + + Missing/failure predictions are rejected before native rating sees them: + upstream get_dimension_rating treats -1 as truthy. Malformed nonempty + strings still receive upstream wrong-answer semantics, not a fallback score. + Native pandas row positions are made contiguous for its .loc assignment; + original stable identities are restored in the saved scored_predictions.tsv. + """ + import pandas as pd + from vlmeval.dataset.mvbench import MVBench + from vlmeval.smp import get_intermediate_file_path, load + + if isinstance(nframe, bool) or not isinstance(nframe, int) or nframe <= 0: + raise ValueError('nframe must be a positive integer') + output_dir = Path(output_dir).expanduser().resolve() + if output_dir.exists(): + raise ValueError('Scoring requires a fresh output directory') + prepared = prepare_dataset(root, smoke=smoke) + trusted = pd.read_csv(prepared['data_file'], sep='\t') + trusted['prediction'] = _verified_predictions(_load_predictions(predictions), trusted) + official_indices = trusted['index'].tolist() + trusted['index'] = range(len(trusted)) + output_dir.mkdir(parents=True, exist_ok=False) + # Never consume a stale native _score cache supplied alongside predictions. + with tempfile.TemporaryDirectory(prefix='mvbench-native-', dir=output_dir) as temporary: + native_file = Path(temporary) / 'trusted.tsv' + trusted.to_csv(native_file, sep='\t', index=False) + rating = MVBench.evaluate(str(native_file), trusted_data=trusted, model='exact_matching') + scored = load(get_intermediate_file_path(str(native_file), '_score')) + if len(scored) != len(trusted) or not scored['score'].isin([0, 1]).all(): + raise ValueError('Native scorer returned missing, failed, or nonbinary scores; no result published') + scored['index'] = official_indices + scored.to_csv(output_dir / 'scored_predictions.tsv', sep='\t', index=False) + summary = { + 'status': 'smoke_only' if smoke else 'complete', 'dataset': 'MVBench', + 'coverage': prepared['coverage'], **_summarize_rating(rating), + 'native_rating': rating, + 'provenance': { + 'dataset_repo': DATASET_REPO, 'dataset_revision': DATASET_REVISION, + 'manifest_sha256': prepared['manifest_sha256'], 'selected_indices': official_indices, + 'annotation_manifest_sha256': prepared['annotation_manifest_sha256'], + 'media_manifest_sha256': prepared['media_manifest_sha256'], + 'media_sha256': prepared['media_sha256'], + 'prediction_sha256': _sha256(predictions), 'scorer': NATIVE_SCORER, + 'scorer_sha256': _sha256(Path(__file__).parent / 'dataset' / 'mvbench.py'), + 'scorer_utils_sha256': _sha256(Path(__file__).parent / 'dataset' / 'utils' / 'mvbench.py'), + 'judge': 'exact_matching', 'network_judge': False, 'nframe': nframe, + 'local_scoring_guards': [ + 'reject missing/failure predictions before native rating', + 'punctuation-only first option token scores zero, never substring credit', + ], + 'prompt_mode': 'native sampled images', 'rights_notice': RIGHTS_NOTICE, + 'frame_resolution': FRAME_RESOLUTION, 'frame_preprocessing': FRAME_PREPROCESSING, + 'reference_recipe': {'nframe': 16, 'resolution': 224}, + 'model_normalization': 'Cosmos serving model, not VideoChat2 tensor normalization', + 'paper_parity': False, + }, + } + _json_write(output_dir / 'summary.json', summary) + return summary diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py deleted file mode 100644 index f76321dfd..000000000 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/scoring_assets.py +++ /dev/null @@ -1,630 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Prepare authorized, private VANTAGE sources in place without network access. - -Run ``python -m vlmeval.scoring_assets --assets-dir ROOT --asset-id REV``. -DVC additionally requires --bert-score-model RELDIR --bert-score-revision REV. -Only canonical tables and scoring-assets.json are written; source labels and media -are never changed or copied. This does not grant redistribution permission. -""" - -import argparse -import ast -import hashlib -import json -import math -import re -import subprocess -from pathlib import Path - -import pandas as pd - -TASKS = ('VQA', 'EventVerification', 'Temporal', 'DVC', '2DGrounding', '2DPointing', 'SOT', 'Astro2D') -VIDEO_TASKS = TASKS[:4] -SOURCE_DIRS = { - 'VQA': 'VQA', - 'EventVerification': 'event_verification_subset/filtered', - 'Temporal': 'Temporal', - 'DVC': 'Dense Video Caption', - '2DGrounding': 'Spatial/2d_referring_expressions/refdrone', - '2DPointing': 'datasets/Metropolis2DPointing', - 'SOT': 'tracking/MetropolisSOT_benchmark_8f', - 'Astro2D': '2dbbox', -} -VQA_FILES = ( - 'metrics_spatial_wo_ss.json', 'Metropolis_VQA_Verification_Final_ITS_Data.json', - 'VANTAGE_VQA_Verification_Final_ITS_Data.json', 'metrics_spatial_ss.json', - 'metrics_temporal_filtered_ss.json', 'metrics_temporal_wo_ss.json', -) -EVENT_FILES = ( - 'metropolis_event_verification/test_annotation.json', - 'tailgating/tailgating_building_r/test_annotation.json', - 'tailgating/tailgating_courtyard/test_annotation.json', - 'warehouse_near_miss/test_annotations.json', -) - - -def _require(condition, message): - if not condition: - raise ValueError(message) - - -def _text(value): - return isinstance(value, str) and bool(value.strip()) - - -def _vqa_option(value): - option = value.strip().upper() if isinstance(value, str) else '' - _require(option in tuple('ABCD'), 'VQA source is missing a valid ground-truth option') - return option - - -def _local_path(root, path, directory=False): - root = Path(root).resolve(strict=True) - path = Path(path) - resolved = (root / path).resolve(strict=True) if not path.is_absolute() else path.resolve(strict=True) - _require(resolved.is_relative_to(root), 'Asset path escapes the local bundle') - _require(resolved.is_dir() if directory else resolved.is_file(), 'Required local asset has the wrong file type') - return resolved - - -def _json(path): - with Path(path).open(encoding='utf-8') as stream: - return json.load(stream) - - -def _records(path, key=None): - raw = _json(path) - if key and isinstance(raw, dict): - raw = raw.get(key) - _require(isinstance(raw, list) and len(raw) > 0, 'Required annotation must contain a nonempty record list') - _require(all(isinstance(row, dict) for row in raw), 'Annotation contains a non-object record') - return raw - - -def _number(value): - _require(not isinstance(value, bool), 'Boolean is not a valid numeric label') - try: - result = float(value) - except (TypeError, ValueError) as exc: - raise ValueError('Missing or malformed numeric ground-truth label') from exc - _require(math.isfinite(result), 'Label contains a nonfinite number') - return result - - -def _timestamp(value): - _require(value is not None and str(value).strip() != '', 'Missing timestamp label') - parts = str(value).strip().split(':') - _require(1 <= len(parts) <= 3, 'Malformed timestamp label') - seconds = 0.0 - for part in parts: - seconds = seconds * 60 + _number(part) - _require(seconds >= 0, 'Negative timestamp label') - return seconds - - -def _span(start, end): - start, end = _timestamp(start), _timestamp(end) - _require(end > start, 'Label interval must have positive duration') - return start, end - - -def _bbox(box): - _require(isinstance(box, (list, tuple)) and len(box) == 4, 'Missing or malformed bounding-box label') - x1, y1, x2, y2 = map(_number, box) - _require(x2 > x1 and y2 > y1, 'Bounding-box label must have positive area') - - -def _events(events): - _require(isinstance(events, list) and len(events) > 0, 'DVC requires a nonempty ground-truth event list') - for event in events: - _require(isinstance(event, dict), 'Malformed DVC ground-truth event') - start = event.get('start', event.get('start_time')) - end = event.get('end', event.get('end_time')) - _span(start, end) - caption = event.get('caption') or event.get('event_caption') or event.get('description') - _require(_text(caption), 'DVC event is missing its ground-truth caption') - - -def _validate_table(frame, task): - required = {'index', 'video', 'question', 'answer'} - _require(required.issubset(frame.columns) and len(frame) > 0, f'{task} table is empty or missing required columns') - for field in required: - _require(frame[field].notna().all() and frame[field].map(lambda value: str(value).strip() != '').all(), - f'{task} table contains a missing {field}') - _require(not frame['index'].map(str).duplicated().any(), f'{task} table contains duplicate indexes') - if task == 'VQA': - _require(frame['answer'].isin(list('ABCD')).all(), 'VQA requires explicit A-D ground-truth answers') - elif task == 'EventVerification': - _require(frame['answer'].map(lambda value: str(value).strip().lower() in ('yes', 'no')).all(), - 'EventVerification requires explicit yes/no ground-truth answers') - elif task == 'Temporal': - _require('duration' in frame, 'Temporal table requires real video duration metadata') - for _, row in frame.iterrows(): - answer = json.loads(row['answer']) - _span(answer.get('start'), answer.get('end')) - _require(_number(row['duration']) > 0, 'Temporal video duration must be positive') - elif task == 'DVC': - _require(not frame['video'].duplicated().any(), 'DVC requires one table row per video') - for answer in frame['answer']: - _events(json.loads(answer)) - else: - raise ValueError(f'Unknown local video task: {task}') - - -def validate_local_video_inputs(local_data_file, local_video_root, task): - """Validate a canonical labeled table before a native constructor loads it. - - No cache or remote source is consulted. Explicit paths must both exist; local - media references are relative to the supplied root and may not escape it. - """ - _require(local_data_file is not None and local_video_root is not None, - 'local_data_file and local_video_root must be supplied together') - data_file = Path(local_data_file).expanduser().resolve(strict=True) - root = Path(local_video_root).expanduser().resolve(strict=True) - _require(root.is_dir() and data_file.is_file(), 'Local video inputs must be a table and a directory') - _require(data_file.suffix.lower() == '.tsv', 'Canonical local video data must be a TSV table') - frame = pd.read_csv(data_file, sep='\t', keep_default_na=False) - _validate_table(frame, task) - for video in frame['video'].unique(): - media = str(video) - if task != 'EventVerification' and not media.endswith('.mp4'): - media += '.mp4' - _require(not Path(media).is_absolute(), 'Local video table must contain relative media identities') - _local_path(root, media) - return {'data_file': str(data_file), 'root': str(root)} - - -def validate_bert_score_model(model_dir): - """Require a complete local roberta-large checkpoint, never a Hub model ID.""" - _require(model_dir is not None, 'Local DVC scoring requires bert_score_model pointing to local roberta-large assets') - path = Path(model_dir).expanduser().resolve(strict=True) - _require(path.is_dir(), 'BERTScore model must be a local directory') - config = _json(path / 'config.json') - _require(config.get('model_type') == 'roberta' and config.get('num_hidden_layers') == 24, - 'Local DVC requires the roberta-large architecture') - _local_path(path, 'vocab.json') - _local_path(path, 'merges.txt') - weights = next((path / name for name in ('model.safetensors', 'pytorch_model.bin') if (path / name).is_file()), None) - if weights is None: - index = next((path / name for name in ('model.safetensors.index.json', 'pytorch_model.bin.index.json') - if (path / name).is_file()), None) - _require(index is not None, 'Local BERTScore model weights are missing') - weight_map = _json(index).get('weight_map') - _require(isinstance(weight_map, dict) and weight_map, 'Malformed local model weight index') - for shard in set(weight_map.values()): - _local_path(path, shard) - else: - _local_path(path, weights) - return path - - -def _annotation_paths(root, task): - source = _local_path(root, SOURCE_DIRS[task], directory=True) - if task in ('VQA', 'Temporal'): - annotations = _local_path(root, source / 'data_jsons/annotations', directory=True) - if task == 'VQA': - paths = [annotations / name for name in VQA_FILES if (annotations / name).is_file()] - required = set(VQA_FILES) - {'Metropolis_VQA_Verification_Final_ITS_Data.json', 'VANTAGE_VQA_Verification_Final_ITS_Data.json'} - _require(required.issubset({p.name for p in paths}) and len(paths) == 5, - 'VQA requires four metrics files and exactly one Verification ITS annotation') - else: - paths = sorted(annotations.glob('*.json')) - elif task == 'EventVerification': - paths = [source / name for name in EVENT_FILES] - elif task == 'DVC': - paths = sorted((source / 'events').glob('*_events.json')) - elif task == '2DGrounding': - paths = [source / 'refdrone_test_llava.json'] - elif task == '2DPointing': - paths = [source / 'Metropolis2DPointing.tsv'] - elif task == 'SOT': - directories = [p for p in sorted(source.iterdir()) if p.is_dir()] - _require(bool(directories), 'SOT has no sequence directories') - paths = [p / 'gt.json' for p in directories] - else: - from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset - paths = [] - for layout in Astro2DDetectionDataset._LAYOUT.values(): - sequence = source / layout['data_root'].rsplit('/', 1)[1] - labels = _local_path(root, sequence / 'labels', directory=True) - paths.extend(sorted(labels.glob('*.txt'))) - _require(bool(paths), f'{task} has no original ground-truth annotations') - return [_local_path(root, path) for path in paths] - - -def validate_task_sources(assets_dir, task, dataset_kwargs=None): - """Validate original labels and media; return required annotation fingerprints. - - The original filtered source layout is authoritative, not a prediction table. - Supplied manifest constructor paths must resolve to these same original - sources, so an alternate bundle-local label tree cannot evade validation. - """ - _require(task in TASKS, 'Unknown scoring asset task') - root = Path(assets_dir).resolve(strict=True) - source = _local_path(root, SOURCE_DIRS[task], directory=True) - paths = _annotation_paths(root, task) - if dataset_kwargs is not None: - expected = {} - if task in VIDEO_TASKS: - expected['local_video_root'] = source if task == 'EventVerification' else source / 'videos' - _local_path(root, dataset_kwargs.get('local_data_file', '')) - elif task == '2DGrounding': - expected = {'data_root': source, 'annotation_file': paths[0]} - elif task == '2DPointing': - expected = {'data_root': source} - elif task == 'SOT': - expected = {'prepared_data_dir': source} - else: - from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset - roots = dataset_kwargs.get('data_roots') - _require(isinstance(roots, dict) and set(roots) == set(Astro2DDetectionDataset._LAYOUT), - 'ASTRO manifest must identify exactly the registered original child sequences') - for name, layout in Astro2DDetectionDataset._LAYOUT.items(): - actual = _local_path(root, roots[name], directory=True) - _require(actual == source / layout['data_root'].rsplit('/', 1)[1], - 'ASTRO manifest path does not identify its original sequence') - for name, original in expected.items(): - _require(name in dataset_kwargs, f'{task} manifest is missing {name}') - actual = _local_path(root, dataset_kwargs[name], directory=original.is_dir()) - _require(actual == original, f'{task} manifest path does not identify its original source') - if task in ('VQA', 'Temporal'): - for path in paths: - for item in _records(path): - video = item.get('q_uid', item.get('vid')) if task == 'VQA' else item.get('vid') - _require(_text(video) and _text(item.get('question')), f'{task} annotation lacks video ID or question') - video = video.removesuffix('.json') if task == 'VQA' else video - filename = video if video.endswith('.mp4') else video + '.mp4' - _local_path(root, source / 'videos' / filename) - if task == 'VQA': - _vqa_option(item.get('gt_option')) - options = item.get('options') - _require(isinstance(options, list) and 2 <= len(options) <= 4 and all(_text(v) for v in options), - 'VQA source requires two to four explicit choices') - _require(_vqa_option(item.get('gt_option')) in tuple('ABCD')[:len(options)], - 'VQA source ground truth does not identify an available choice') - else: - answer = item.get('answer') - _require(isinstance(answer, str), 'Temporal source is missing ground-truth timestamps') - match = re.match(r'<([\d.]+)>\s*<([\d.]+)>', answer) - _require(match is not None, 'Malformed Temporal source timestamps; no default interval is permitted') - _span(*match.groups()) - elif task == 'EventVerification': - from .dataset.metropolis_event_verification import MetropolisEventVerification - for path in paths: - for item in _records(path, 'bcq'): - _require(item.get('id') is not None and str(item['id']).strip() != '', 'Event source lacks original ID') - _require(_text(item.get('question')), 'Event source lacks question') - _require(str(item.get('answer', '')).strip().lower() in ('yes', 'no'), 'Event source lacks yes/no ground truth') - video = item.get('video', item.get('video_id')) - _require(_text(video), 'Event source lacks video ID') - _local_path(root, path.parent / video) - items = MetropolisEventVerification._load_items(source) - identities = [(item.get('source_corpus', ''), str(item['id'])) for item in items] - _require(len(set(identities)) == len(identities), 'Duplicate original EventVerification IDs within a corpus') - elif task == 'DVC': - from .dataset.metropolis_dvc import MetropolisDVC - adapter = MetropolisDVC(test_mode=True) - resolved = [] - for path in paths: - raw = _json(path) - _events(raw.get('events') if isinstance(raw, dict) else raw) - video_id = path.name.removesuffix('_events.json') - videos = source / 'videos' - exact = videos / (video_id + '.mp4') - with_ext = videos / video_id - if not exact.is_file() and not (video_id.endswith('.mp4') and with_ext.is_file()): - matches = list(videos.glob(video_id.replace('[', '[[]') + '*.mp4')) - _require(len(matches) == 1, 'DVC source video ID has missing or ambiguous media') - stem = adapter._resolve_video_stem(videos, video_id) - _require(stem is not None, 'DVC source video cannot be resolved') - _local_path(root, videos / (stem + '.mp4')) - resolved.append(stem) - _require(len(set(resolved)) == len(resolved), 'DVC has multiple annotations for the same video') - elif task == '2DGrounding': - for item in _records(paths[0]): - _require(_text(item.get('media')), 'Grounding source lacks image identity') - _local_path(root, source / item['media']) - meta = item.get('_meta') or {} - _require(_text(meta.get('sentence')), 'Grounding source lacks referring expression') - _require(_number(meta.get('image_width')) > 0 and _number(meta.get('image_height')) > 0, - 'Grounding source lacks real image dimensions') - turns = [turn for turn in item.get('conversations', []) if turn.get('from') == 'gpt'] - _require(len(turns) == 1, 'Grounding source lacks an unambiguous ground-truth response') - boxes = json.loads(turns[0].get('value', '')) - _require(isinstance(boxes, list) and bool(boxes), 'Grounding source lacks bounding-box labels') - for box in boxes: - _bbox(box) - elif task == '2DPointing': - frame = pd.read_csv(paths[0], sep='\t', keep_default_na=False) - _require({'index', 'question', 'answer', 'target_point'}.issubset(frame.columns) and len(frame) > 0, - 'Pointing source lacks original IDs, answers, or target points') - _require(not frame['index'].map(str).duplicated().any(), 'Pointing source contains duplicate IDs') - _require(frame['answer'].isin(list('ABCD')).all(), 'Pointing source lacks A-D ground truth') - _require(frame['target_point'].map(lambda value: str(value).strip() != '').all(), 'Pointing source lacks target-point labels') - for value in frame['target_point']: - try: - point = ast.literal_eval(str(value)) - except (SyntaxError, ValueError) as exc: - raise ValueError('Malformed Pointing target-point ground truth') from exc - _require(isinstance(point, (list, tuple)) and len(point) == 2, 'Pointing target must have two coordinates') - for coordinate in point: - _number(coordinate) - from .smp import toliststr - _require('image_path' in frame, 'Pointing source lacks local image identities') - for _, row in frame.iterrows(): - images = toliststr(row['image_path']) - _require(len(images) == 1 and _text(images[0]), 'Pointing source requires one local image identity') - _local_path(root, source / images[0]) - elif task == 'SOT': - metadata = {} - labels = set() - metadata_path = (dataset_kwargs or {}).get('metadata_path') - if metadata_path: - metadata_path = _local_path(root, metadata_path) - for line in metadata_path.read_text(encoding='utf-8').splitlines(): - if line.strip(): - entry = json.loads(line) - _require(_text(entry.get('seq_id')) and entry['seq_id'] not in metadata, 'Malformed or duplicate SOT metadata identity') - metadata[entry['seq_id']] = entry - for path in paths: - meta = _json(path) - if 'frame_ids' in meta: - ids = meta['frame_ids'] - boxes = meta.get('gt_bboxes') - _require(_text(meta.get('label')), 'SOT source lacks original sequence label') - _require(isinstance(boxes, dict), 'SOT source lacks per-frame ground truth') - _require(isinstance(ids, list) and len(ids) >= 2 and all(type(i) is int for i in ids) - and len(set(ids)) == len(ids), 'Malformed SOT frame identities') - expected = {str(i) for i in ids[1:]} - _require(expected.issubset(boxes) and set(boxes).issubset({str(i) for i in ids}), - 'SOT ground truth must explicitly label every evaluation frame, including absent objects') - else: - count = meta.get('nframes') - _require(type(count) is int and count >= 2, 'Malformed compact SOT frame count') - ids = list(range(count)) - boxes = meta.get('bboxes') - _require(isinstance(boxes, dict), 'Compact SOT lacks frame labels') - expected = {f'frame_{i}' for i in ids[1:]} - _require(set(boxes) == expected, 'Compact SOT must explicitly label every evaluation frame') - entry = metadata.get(path.parent.name, {}) - init_bbox = meta.get('init_bbox') or entry.get('init_bbox') - _bbox(init_bbox) - if 'frame_ids' in meta and str(ids[0]) in boxes: - _require(boxes[str(ids[0])] == init_bbox, 'SOT initialization box disagrees with its frame ground truth') - label = meta.get('label') if 'frame_ids' in meta else entry.get('label') or path.parent.name.replace('__', '/') - _require(_text(label) and label not in labels, 'SOT sequence labels must be present and unique') - labels.add(label) - for box in boxes.values(): - if box is not None: - _bbox(box) - for index in range(len(ids)): - _local_path(root, path.parent / 'frames' / f'f{index:02d}.png') - _local_path(root, path.parent / 'frames/crop.png') - if metadata_path: - paths.append(metadata_path) - else: - from .dataset.metropolis2d.astro_2d_dataset import Astro2DDetectionDataset, parse_kitti_label - for layout in Astro2DDetectionDataset._LAYOUT.values(): - sequence = source / layout['data_root'].rsplit('/', 1)[1] - images_dir = _local_path(root, sequence / 'images', directory=True) - images = [p for p in sorted(images_dir.iterdir()) if p.suffix.lower() in ('.jpg', '.jpeg', '.png', '.bmp', '.tiff')] - _require(bool(images), 'ASTRO sequence has no images') - stems = [p.stem for p in images] - _require(len(set(stems)) == len(stems), 'ASTRO image identities have ambiguous label stems') - labels = {p.stem: p for p in paths if p.parent == sequence / 'labels'} - _require(set(stems) == set(labels), 'ASTRO requires an independent label file for every image, and no unmatched labels') - for image in images: - _local_path(root, image) - label = labels[image.stem] - lines = [line for line in label.read_text(encoding='utf-8').splitlines() if line.strip()] - for line in lines: - fields = line.split() - _require(len(fields) >= 15, 'Malformed KITTI ground-truth label') - # Preserve the native KITTI policy, including finite inverted - # boxes. The scorer clamps their area to zero; deleting or - # repairing them would change its ground-truth denominator. - for value in fields[4:8]: - _number(value) - _require(len(parse_kitti_label(str(label))) == len(lines), 'Native ASTRO parser skipped a malformed label') - return paths - - -def _duration(video): - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', str(video)], - check=True, capture_output=True, text=True, - ) - duration = _number(result.stdout.strip()) - _require(duration > 0, 'Video has no usable duration metadata') - return duration - - -def _sha256(path): - digest = hashlib.sha256() - with Path(path).open('rb') as stream: - for block in iter(lambda: stream.read(1024 * 1024), b''): - digest.update(block) - return digest.hexdigest() - - -def _snapshot(frame, root, target): - frame = frame.copy() - for column in ('image_path', 'video_path'): - if column in frame: - frame[column] = frame[column].map( - lambda value: str(Path(value).relative_to(root)) - if _text(value) and Path(value).is_absolute() and Path(value).is_relative_to(root) else value - ) - frame.to_csv(target, sep='\t', index=False) - - -def prepare_assets(assets_dir, asset_id, tasks=None, bert_score_model=None, bert_score_revision=None, - sot_metadata_path=None): - """Build native canonical tables and a fingerprinted local scoring manifest.""" - from .dataset import ( - MetropolisVQA, MetropolisTemporal, MetropolisEventVerification, MetropolisDVC, - Metropolis2DGroundingDataset, VANTAGE_2DPointing, VANTAGE_SOT, Astro2DDetectionDataset, - ) - root = Path(assets_dir).expanduser().resolve(strict=True) - _require(root.is_dir() and _text(asset_id), 'Preparation requires a local source directory and nonempty asset ID') - selected = list(tasks or TASKS) - _require(bool(selected) and len(set(selected)) == len(selected) and set(selected).issubset(TASKS), 'Invalid or duplicate preparation tasks') - manifest = {'format_version': 1, 'asset_id': asset_id, 'tasks': {}, 'files': {}} - if 'DVC' in selected: - _require(_text(bert_score_revision), 'DVC model requires an explicit immutable revision identifier') - _require(bert_score_model is not None and not Path(bert_score_model).is_absolute(), 'BERTScore model path must be relative to the bundle') - model = _local_path(root, bert_score_model, directory=True) - validate_bert_score_model(model) - model_files = {} - for path in sorted(model.rglob('*')): - if path.is_file() and '.cache' not in path.relative_to(model).parts: - _local_path(root, path) - model_files[path.relative_to(model).as_posix()] = _sha256(path) - manifest['metric_models'] = {'roberta_large': { - 'path': model.relative_to(root).as_posix(), 'revision': bert_score_revision, - 'files': model_files, - }} - # Validate every requested original source before writing generated tables. - sot_kwargs = {} - if sot_metadata_path: - _require(not Path(sot_metadata_path).is_absolute(), 'SOT metadata path must be relative to the bundle') - sot_kwargs['metadata_path'] = _local_path(root, sot_metadata_path).relative_to(root).as_posix() - originals = { - task: validate_task_sources(root, task, {'prepared_data_dir': SOURCE_DIRS[task], **sot_kwargs} if task == 'SOT' else None) - for task in selected - } - for task in selected: - source_diagnostics = {} - source = root / SOURCE_DIRS[task] - table = root / f'VANTAGE_{task}.tsv' - _require(not table.is_symlink() and (not table.exists() or table.stat().st_nlink == 1), - 'Refusing to overwrite a linked canonical output table') - kwargs = {} - if task in ('VQA', 'Temporal'): - adapter = (MetropolisVQA if task == 'VQA' else MetropolisTemporal)(test_mode=True) - for mapping_dir in (source / 'data_jsons/category_mapping', source / 'mappings'): - if mapping_dir.is_dir(): - for mapping in mapping_dir.glob('*.json'): - _local_path(root, mapping) - _require(isinstance(_json(mapping), dict), 'Malformed native category mapping') - adapter.category_mapping.update(adapter._load_category_mapping(str(mapping_dir))) - rows = [] - durations = {} - for path in originals[task]: - for item in _records(path): - record = {**item, 'gt_option': _vqa_option(item.get('gt_option'))} if task == 'VQA' else item - row = adapter._process_annotation_item(record) - _require(row is not None, f'Native {task} converter rejected a validated source record') - if task == 'Temporal' and 'duration' not in item: - filename = row['video'] if row['video'].endswith('.mp4') else row['video'] + '.mp4' - video = source / 'videos' / filename - if video not in durations: - durations[video] = _duration(video) - row['duration'] = durations[video] - rows.append(row) - rows.sort(key=lambda row: (row['video'], row.get('qid', ''))) - for index, row in enumerate(rows): - row['index'] = index - frame = pd.DataFrame(rows) - elif task == 'EventVerification': - frame = pd.DataFrame(MetropolisEventVerification._load_items(source)) - if 'index' not in frame: - frame['index'] = range(len(frame)) - frame['index'] = frame['index'].astype(str) - elif task == 'DVC': - adapter = MetropolisDVC(test_mode=True) - adapter._generate_tsv_from_events(source, table) - frame = pd.read_csv(table, sep='\t', keep_default_na=False) - _require(len(frame) == len(originals[task]), 'Native DVC conversion dropped original annotations') - elif task == '2DGrounding': - kwargs = {'data_root': SOURCE_DIRS[task], 'annotation_file': originals[task][0].relative_to(root).as_posix()} - dataset = Metropolis2DGroundingDataset(dataset='VANTAGE_2DGrounding', model_family='cr', - data_root=str(source), annotation_file=str(originals[task][0])) - frame = dataset.data - _require(len(frame) == len(_records(originals[task][0])), 'Native Grounding loader dropped original annotations') - elif task == '2DPointing': - kwargs = {'data_root': SOURCE_DIRS[task]} - dataset = VANTAGE_2DPointing(data_root=str(source), model_family='cr') - frame = dataset.data - _require(len(frame) == len(pd.read_csv(originals[task][0], sep='\t')), 'Native Pointing loader dropped original annotations') - elif task == 'SOT': - kwargs = {'prepared_data_dir': SOURCE_DIRS[task], **sot_kwargs} - native_kwargs = {'prepared_data_dir': str(source), 'model_family': 'cr'} - if sot_metadata_path: - native_kwargs['metadata_path'] = str(_local_path(root, sot_metadata_path)) - frame = VANTAGE_SOT(**native_kwargs).data - _require(len(frame) == sum(path.name == 'gt.json' for path in originals[task]), - 'Native SOT loader dropped original sequences') - else: - from .dataset.metropolis2d.astro_2d_dataset import compute_bbox_area - roots = {} - frames = [] - offset = 0 - for name, layout in Astro2DDetectionDataset._LAYOUT.items(): - sequence = source / layout['data_root'].rsplit('/', 1)[1] - roots[name] = sequence.relative_to(root).as_posix() - child = Astro2DDetectionDataset(dataset=name, data_root=str(sequence), model_family='cr') - non_positive = sum( - obj['label'] == 'person' and compute_bbox_area(obj['bbox']) == 0 - for objects in child._gt_cache.values() for obj in objects - ) - if non_positive: - source_diagnostics[name] = { - 'non_positive_area_person_ground_truth': non_positive, - 'retained_by_native_min_area': non_positive if child.min_bbox_area <= 0 else 0, - 'native_min_bbox_area': child.min_bbox_area, - 'policy': 'Original labels unchanged; native filtering and denominators preserved', - } - child_frame = child.data.copy() - child_frame['original_index'] = child_frame['index'] - child_frame['SUB_DATASET'] = name - child_frame['index'] = range(offset, offset + len(child_frame)) - offset += len(child_frame) - frames.append(child_frame) - frame = pd.concat(frames, ignore_index=True) - kwargs = {'data_roots': roots} - if task in VIDEO_TASKS: - _validate_table(frame, task) - frame.to_csv(table, sep='\t', index=False) - video_root = source if task == 'EventVerification' else source / 'videos' - validate_local_video_inputs(table, video_root, task) - kwargs = {'local_data_file': table.name, 'local_video_root': video_root.relative_to(root).as_posix()} - else: - _snapshot(frame, root, table) - manifest['tasks'][task] = {'rows': len(frame), 'dataset_kwargs': kwargs} - if source_diagnostics: - manifest['tasks'][task]['annotation_diagnostics'] = source_diagnostics - for path in originals[task] + [table]: - manifest['files'][path.relative_to(root).as_posix()] = _sha256(path) - # Category mappings influence native category-level metrics and must also - # be fingerprinted when used by the video converters. - if task in ('VQA', 'Temporal'): - for mapping_dir in (source / 'data_jsons/category_mapping', source / 'mappings'): - if mapping_dir.is_dir(): - for path in sorted(mapping_dir.glob('*.json')): - path = _local_path(root, path) - manifest['files'][path.relative_to(root).as_posix()] = _sha256(path) - output = root / 'scoring-assets.json' - _require(not output.is_symlink() and (not output.exists() or output.stat().st_nlink == 1), - 'Refusing to overwrite a linked scoring manifest') - output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n', encoding='utf-8') - return manifest - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--assets-dir', required=True) - parser.add_argument('--asset-id', required=True) - parser.add_argument('--tasks', nargs='+', choices=TASKS) - parser.add_argument('--bert-score-model', help='Relative local roberta-large model directory inside the bundle') - parser.add_argument('--bert-score-revision', help='Immutable source revision of the local model assets') - parser.add_argument('--sot-metadata-path', help='Optional relative original SOT metadata JSONL inside the bundle') - args = parser.parse_args(argv) - manifest = prepare_assets(args.assets_dir, args.asset_id, args.tasks, args.bert_score_model, - args.bert_score_revision, args.sot_metadata_path) - print(json.dumps({'asset_id': manifest['asset_id'], 'rows': {task: item['rows'] for task, item in manifest['tasks'].items()}}, sort_keys=True)) - - -if __name__ == '__main__': - main() From 32f5432f529edded8c2ad7bd2e269c06ab9c84cb Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Thu, 29 Oct 2026 20:18:49 -0500 Subject: [PATCH 4/6] Record MVBench native notebook validation and NTU coverage gate Signed-off-by: Jon Patterson --- .../cosmos3/reasoner/vlmevalkit/launchable/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 04ec8a700..a507f2dba 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -84,7 +84,16 @@ The local wrapper additionally prevents punctuation-only option tokens from rece `run.json` records scope, exact selected IDs, coverage, dataset revision/manifest hash, complete emitted native config, frame count, API concurrency, generation parameters, model/deployment revision, Framework revision, source hashes, prediction location, metric summary and state transitions. `config.json`, streamed `run.log`, native predictions, `scores/summary.json`, and `/opt/evaluation-packages.txt` provide private reproducibility artifacts. Summaries retain prediction/scorer provenance. API concurrency affects scheduling and host RAM usage, not the frame count or generation protocol. Increase it only after measuring host resources. -No published-score reproduction, reference-result parity, immutable cold-launch reliability, or complete customer-data agent is implied by implementation. Full NTU coverage is a real acquisition prerequisite. This change's empirical hardware/notebook status must be established separately; do not carry forward evidence from a different benchmark. No benchmark labels/media, credentials, executed notebook outputs, or private result artifacts belong in the PR. +### Observed validation (2026-10-29) + +Runtime candidate `b0861ce448a648b2eb59c41a9358980bbb5df31c` was built and exercised on native Linux x86-64: + +- All **28 behavioral regressions passed inside the newly built evaluator image**. +- JupyterLab **Run All Cells** executed all six code cells against the pinned Nano endpoint on one RTX PRO 6000 Blackwell Server Edition. The fixed five-question subset produced five valid predictions with matching identities and native local scores; correctness results remain private and are not a full benchmark score. +- The saved real predictions were rescored in a separate CPU-only container with **`--network none`**, with the same correctness counts. +- SHA-verified current public archives were downloaded and safely extracted. Actual full preparation found media for **3,800 of 4,000 rows** and rejected the run before inference because the **200 Fine-grained Pose NTU clips** were missing. No rows were dropped and no partial full score was produced. + +This reused an existing warm GPU endpoint with a manually staged evaluator; it was **not a cold one-click launch**. The complete 20-task run remains blocked on authorized NTU acquisition. No published-score reproduction, reference-result parity, commercial/distribution approval, or customer-data agent implementation is claimed. No benchmark labels/media, credentials, executed notebook outputs, or private result artifacts belong in the PR. ## Stop, restart, and maintain From b7bece133db0bf88b32326ec10073585eaf77427 Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Fri, 30 Oct 2026 11:21:42 -0500 Subject: [PATCH 5/6] Add public Perception Test validation to the Nano Brev notebook --- .../cosmos3/reasoner/vlmevalkit/.dockerignore | 9 +- .../vlmevalkit/EXTERNAL_CONFIGURATION.md | 84 +-- .../cosmos3/reasoner/vlmevalkit/README.md | 22 +- .../reasoner/vlmevalkit/launchable/Dockerfile | 2 +- .../launchable/Evaluate_Cosmos3.ipynb | 137 ++--- .../reasoner/vlmevalkit/launchable/README.md | 252 ++++++--- .../vlmevalkit/launchable/compose.yaml | 12 +- .../vlmevalkit/launchable/launchable.py | 201 ++++--- .../reasoner/vlmevalkit/launchable/setup.sh | 18 +- .../vlmevalkit/launchable/test_launchable.py | 238 +++++--- .../vlmevalkit/launchable/test_mvbench.py | 368 ------------ .../launchable/test_perception_test.py | 462 +++++++++++++++ .../vlmevalkit/vlmeval/dataset/__init__.py | 5 +- .../vlmevalkit/vlmeval/dataset/mvbench.py | 126 +---- .../vlmeval/dataset/perception_test.py | 147 +++++ .../vlmevalkit/vlmeval/mvbench_local.py | 525 ------------------ .../vlmeval/perception_test_local.py | 494 ++++++++++++++++ 17 files changed, 1720 insertions(+), 1382 deletions(-) delete mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_perception_test.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/perception_test.py delete mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/perception_test_local.py diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore index 1969035f7..8153879df 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore @@ -14,8 +14,8 @@ launchable/state **/*.key **/scoring-assets.json **/scoring-assets -**/.mvbench -**/mvbench-data +**/.perception-test +**/perception-test-data **/*.tsv **/*.xlsx **/*.pkl @@ -24,3 +24,8 @@ launchable/state cosmos_eval_out out outputs +**/cache +**/*.mp4 +**/*.zip +**/mc_question_valid.json +**/cut_frame_mapping_valid.json diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md index 6a8d77271..7454e4706 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md @@ -41,34 +41,37 @@ remain available. Missing or nonexistent grounding annotation files produce a clear `ValueError`. Omni3D retains its existing private configuration and can be pointed at another authorized dataset through `raw_image_path` and `json_path`. -## Brev Nano and public MVBench evaluation +## Brev Nano and public Perception Test validation The [Brev launchable](launchable/README.md) uses the Framework-native kit, -the pinned Nano revision, and `OpenGVLab/MVBench` at -`230a2d4fac8900333c61754641c7a13e069ac9c6`. It provides public download, -coverage validation, native inference, and local exact-match scoring for 20 -tasks and 4,000 annotated multiple-choice questions. No private annotation -bundle, private S3, GPU, or model endpoint is required for score-only execution. +the pinned public Nano checkpoint, and Google DeepMind's public Perception Test +validation MCQ data: 19,140 questions over 5,260 videos. Default smoke selects +`video_8241`, official question IDs 0–4 (stable indices 13404–13408). +Full validation is explicit opt-in, never the held-out test split. +Dataset source revision is `3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf`; +official annotations, cut mapping, README and public archives are SHA-256 pinned +in `vlmeval/perception_test_local.py`. No private dataset or dataset credentials +are required. The Perception Test authors' non-code materials are +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); retain attribution to +the [authors/source](https://github.com/google-deepmind/perception_test). +Code is Apache 2.0. Public access is not an additional clearance determination. Set `LAUNCHABLE_STATE_DIR` to an absolute writable host directory outside the -checkout. `MVBENCH_DATA_DIR` defaults to its `mvbench` child and is mounted -read-write at `/mvbench`; cache and results use separate mounts. The notebook -downloads the selected public scope itself. Approved media terms restrict use -to academic research; public access is not commercial-use or redistribution -permission. Full coverage requires manually acquired authorized NTU RGB+D -media under `video/nturgbd`; removed videos are never fetched from historical -revisions or mirrors. Missing media produces a coverage error, not an incomplete -full score. Default smoke is five fixed Egocentric Navigation rows. +checkout. `PERCEPTION_TEST_DATA_DIR` defaults to its `perception-test` child, +mounted read-write at `/perception-test`; cache and results use separate mounts. +Compose project is `cosmos3-perception-test`, evaluator service `perception-eval`. +Authenticated Jupyter binds only to a private or loopback address. The local +model has no host port; no Docker socket or cloud credentials are mounted. Native `run.py` configuration uses the local dataset class directly: ```json { "data": { - "MVBench": { - "class": "MVBenchLocal", - "dataset": "MVBench", - "data_root": "/mvbench", + "PerceptionTest": { + "class": "PerceptionTestLocal", + "dataset": "PerceptionTest", + "data_root": "/perception-test", "nframe": 16, "smoke": false } @@ -77,18 +80,33 @@ Native `run.py` configuration uses the local dataset class directly: ``` The controller composes this with `CosmosReason2` and the existing generation -recipe, defaulting to one API worker. Temporal sampling and prompts are native -MVBench, with sampled-image input and isolated clip/frame caches. The starting -16-frame target and scale/center-crop-224 frame preprocessing follow the original -recipe; Cosmos image encoding/normalization and answer parsing still differ from VideoChat2. -Native count-based accuracy (fraction), task macro, coverage and provenance -remain explicit. No LLM judge or published-score claim is involved. - -Use `launchable.py score --scope smoke|full` with either an existing controller -`--work-dir` or native `--predictions` plus a fresh `--score-dir`. Exact selected -identities and complete coverage are required. Trusted answers come from pinned -annotations, never prediction-supplied fields. Score-only skips endpoint checks -and model construction. BYOM uses `COSMOS_API_BASE`, `COSMOS_MODEL`, optional -environment-only `COSMOS_API_KEY`, and an explicit `COSMOS_MODEL_REVISION`; -setup then skips local serving. This launchable does not implement a -customer-data evaluation agent or alter other model deployment tracks. +recipe, defaulting to one API worker. The launchable prototype samples 16 frames +uniformly inside official source cutpoints: `-1` means all frames, positive cuts +are exclusive stops and invalid cuts fail. Aspect-preserving downscale only to +maximum side 768, without cropping, precedes direct sampled-image input. This +protocol and text-to-option parser are adaptations, not official parser parity +or a published-model/reference-score reproduction. + +`launchable.py download --scope metadata|smoke|full` stages public inputs. +`prepare --smoke` and `prepare` require complete smoke/full media respectively. +`run --mode infer|all` generates predictions and optionally scores them; the +notebook defaults to `all`, not a manual Docker benchmark command. +`score --scope smoke|full` accepts either an existing successful inference +`--work-dir` or native `--predictions` plus a fresh `--score-dir`. Score-only +downloads metadata only and prepares with `require_media=False`: no video files, +endpoint, GPU, or model construction. Canonical annotation/cut/scope fingerprints +are independent of media presence; media hashes are recorded separately. + +Exact selected identities are mandatory. Missing/blank/transport-failure rows +reject incomplete runs; nonempty malformed/ambiguous answers count wrong and +invalid. Trusted labels come only from pinned annotations. Primary metric is +local top1 (`correct / total selected questions`), with per-area, per-reasoning +and per-tag diagnostics, never an API judge. Fresh outputs, failed-run artifacts, +streamed progress, source/model revisions and UTC elapsed timings are retained. +The notebook shows concise metrics/coverage and artifact links. + +BYOM uses complete `COSMOS_API_BASE`, exact `COSMOS_MODEL`, optional +environment-only `COSMOS_API_KEY`, and explicit `COSMOS_MODEL_REVISION`; +setup then skips local serving. Score-only setup uses `bash setup.sh --score-only`. +This launchable does not implement a customer-ingestion evaluation agent or alter +other model deployment tracks. Source changes are not hardware or accuracy proof. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md index 934333e4b..0ff62e995 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Cosmos3 evaluation: MVBench launchable and native benchmark recipes +# Cosmos3 evaluation: Perception Test launchable and native benchmark recipes This directory is an **NVIDIA-customized [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)** maintained in Cosmos Framework. It contains reference settings for NVIDIA @@ -15,16 +15,16 @@ datasets publicly reproducible or establish official score parity. For user-hosted endpoints and local dataset inputs, see [Deployment and dataset configuration](EXTERNAL_CONFIGURATION.md). For the Nano-only Brev notebook, use the -[MVBench launchable](launchable/README.md): 20 tasks, 4,000 public annotated -multiple-choice questions, pinned public downloads, native inference, and local -exact-match scoring without an API judge. Its default smoke is five Egocentric -Navigation questions, not a full score; full evaluation is explicit opt-in. -The approved dataset revision requires manual authorized acquisition of 320 NTU -RGB+D videos and restricts video use to academic research. Public access does -not establish commercial rights, redistribution permission, or published-score -parity. Data and state stay outside the checkout. This is not a customer-data -evaluation-agent implementation. Existing native recipes below are independent -of this launchable and remain unchanged. +[Perception Test launchable](launchable/README.md): 19,140 public validation +multiple-choice questions across 5,260 videos, pinned public downloads, native +inference, and local top1 scoring without an API judge. Default smoke selects +five questions from `video_8241`; full validation is explicit opt-in, not the +held-out test split. Public dataset materials are CC BY 4.0 with attribution to +the Perception Test authors; code is Apache 2.0. The prototype uses 16 sampled +images within official source cutpoints, aspect-preserving downscale to a +maximum side of 768, and makes no reference/published-score parity claim. +Data and state stay outside the checkout. This is not a customer-ingestion +evaluation agent. Existing native recipes below remain independent and unchanged. The kit ships, for each benchmark, reference `run.py --config` settings — split into a shared dataset layer (`cosmos_eval/data/`) and a per-family model layer diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile index bbd99db88..1f3623987 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile @@ -16,6 +16,6 @@ RUN pip install --no-cache-dir --no-deps -e . \ ARG COSMOS_COMMIT=unknown ENV COSMOS_COMMIT=${COSMOS_COMMIT} PYTHONUNBUFFERED=1 HF_HOME=/cache/huggingface \ AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null \ - AWS_EC2_METADATA_DISABLED=true MVBENCH_DATA_DIR=/mvbench LMUData=/cache/lmudata + AWS_EC2_METADATA_DISABLED=true PERCEPTION_TEST_DATA_DIR=/perception-test LMUData=/cache/lmudata WORKDIR /opt/vlmevalkit/launchable CMD ["python", "jupyter_config.py"] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index 6d3774e61..9c91b7be9 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -4,26 +4,24 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Evaluate Cosmos3-Nano on MVBench\n", + "# Evaluate Cosmos3-Nano on Perception Test validation MCQ\n", "\n", - "**Download public inputs → validate coverage → infer → score locally.** The default is the fixed five-question Egocentric Navigation smoke, not a full score. Set `RUN_FULL = True` to select full evaluation directly; it does not rerun smoke first.\n", + "**Public download → validate coverage → native inference → local top1.** Default smoke selects five questions (IDs 0–4) from `video_8241`, stable indices 13404–13408. Full validation is explicit opt-in: 19,140 questions / 5,260 MCQ videos, **not the held-out test split**.\n", "\n", - "[MVBench](https://arxiv.org/abs/2311.17005) contains 4,000 annotated multiple-choice questions across 20 video-understanding tasks. This notebook uses the native VLMEvalKit MVBench task definitions, temporal bounds, prompts, and exact-match evaluator. There is no LLM judge or automatic upload. The Nano serving revision is `e59a53c25979a090fa8706c9acc0c254a6e89b92`.\n", + "Dataset attribution: [Perception Test authors / Google DeepMind](https://github.com/google-deepmind/perception_test), [paper](https://arxiv.org/abs/2305.13786). Their code is Apache 2.0 and other materials [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); retain attribution/license link and identify adaptations. This launchable is an adaptation, not author endorsement or a customer-ingestion evaluation agent.\n", "\n", - "**Research-only media terms:** public access does not establish commercial rights or redistribution permission. The card mentions 320 NTU videos, but this pinned revision's manual list contains 200 RGB AVI files for its 200 Fine-grained Pose questions. Obtain authorized access from https://rose1.ntu.edu.sg/dataset/actionRecognition/ and use the pinned video/MVBench_videos_ntu.txt list. Removed media is never downloaded from historical revisions or mirrors. No published-score reproduction or commercial-rights claim is made. This is not a customer-data evaluation-agent implementation.\n" + "The public Nano checkpoint is pinned to `e59a53c25979a090fa8706c9acc0c254a6e89b92`. This committed notebook is unexecuted; no empirical validation or reference-score parity is asserted." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1 · Choose one execution scope\n", + "## 1 · Choose scope and execution mode\n", "\n", - "The external host `MVBENCH_DATA_DIR` is writable at `/mvbench`; caches and results are separate external mounts. No AWS credentials, Docker socket, labels, media, or model weights are embedded in this image.\n", + "The external `PERCEPTION_TEST_DATA_DIR` is mounted at `/perception-test`; cache and results are separate. No model/data credentials, Docker socket or dataset artifacts are embedded. `RUN_FULL = True` selects full validation directly without rerunning smoke.\n", "\n", - "For **score-only**, choose `SCORE_SCOPE` explicitly (`smoke` or `full`) and set exactly one of `PREDICTIONS_FILE` (native TSV/XLSX/JSON) or `EXISTING_WORK` (a completed controller inference directory). No endpoint check, GPU, model construction, or private S3 access is used. The selected scope must match the predictions exactly; arbitrary partial scores are not accepted.\n", - "\n", - "For full runs, stage authorized NTU files under `/mvbench/video/nturgbd` using Jupyter's authenticated file browser/terminal before preparation. The download cell retrieves only currently approved public files. Preparation reports missing media and refuses to run an incomplete full benchmark.\n" + "For **score-only**, choose explicit `SCORE_SCOPE` and exactly one of `PREDICTIONS_FILE` or `EXISTING_WORK`. Only pinned metadata is needed: no video, model, GPU or endpoint. Set `DOWNLOAD_PUBLIC = False` for already-staged offline scoring." ] }, { @@ -43,20 +41,20 @@ "SCORE_ONLY = os.environ.get('EVAL_SCORE_ONLY') == '1'\n", "PREDICTIONS_FILE = None\n", "EXISTING_WORK = None\n", - "SCORE_SCOPE = 'smoke' # Explicitly change to 'full' for 4,000 existing predictions.\n", - "RUN_FULL = False # Full inference is opt-in; selecting it skips the smoke.\n", + "SCORE_SCOPE = 'smoke' # Choose 'full' explicitly for 19,140 existing predictions.\n", + "RUN_FULL = False\n", "DOWNLOAD_PUBLIC = True\n", - "NFRAME = 16 # Reference frame count; not a model-score parity claim.\n", + "NFRAME = 16 # Fixed launchable prototype protocol.\n", "API_NPROC = 1\n", - "DATA = Path(os.environ.get('MVBENCH_DATA_DIR', '/mvbench'))\n", + "DATA = Path(os.environ.get('PERCEPTION_TEST_DATA_DIR', '/perception-test'))\n", "SCOPE = SCORE_SCOPE if SCORE_ONLY else ('full' if RUN_FULL else 'smoke')\n", "STAMP = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')\n", - "WORK = Path('/results') / f'mvbench-{SCOPE}-{STAMP}'\n", - "SCORE_DIR = Path('/results') / f'mvbench-score-{SCOPE}-{STAMP}'\n", - "CLI = [sys.executable, '-u', str(Path('/opt/vlmevalkit/launchable/launchable.py'))]\n", + "WORK = Path('/results') / f'perception-test-{SCOPE}-{STAMP}'\n", + "SCORE_DIR = Path('/results') / f'perception-test-score-{SCOPE}-{STAMP}'\n", + "CLI = [sys.executable, '-u', '/opt/vlmevalkit/launchable/launchable.py']\n", "\n", "def invoke(command):\n", - " # Stream instead of buffering an entire long-running inference job.\n", + " # Stream each line into Jupyter, not a buffer held until the run ends.\n", " with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,\n", " text=True, bufsize=1) as process:\n", " for line in process.stdout:\n", @@ -67,18 +65,20 @@ "if SCOPE not in {'smoke', 'full'}:\n", " raise ValueError('Choose explicit smoke or full scope.')\n", "if SCORE_ONLY and (PREDICTIONS_FILE is None) == (EXISTING_WORK is None):\n", - " raise ValueError('Set exactly one of PREDICTIONS_FILE or EXISTING_WORK for score-only execution.')\n", - "print('Score existing predictions' if SCORE_ONLY else 'Generate and score predictions', SCOPE)\n", - "print('Data:', DATA, '| frames:', NFRAME, '| concurrent API workers:', API_NPROC)\n" + " raise ValueError('Set exactly one of PREDICTIONS_FILE or EXISTING_WORK.')\n", + "print(('Score existing predictions' if SCORE_ONLY else 'Infer and score'), SCOPE)\n", + "print('Data:', DATA, '| frames:', NFRAME, '| API workers:', API_NPROC)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2 · Download the chosen public scope\n", + "## 2 · Download and prepare the selected public scope\n", + "\n", + "The helper uses direct official public URLs, pinned SHA-256, and safe extraction. Source repository revision: `3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf`. Metadata comprises `mc_question_valid.json`, `cut_frame_mapping_valid.json`, and the attribution README. Smoke downloads the approximately 225 MB official sample archive; full downloads the 75,375,143,328-byte official validation archive, plus extraction space.\n", "\n", - "The downloader pins `OpenGVLab/MVBench` to `230a2d4fac8900333c61754641c7a13e069ac9c6`, fetches all 20 official annotation JSONs and licensing/NTU notes, and safely extracts selected current public archives, including the June-2024 `data0613` overlay where applicable. Smoke needs only the approximately 40 MB `vlnqa.zip` archive. Full download is substantially larger and **cannot supply the manual NTU videos**. Existing approved files are reused. Set `DOWNLOAD_PUBLIC = False` for offline scoring with already staged inputs.\n" + "Inference preparation requires every selected video before contacting the model. Score-only downloads metadata only and prepares with `require_media=False`. Canonical annotation/cut/scope fingerprints are independent of video presence, with media hashes recorded separately." ] }, { @@ -88,60 +88,26 @@ "outputs": [], "source": [ "if DOWNLOAD_PUBLIC:\n", - " invoke(CLI + ['download', '--data-root', str(DATA), '--scope', SCOPE])\n", + " invoke(CLI + ['download', '--data-root', str(DATA),\n", + " '--scope', 'metadata' if SCORE_ONLY else SCOPE])\n", "else:\n", - " print('Using already staged public annotations and media.')\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3 · Prepare trusted annotations and require complete selected coverage\n", - "\n", - "Smoke selects exactly stable indices 3400–3404: five of 4,000 rows, one of 20 tasks. Full requires all 4,000 rows and all 20 tasks; no missing clips or questions are silently dropped. Missing full NTU media is an actionable prerequisite failure, not a partial full score. A structured coverage report is retained under `/mvbench/.mvbench/`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + " print('Using staged pinned inputs; no download.')\n", "scope_flag = ['--smoke'] if SCOPE == 'smoke' else []\n", - "invoke(CLI + ['prepare', '--data-root', str(DATA)] + scope_flag)\n" + "invoke(CLI + ['prepare', '--data-root', str(DATA)] + scope_flag\n", + " + (['--metadata-only'] if SCORE_ONLY else []))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 4 · Check the endpoint only for inference\n", + "## 3 · Infer and score locally, or rescore existing predictions\n", "\n", - "BYOM can be selected before host setup with `COSMOS_API_BASE` (complete `/v1/chat/completions` URL), exact `COSMOS_MODEL`, optional `COSMOS_API_KEY`, and `COSMOS_MODEL_REVISION`. Keep keys in the environment, not this notebook. The endpoint must accept sampled-image messages; fixed frame count is meaningful because no re-encoded video is sent. Generation uses the existing CosmosReason2 recipe: deterministic temperature 0, seed 1, no thinking, maximum 8,192 output tokens.\n", + "**Prototype protocol:** exactly 16 uniform frames within official source cutpoints; `-1` means all frames, positive cuts are exclusive stop frames. Bad cuts fail rather than clamp. Aspect-preserving downscale only to maximum side 768, no crop. Sampled images go directly to the endpoint, never re-encoded video. This is not a reference-score or official letter-parser parity claim.\n", "\n", - "The default 16 frames and bicubic scale/center-crop to 224 follow the reference recipe. Frames are sent as sampled images; Cosmos owns downstream image encoding/normalization, so VideoChat2 tensor normalization is not applied and model/parser differences remain. There is no paper-score parity claim.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not SCORE_ONLY:\n", - " invoke(CLI + ['check', '--wait', '1800'])\n", - "else:\n", - " print('Score-only: endpoint check and model inference skipped.')\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5 · Native inference and local scoring, or score existing outputs\n", + "Native `CosmosReason2` generation uses temperature 0, seed 1, no thinking, max 8,192 output tokens. BYOM must accept sampled-image messages; configure endpoint/model/revision before host setup and keep API keys environment-only.\n", "\n", - "Progress streams here and into the private inference `run.log`. Fresh directories prevent stale prediction reuse. Inference and scoring reject wrong/missing/duplicate identities and blank or transport-failure outputs; the scorer reconstructs trusted identities and labels from pinned annotations. Malformed but nonempty model answers receive native wrong-answer semantics, not fabricated successes. Native exact matching runs locally without an API judge.\n" + "Progress streams here and into `run.log`. Fresh output directories prevent stale reuse. Exact selected IDs are required. Blank, missing, duplicate, unexpected or transport-failure rows reject incomplete runs before scoring. Nonempty malformed/ambiguous choices count wrong and invalid, not dropped; literal text `nan` remains model text. Trusted labels come only from pinned annotations, never prediction-supplied answers. No API judge, dummy answer or partial-full-score fallback." ] }, { @@ -157,19 +123,20 @@ " '--score-dir', str(SCORE_DIR), '--nframe', str(NFRAME)] + source)\n", " SUMMARY = SCORE_DIR / 'summary.json'\n", "else:\n", + " # Controller validates coverage before checking the model endpoint.\n", " invoke(CLI + ['run', '--data-root', str(DATA), '--mode', 'all',\n", " '--nframe', str(NFRAME), '--api-nproc', str(API_NPROC),\n", " '--work-dir', str(WORK)] + scope_flag)\n", - " SUMMARY = WORK / 'scores' / 'summary.json'\n" + " SUMMARY = WORK / 'scores' / 'summary.json'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 6 · Inspect metrics, coverage, and provenance\n", + "## 4 · Human result and durable artifacts\n", "\n", - "The primary metric is count-based multiple-choice accuracy as a **fraction**, with per-task metrics and the task-macro average reported separately. Smoke is prominently marked as a subset. Native scorer version/source, dataset revision and manifest, prediction hashes, selected IDs, frame protocol, generation parameters, and model/source revisions are recorded in private artifacts. A BYOM display name alone does not establish its checkpoint revision. No reference/published-score parity is claimed until independently established with an agreed protocol.\n" + "Top1 is correct option ID / **all selected questions**, with per-area, per-reasoning and per-tag diagnostics in the summary. Smoke is not a full benchmark. Artifact provenance records annotation/cut identities, separate media hashes, prediction hash, source/model revisions and frame protocol. Controller artifacts include useful UTC start/end and measured elapsed time, not invented cost or peak memory." ] }, { @@ -179,37 +146,35 @@ "outputs": [], "source": [ "summary = json.loads(SUMMARY.read_text())\n", - "print(json.dumps({key: summary[key] for key in\n", - " ('status', 'coverage', 'primary_metric', 'per_task', 'provenance')}, indent=2))\n", - "display(FileLink(str(SUMMARY), url_prefix='/files'))\n", + "metric = summary['primary_metric']\n", + "print(f\"Perception Test validation — {SCOPE}: {summary['status']}\")\n", + "print(f\"Top1: {metric['value']:.2%} ({metric['correct']}/{metric['total']})\")\n", + "print('Coverage:', json.dumps(summary['coverage']))\n", + "print('Prototype protocol; no reference-score parity claim.')\n", + "artifacts = [SUMMARY, SUMMARY.parent / 'controller.json']\n", "if not SCORE_ONLY:\n", - " invoke(CLI + ['results', '--work-dir', str(WORK)])\n", - " display(FileLink(str(WORK / 'run.json'), url_prefix='/files'))\n" + " artifacts += [WORK / 'run.json', WORK / 'run.log', WORK / 'config.json']\n", + "for artifact in artifacts:\n", + " if artifact.is_file():\n", + " display(FileLink(str(artifact), url_prefix='/files'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Stop and retain private results\n", + "## Stop and retain results\n", "\n", - "Container lifecycle commands belong in the **host** terminal, not this notebook: `docker compose --profile local stop` preserves state. `bash setup.sh --score-only` starts only authenticated Jupyter for existing-prediction scoring. There is no Docker socket in the evaluator and the local model API has no host port. Stopping containers does not stop underlying Brev VM billing; stop that instance separately after retaining authorized results.\n", + "In the **host terminal**, `docker compose --profile local stop` preserves mounted state. `bash setup.sh --score-only` starts only authenticated Jupyter. Evaluator service: `perception-eval`; Compose project: `cosmos3-perception-test`. There is no Docker socket or host model port. Stopping containers does not stop Brev VM billing; stop the instance separately.\n", "\n", - "Clear outputs before sharing. Do not commit restricted media, labels, credentials, or answer-bearing artifacts. Native Linux x86-64 is the supported runtime; code changes alone do not demonstrate notebook/hardware execution or full benchmark reproduction.\n" + "Do not commit credentials, videos, labels, predictions or answer-bearing artifacts. Clear outputs before sharing. The supported deployment runtime is native Linux x86-64. A source-only change does not demonstrate notebook/GPU execution or full validation reproduction." ] } ], "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.14" - } + "kernelspec": {"display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3"}, + "language_info": {"name": "python", "version": "3.11"} }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index a507f2dba..eb31a921e 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -1,111 +1,203 @@ -# Cosmos3-Nano MVBench evaluation on Brev +# Cosmos3-Nano · Perception Test validation MCQ on Brev -Authenticated Jupyter orchestration for **public download → preparation → native inference → local scoring → results**, entirely inside the notebook after host startup. Existing native predictions can also be scored on CPU without a model endpoint, GPU, private S3, API judge, or external scoring submission. This is a benchmark launchable, **not a customer-data evaluation-agent implementation**. +Run the public **Perception Test validation split**, from download to local top1, +inside an authenticated notebook. The default smoke is five questions (IDs 0–4) +from `video_8241`, stable indices 13404–13408. Full validation is opt-in: **19,140 +questions / 5,260 MCQ videos**. This is not the held-out test split. Source changes +and an unexecuted notebook are not evidence of hardware execution or accuracy. -[MVBench](https://arxiv.org/abs/2311.17005) has **20 tasks and 4,000 annotated multiple-choice questions**. The paper's [Ask-Anything repository](https://github.com/OpenGVLab/Ask-Anything/tree/main/video_chat2) supplies its original evaluation recipe. This launchable pins public [OpenGVLab/MVBench](https://huggingface.co/datasets/OpenGVLab/MVBench/tree/230a2d4fac8900333c61754641c7a13e069ac9c6) data to `230a2d4fac8900333c61754641c7a13e069ac9c6` and reuses native VLMEvalKit MVBench task definitions, temporal bounds, sampling and prompt construction, plus the native exact-match scorer. No labeled private annotation bundle is required. +This launchable is a standalone public-data validation workflow, not a +customer-ingestion evaluation agent. It needs no private annotation bundle, +internal dataset service, dataset credentials, or API judge. -**Licensing and full-data prerequisite:** read the pinned dataset card and individual source terms before downloading. Its video terms are academic-research-only; public access does not imply commercial-use or redistribution rights. The card mentions 320 NTU RGB+D videos, but this pinned revision's exact manual list contains **200 RGB AVI files**, matching its 200 Fine-grained Pose questions. Request the exact [pinned file list](https://huggingface.co/datasets/OpenGVLab/MVBench/blob/230a2d4fac8900333c61754641c7a13e069ac9c6/video/MVBench_videos_ntu.txt) through [NTU/ROSE Lab's account and release-agreement process](https://rose1.ntu.edu.sg/dataset/actionRecognition/); only RGB is needed. Supply authorized media under `video/nturgbd` in the external data root. The downloader never recovers removed media from old revisions or mirrors. Full preparation fails with an actionable coverage report while required media is missing; an incomplete download is not a full benchmark score. +## Start on Brev -## Start authenticated Jupyter +Use a native Linux x86-64 GPU instance with Docker Compose, NVIDIA container +support, and enough GPU capacity for the public Nano checkpoint. Full video +archive is 75,375,143,328 bytes before extraction; provision room for the archive, +extracted videos, model cache, and outputs. Smoke downloads the approximately +225 MB official sample archive; only `video_8241` is selected for inference. -Use native Linux x86-64, Docker Engine/Compose v2, and NVIDIA Container Toolkit for local Nano serving. The maintained source is [NVIDIA/cosmos-framework](https://github.com/NVIDIA/cosmos-framework), not `NVIDIA/cosmos`. Select the approved branch/commit containing this launchable and retain its exact revision. Disable Brev's managed Jupyter because Compose provides its own authenticated service. - -From the Framework checkout root: +From this directory on the host: ```bash -export LAUNCHABLE_STATE_DIR=/absolute/path/outside/checkouts/private-mvbench-state -# Optional: defaults to "$LAUNCHABLE_STATE_DIR/mvbench". -export MVBENCH_DATA_DIR=/absolute/path/outside/checkouts/mvbench-data -bash .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh --brev +export LAUNCHABLE_STATE_DIR=/mnt/cosmos3-perception-test-state +# Optional separate data disk; defaults to $LAUNCHABLE_STATE_DIR/perception-test. +export PERCEPTION_TEST_DATA_DIR=/mnt/perception-test-data +bash setup.sh --brev ``` -Omit `--brev` on a non-Brev host. Add **`--score-only`** to start only the CPU evaluator/Jupyter service, never a GPU endpoint. Public annotations and videos can be downloaded by the notebook itself; setup does not require preexisting dataset files or start inference. Host path/configuration validation completes before starting a paid model service. - -State and data directories must be absolute, writable, outside the checkout, and separate from each other’s cache/results mounts. Compose project `cosmos3-mvbench` mounts the external data directory **read-write at `/mvbench`**, cache at `/cache`, and private results at `/results`. Keep these path exports in the host shell for subsequent lifecycle commands. No weights or dataset contents are embedded in the image/repository. No Docker socket, AWS profile, home directory, or private key is mounted. - -`--brev` discovers the private `wt0` mesh IPv4 address and fails without it. An explicit private `JUPYTER_BIND_ADDRESS` takes precedence; non-Brev defaults to loopback. There is no public/wildcard fallback. Setup persists only the notebook token and non-secret bind address in restricted `.env`. Use Brev's authenticated secure port-8888 link or an SSH tunnel. From the launchable directory on the **host**, retrieve the login token with: +Without Brev, `bash setup.sh` binds Jupyter to loopback by default; use an SSH +tunnel. `--brev` discovers the private `wt0` mesh address and refuses public or +wildcard binding. Jupyter always requires a generated token. Retrieve the login +URL in the host terminal with: ```bash -docker compose exec mvbench-eval jupyter server list +docker compose exec perception-eval jupyter server list ``` -Never share that token or include it in screenshots. Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb`. Jupyter's authenticated file root is `/`, so authorized NTU files can be staged via its file browser/terminal and result links work. The local model API has **no host port**, only the private container network. Host Docker commands are for setup/stop/lifecycle, not benchmark execution. - -### Pinned Nano and BYOM - -The local service targets `nvidia/Cosmos3-Nano`, revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`, with serving image `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55`. `VLLM_IMAGE` can override the image, with the override recorded. Allow storage for weights, images, public video archives, extracted media, frames, and results. The launchable does not change Ultra/LaunchPad workflows. - -For BYOM, set `COSMOS_API_BASE` to the full reachable `/v1/chat/completions` URL and `COSMOS_MODEL` to the exact served ID **before setup**; optionally supply `COSMOS_API_KEY` in the environment. Setup skips local serving. Set `COSMOS_MODEL_REVISION` to your real deployment/checkpoint revision; unknown BYOM revisions remain explicitly `unknown` rather than being attributed to the pinned Nano checkpoint. Container `localhost` is not the host. The endpoint must accept sampled-image messages and expose `/v1/models`. Keys stay in the environment, never config/notebook source. - -## Execute one chosen notebook scope - -The notebook starts with `RUN_FULL = False`: exactly the first five Egocentric Navigation questions (stable native indices **3400–3404**), one task and five of 4,000 rows. It is prominently identified as a **smoke subset**, never a full result. Its public archive is approximately 40 MB (`video/vlnqa.zip`). - -Set **`RUN_FULL = True`** to opt into full evaluation directly. The notebook downloads the chosen scope, prepares it, checks the endpoint only for inference, runs inference plus scoring, and displays coverage/metrics/provenance. It does **not** unnecessarily rerun smoke before full. Full preparation requires all 20 official JSON files, all 4,000 questions and required media, including manually acquired NTU. Downloading current public archives alone cannot meet that gate. Missing coverage is written to `/mvbench/.mvbench/coverage-full.json` and reported clearly; nothing is silently truncated or dropped. - -For existing predictions, set `SCORE_ONLY = True`, choose explicit `SCORE_SCOPE = 'smoke'` or `'full'`, and provide exactly one of `PREDICTIONS_FILE` or `EXISTING_WORK`. Set `DOWNLOAD_PUBLIC = False` when using already staged inputs offline. Score-only does not check the endpoint or construct an inference model. - -### CLI equivalents inside Jupyter's evaluator environment - -These commands can also be invoked from Jupyter's terminal; no host Docker benchmark command is necessary: +Open **Evaluate_Cosmos3.ipynb** and run all cells. The notebook downloads the +selected public data, validates coverage **before any model request**, streams +native inference progress, scores locally, and displays top1/counts, coverage, +and artifact links. Leave `RUN_FULL = False` for smoke; set it to `True` to run +full validation directly, without first rerunning smoke. There is no manual +Docker benchmark command required. Committed notebook outputs are empty. + +Compose project is `cosmos3-perception-test`; evaluator service is +`perception-eval`. Data is mounted read-write at `/perception-test`, cache at +`/cache`, and fresh run directories at `/results`. Host paths must be absolute, +outside the checkout, with data separate from cache/results. No Docker socket, +cloud credential directory, model weights, videos, annotations, or answer-bearing +results are baked into the evaluator. The model service has **no host port**. +Only authenticated Jupyter is exposed on the selected private address. + +### Public model and serving pins + +- Model: [`nvidia/Cosmos3-Nano`](https://huggingface.co/nvidia/Cosmos3-Nano), revision + `e59a53c25979a090fa8706c9acc0c254a6e89b92`. +- Serving image: `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55`. +- Existing `CosmosReason2` generation recipe: temperature 0, seed 1, thinking + disabled, maximum 8,192 output tokens; default one API worker. + +Public downloads do not require a Hugging Face token. For an existing compatible +endpoint, export complete `COSMOS_API_BASE` (`.../v1/chat/completions`), exact +`COSMOS_MODEL`, and explicit `COSMOS_MODEL_REVISION` before setup. Optionally set +`COSMOS_SERVING_IMAGE` and environment-only `COSMOS_API_KEY`. Setup skips local +GPU serving for BYOM; do not write keys into the notebook or source. An endpoint's +model display name does not prove its checkpoint revision. + +## Dataset identity, attribution, and source cuts + +Dataset: [Perception Test](https://github.com/google-deepmind/perception_test), +by the Perception Test authors / Google DeepMind; see the authors' [dataset +README](https://github.com/google-deepmind/perception_test/blob/3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf/README.md) +and [paper](https://arxiv.org/abs/2305.13786). Their repository identifies code as +Apache 2.0 and other materials as **[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)**. +Retain attribution, license link, and describe modifications when redistributing +derived materials. The launchable protocol below is an adaptation, not an +endorsement or additional commercial clearance determination. + +`vlmeval/perception_test_local.py` pins direct official public URLs and verifies +SHA-256 before safe extraction. It retains the official README locally. The +validation annotation is `mc_question_valid.json`; source frame bounds are +`cut_frame_mapping_valid.json`. Source repository revision is +`3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf`. Questions are enumerated by numeric +`video_` and official question ID, retaining original option order, so full +stable indices are 0–19139. Video and question identity are retained separately. +The full archive may contain videos outside the MCQ selection; these are not +additional scored questions. + +Prototype frame protocol: + +- Exactly **16** uniformly sampled frames in the source-allowed interval. +- Source cut `-1` means all frames; a positive cut is the **exclusive** stop + frame. Every selected MCQ video must have a mapping; 362 of the 5,260 MCQ + videos have positive cuts. Bad cuts are rejected, not silently clamped. +- Aspect-preserving **downscale only**, maximum frame side **768**, no crop. +- Send sampled images directly; never re-encode a video and let the endpoint + choose a different sampling policy. + +This is a **launchable prototype protocol**, not reference implementation parity +or a published Nano score reproduction. The text-to-option parser is an adapter +protocol, not a claim of official letter-parser parity. + +## Controller and native APIs + +The notebook calls `launchable.py`; the same public controller is available +inside the evaluator terminal: ```bash -python launchable.py download --scope smoke +python launchable.py download --scope metadata # annotations, source cuts, README +python launchable.py download --scope smoke # also official sample videos +python launchable.py download --scope full # also official validation videos python launchable.py prepare --smoke -python launchable.py run --smoke --mode all --work-dir /results/smoke-unique -python launchable.py results --work-dir /results/smoke-unique - -# Metadata-only acquisition, without video downloads: -python launchable.py download --scope metadata - -# Full scope: authorized NTU files must already be placed under /mvbench/video/nturgbd. -python launchable.py download --scope full -python launchable.py prepare -python launchable.py run --mode all --work-dir /results/full-unique - -# Separate native inference from local scoring: -python launchable.py run --smoke --mode infer --work-dir /results/infer-unique -python launchable.py score --scope smoke --work-dir /results/infer-unique - -# Native predictions from another run, retaining original identity columns: -python launchable.py score --scope smoke --predictions /results/native-predictions.tsv \ - --score-dir /results/score-unique +python launchable.py prepare # full coverage required +python launchable.py run --smoke --mode all --work-dir /results/new-smoke +python launchable.py results --work-dir /results/new-smoke ``` -`--data-root` overrides the `MVBENCH_DATA_DIR` default (`/mvbench`). `run` defaults to **full scope**, `--mode all`, `--nframe 16`, and **`--api-nproc 1`**. Full is never silently limited. All run/score output directories must be fresh. `--dry-run` still requires valid selected annotations/media and emits configuration only; it does not prove model compatibility. Score scope is mandatory and must match all expected identities exactly; there is no arbitrary partial-full option. - -## Protocol, metrics, and provenance - -The original VideoChat2 MVBench notebook uses **16 frames and resolution 224**, and computes average accuracy as total correct / total questions. This launchable starts with **16 frames** (configurable via `--nframe`), applies reference bicubic GroupScale(224) and GroupCenterCrop(224) before native frame stacking/JPEG output, and uses the existing CosmosReason2 generation recipe: temperature 0, seed 1, thinking disabled, maximum 8,192 output tokens, presence penalty unset, ten API retries, and 600-second request timeout. It preserves native MVBench temporal bounds and sampled-image prompts, avoiding lossy video re-encoding and second-stage server video sampling. Cosmos image encoding/normalization and answer parsing still differ from VideoChat2; reference-model tensor normalization is not applied to API images. This is **not a paper-score reproduction or parity claim**. Revision/media-path/temporal-bound/frame-count/preprocessing-isolated caches prevent distinct clips or protocols from sharing stale frames. +`run --mode infer` retains predictions without scoring; `--mode all` performs +both. Every work directory and scoring directory must be fresh; failed runs and +their logs remain available for diagnosis and are not eligible for scoring. +`--dry-run` prepares real selected media and writes a plan without model access; +it is never a scored run. The controller fixes `--nframe 16` to the documented +prototype protocol. -Scoring uses native `model='exact_matching'`, never an LLM/API judge. The primary metric is **count-based question accuracy as a fraction**. Native counts, per-task metrics and task-macro accuracy are reported separately. A five-question subset cannot stand in for the full 20-task metric. Supplied predictions are checked for exact identities/row counts, duplicates, blanks, and failed transport outputs; trusted labels and identity columns come from pinned official annotations, never prediction-supplied answers/paths. Nonempty malformed model answers retain native wrong-answer semantics. Failed scoring is not a successful zero or fabricated result. +Native `run.py` configuration uses: -The local wrapper additionally prevents punctuation-only option tokens from receiving the upstream matcher's accidental substring credit and preserves literal model strings such as `nan` as wrong answers rather than missing values. These guards are recorded in scorer provenance; they are not a claim of byte-for-byte VideoChat2 evaluator parity. - -`run.json` records scope, exact selected IDs, coverage, dataset revision/manifest hash, complete emitted native config, frame count, API concurrency, generation parameters, model/deployment revision, Framework revision, source hashes, prediction location, metric summary and state transitions. `config.json`, streamed `run.log`, native predictions, `scores/summary.json`, and `/opt/evaluation-packages.txt` provide private reproducibility artifacts. Summaries retain prediction/scorer provenance. API concurrency affects scheduling and host RAM usage, not the frame count or generation protocol. Increase it only after measuring host resources. +```json +{"data":{"PerceptionTest":{"class":"PerceptionTestLocal","dataset":"PerceptionTest","data_root":"/perception-test","nframe":16,"smoke":true}}} +``` -### Observed validation (2026-10-29) +The controller composes that with `cosmos_eval/models/cosmos.json`. Native +predictions are `cosmos_PerceptionTest.tsv` (or XLSX/JSON) in the native dated run +directory. Helper API in `vlmeval.perception_test_local`: -Runtime candidate `b0861ce448a648b2eb59c41a9358980bbb5df31c` was built and exercised on native Linux x86-64: +- `fetch_dataset(root, scope='smoke')`: metadata / smoke / full public download. +- `prepare_dataset(root, smoke=False, require_media=True)`: trusted annotation, + cut and scope manifest, stable selection, coverage, separate media hashes. +- `score_predictions(predictions, root, output_dir, smoke=False, nframe=16)`: + reconstruct trusted labels/identities and score locally into a fresh directory. -- All **28 behavioral regressions passed inside the newly built evaluator image**. -- JupyterLab **Run All Cells** executed all six code cells against the pinned Nano endpoint on one RTX PRO 6000 Blackwell Server Edition. The fixed five-question subset produced five valid predictions with matching identities and native local scores; correctness results remain private and are not a full benchmark score. -- The saved real predictions were rescored in a separate CPU-only container with **`--network none`**, with the same correctness counts. -- SHA-verified current public archives were downloaded and safely extracted. Actual full preparation found media for **3,800 of 4,000 rows** and rejected the run before inference because the **200 Fine-grained Pose NTU clips** were missing. No rows were dropped and no partial full score was produced. +## Score existing predictions without videos or a model -This reused an existing warm GPU endpoint with a manually staged evaluator; it was **not a cold one-click launch**. The complete 20-task run remains blocked on authorized NTU acquisition. No published-score reproduction, reference-result parity, commercial/distribution approval, or customer-data agent implementation is claimed. No benchmark labels/media, credentials, executed notebook outputs, or private result artifacts belong in the PR. +Start only authenticated Jupyter with `bash setup.sh --score-only` (add `--brev` +for private mesh access). Set `SCORE_ONLY = True`, choose explicit `SCORE_SCOPE`, +and set exactly one of `EXISTING_WORK` or `PREDICTIONS_FILE` in the notebook. +The notebook downloads **metadata only** and prepares with +`require_media=False`; video files, model construction, endpoint requests and GPU +are not required. Set `DOWNLOAD_PUBLIC = False` to rescore completely offline +with already staged pinned metadata. Equivalent evaluator-terminal commands: -## Stop, restart, and maintain +```bash +python launchable.py score --scope smoke --work-dir /results/inferred-run \ + --score-dir /results/fresh-rescore +python launchable.py score --scope full --predictions /results/predictions.tsv \ + --score-dir /results/fresh-full-score +``` -From the launchable directory on the **host**: +Annotation/cut/scope fingerprints must match inference even after videos are +removed; media hashes are separate, not part of that canonical identity. Exact +selected IDs are required: missing, duplicate, unexpected, blank, or +transport-failure rows reject the run before scoring. A malformed or ambiguous +**nonempty** model answer is wrong and counted invalid, never silently dropped. +The literal model text `nan` is not treated as a missing value. Prediction-supplied +labels never override trusted annotations. Top1 is `correct / all selected +questions`, with per-area, per-reasoning, and per-tag diagnostics; no network judge, +dummy answers, random answers, or partial-full-score fallback. + +## Results and lifecycle + +The notebook shows one concise human result and links to the score summary, +run provenance and log. `run.json` records source `perception-test-local`, scope, +annotation and media fingerprints, source hashes, model/source revisions, +generation parameters, frame protocol, and UTC start/end plus measured elapsed +seconds. Score artifacts include `summary.json` and `controller.json` with local +scoring timing/provenance. No cost or peak-memory estimate is invented. ```bash -docker compose --profile local ps +# Host terminal: preserve mounted data/results while stopping containers. docker compose --profile local stop -# Restart only Jupyter to score retained predictions: -bash setup.sh --score-only ``` -Stopping containers does not stop Brev VM billing. Stop the instance separately after preserving authorized results; do not delete state/volumes merely to stop a run. Clear notebook outputs before sharing. - -Maintainers should run `python -m unittest discover -s launchable -p 'test_*.py'` after integration, then exercise notebook smoke, offline CPU rescore, missing-NTU full-coverage failure, and an authorized complete full run when its prerequisites are available. Behavioral fixtures use synthetic artifacts and do not establish model/hardware results. The committed notebook is unexecuted. Native Linux x86-64 is the supported runtime; local ARM64 Docker emulation is not notebook/hardware validation. +Stopping containers does not stop underlying Brev VM billing; stop the VM +separately after retaining needed results. Do not commit credentials, videos, +labels, prediction tables, or answer-bearing artifacts. + +## Observed validation · 2026-10-30 + +- On an existing Brev instance with one RTX PRO 6000 Blackwell Server Edition + (96 GB), the actual Jupyter **Run All Cells** completed all four code cells. + Nano returned five valid choices for the exact smoke IDs; local top1 was + **4/5 (80%)**, with zero malformed answers. +- First local score was produced about **44 seconds** after Run All, using an + already-running model and staged smoke inputs. Controller inference took + 20.53 seconds and local scoring 0.74 seconds. This is **not a cold launch**. +- A separate CPU container with **network disabled and no videos or GPU** + rescored the actual predictions identically, including diagnostic counts and + annotation/prediction identities. +- All **43 native behavioral checks** passed in the Linux CPU evaluator. + +These observations cover five questions from one video, not full validation, +the held-out test leaderboard, reference-score parity, BYOM, or a fresh-VM +launch. Driver was 595.112.07; peak memory and cost were not measured. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml index 59240d591..2f90c9820 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: cosmos3-mvbench +name: cosmos3-perception-test services: cosmos-endpoint: profiles: [local] @@ -16,7 +16,6 @@ services: capabilities: [gpu] ipc: host environment: - HF_TOKEN: ${HF_TOKEN:-} HF_HOME: /cache/huggingface volumes: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache @@ -26,8 +25,6 @@ services: - --revision - e59a53c25979a090fa8706c9acc0c254a6e89b92 - --async-scheduling - - --allowed-local-media-path - - / - --port - '8080' - --max-model-len @@ -49,7 +46,7 @@ services: timeout: 10s retries: 120 start_period: 60s - mvbench-eval: + perception-eval: platform: linux/amd64 build: context: .. @@ -60,7 +57,6 @@ services: ports: - '${JUPYTER_BIND_ADDRESS:-127.0.0.1}:8888:8888' environment: - HF_TOKEN: ${HF_TOKEN:-} COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} COSMOS_API_BASE: ${COSMOS_API_BASE:-http://cosmos-endpoint:8080/v1/chat/completions} COSMOS_MODEL: ${COSMOS_MODEL:-nvidia/Cosmos3-Nano} @@ -69,7 +65,7 @@ services: COSMOS_API_KEY: ${COSMOS_API_KEY:-EMPTY} JUPYTER_TOKEN: ${JUPYTER_TOKEN:?Run bash setup.sh to generate a Jupyter token} LMUData: /cache/lmudata - MVBENCH_DATA_DIR: /mvbench + PERCEPTION_TEST_DATA_DIR: /perception-test EVAL_SCORE_ONLY: ${EVAL_SCORE_ONLY:-0} AWS_CONFIG_FILE: /dev/null AWS_SHARED_CREDENTIALS_FILE: /dev/null @@ -77,7 +73,7 @@ services: volumes: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/results:/results - - ${MVBENCH_DATA_DIR:?Set an external writable MVBench data directory}:/mvbench + - ${PERCEPTION_TEST_DATA_DIR:?Set an external writable Perception Test data directory}:/perception-test healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8888/login', timeout=5)"] interval: 15s diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py index e0853ab61..0ceecb659 100755 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Notebook/CLI orchestration for pinned MVBench and native VLMEvalKit scoring.""" +"""Notebook/CLI orchestration for public Perception Test validation MCQ.""" from __future__ import annotations import argparse +from datetime import datetime, timezone import hashlib import json import os @@ -18,6 +19,16 @@ ROOT = Path(__file__).resolve().parents[1] MODEL_REVISION = "e59a53c25979a090fa8706c9acc0c254a6e89b92" SERVING_IMAGE = "vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55" +SOURCE = "perception-test-local" +PROTOCOL = {"nframe": 16, "max_frame_side": 768, + "resize": "aspect-preserving downscale only; no crop", + "sampling": "uniform within source cut interval; positive cut is exclusive", + "input": "sampled images, never re-encoded video", + "claim": "launchable prototype, not reference-score parity"} + + +def utc_now(): + return datetime.now(timezone.utc).isoformat() def endpoint(wait=0): @@ -55,11 +66,11 @@ def endpoint(wait=0): time.sleep(min(5, max(0, deadline - time.monotonic()))) -def prepare(data_root, smoke=False): - from vlmeval.mvbench_local import prepare_dataset +def prepare(data_root, smoke=False, require_media=True): + from vlmeval.perception_test_local import prepare_dataset - prepared = prepare_dataset(data_root, smoke=smoke) - print(json.dumps(prepared["coverage"], indent=2), flush=True) + prepared = prepare_dataset(data_root, smoke=smoke, require_media=require_media) + print("PerceptionTest coverage: " + json.dumps(prepared["coverage"]), flush=True) return prepared @@ -86,50 +97,50 @@ def run_command(command, log_path=None): def prediction_files(work): - return sorted(path for path in work.rglob("cosmos_MVBench.*") + return sorted(path for path in work.rglob("cosmos_PerceptionTest.*") if path.parent.name.startswith("T") and path.suffix in {".tsv", ".xlsx", ".json"}) def verify_predictions(path, selected_indices): - """Check complete selected identities and transport failures before native scoring.""" + """Reject incomplete identities/transport failures, not wrong nonempty answers.""" import pandas as pd path = Path(path) if path.is_dir(): files = prediction_files(path) if len(files) != 1: - raise RuntimeError(f"MVBench: expected one prediction table, found {len(files)}") + raise RuntimeError(f"PerceptionTest: expected one prediction table, found {len(files)}") path = files[0] if path.suffix == ".tsv": frame = pd.read_csv(path, sep="\t", keep_default_na=False, dtype={"index": str, "prediction": str}) elif path.suffix == ".json": - frame = pd.read_json(path, dtype={"index": str}) + frame = pd.DataFrame(json.loads(path.read_text())) elif path.suffix == ".xlsx": frame = pd.read_excel(path, keep_default_na=False, dtype={"index": str, "prediction": str}) else: raise ValueError("Predictions must be a native TSV, XLSX, or JSON table") if not {"index", "prediction"}.issubset(frame.columns) or frame.empty: - raise RuntimeError("MVBench: empty or invalid prediction output") + raise RuntimeError("PerceptionTest: empty or invalid prediction output") expected = [str(index) for index in selected_indices] if len(frame) != len(expected): - raise RuntimeError(f"MVBench: expected {len(expected)} predictions, found {len(frame)}") + raise RuntimeError(f"PerceptionTest: expected {len(expected)} predictions, found {len(frame)}") indices = frame["index"].astype(str) if indices.duplicated().any() or set(indices) != set(expected): - raise RuntimeError("MVBench: duplicate, missing, or unexpected prediction identities") + raise RuntimeError("PerceptionTest: duplicate, missing, or unexpected prediction identities") text = frame["prediction"].astype(str).str.strip().str.lower() - failed = (frame["prediction"].isna() - | text.eq("") + failed = (frame["prediction"].isna() | text.eq("") | text.str.contains("failed to obtain answer", regex=False) | text.str.contains("video_load_failed", regex=False)) if failed.any(): - raise RuntimeError(f"MVBench: {int(failed.sum())} failed/empty predictions; inspect the run log") - print(f"MVBench: {len(frame)} predictions verified", flush=True) + raise RuntimeError(f"PerceptionTest: {int(failed.sum())} failed/empty predictions; inspect the run log") + print(f"PerceptionTest: {len(frame)} predictions verified", flush=True) return path def source_provenance(): - files = ("run.py", "vlmeval/mvbench_local.py", "vlmeval/dataset/video_dataset_config.py", - "vlmeval/dataset/mvbench.py", "vlmeval/api/cosmos_reason.py", "launchable/launchable.py") + files = ("run.py", "vlmeval/perception_test_local.py", "vlmeval/dataset/perception_test.py", + "vlmeval/dataset/video_dataset_config.py", "vlmeval/api/cosmos_reason.py", + "launchable/launchable.py") return {"framework_revision": os.environ.get("COSMOS_COMMIT", "unknown"), "source_sha256": {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in files if (ROOT / name).is_file()}} @@ -139,45 +150,51 @@ def _write_state(work, state): (work / "run.json").write_text(json.dumps(state, indent=2, default=str) + "\n") +def _protocol(nframe): + if nframe != 16: + raise ValueError("The launchable prototype protocol requires --nframe 16") + + def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): - if api_nproc < 1 or nframe < 1: - raise ValueError("--api-nproc and --nframe must be positive") + work, data_root = Path(work), Path(data_root) + if api_nproc < 1: + raise ValueError("--api-nproc must be positive") + _protocol(nframe) if mode not in {"infer", "all"}: raise ValueError("--mode must be infer or all") if work.exists(): raise ValueError(f"Use a fresh --work-dir (already exists: {work}) to avoid stale results") - # All media and selected identities validate before contacting a paid endpoint. - prepared = prepare(data_root, smoke) - if not dry_run: - endpoint(wait=1800) + # Validate all selected media before contacting an endpoint or creating run artifacts. + started, clock = utc_now(), time.monotonic() + prepared = prepare(data_root, smoke, require_media=True) recipe = json.loads((ROOT / "cosmos_eval/models/cosmos.json").read_text()) model = {"class": recipe["class"], **recipe["defaults"]} model.update(api_base=os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions"), model=os.environ.get("COSMOS_MODEL", "nvidia/Cosmos3-Nano")) - config = {"model": {"cosmos": model}, "data": {"MVBench": { - "class": "MVBenchLocal", "dataset": "MVBench", "data_root": str(data_root), + config = {"model": {"cosmos": model}, "data": {"PerceptionTest": { + "class": "PerceptionTestLocal", "dataset": "PerceptionTest", "data_root": str(data_root), "nframe": nframe, "smoke": smoke}}} work.mkdir(parents=True, mode=0o700) config_path = work / "config.json" - config_path.write_text(json.dumps(config, indent=2) + "\n") - state = {"format_version": 3, "benchmark": "MVBench", "source": "mvbench-local", + local = model["api_base"] == "http://cosmos-endpoint:8080/v1/chat/completions" + state = {"format_version": 4, "benchmark": "PerceptionTest", "source": SOURCE, "status": "planned" if dry_run else "running", "mode": mode, "scope": "smoke" if smoke else "full", "smoke": smoke, "dry_run": dry_run, - "dataset_revision": prepared["dataset_revision"], - "manifest_sha256": prepared["manifest_sha256"], "coverage": prepared["coverage"], - "selected_indices": prepared["selected_indices"], "data_root": str(data_root), - "nframe": nframe, "api_nproc": api_nproc, "config": config, + **{key: prepared[key] for key in ( + "dataset_revision", "manifest_sha256", "annotation_manifest_sha256", "coverage", + "selected_indices", "media_sha256", "media_manifest_sha256")}, + "data_root": str(data_root), "nframe": nframe, "frame_protocol": PROTOCOL, + "api_nproc": api_nproc, "config": config, "generation_parameters": {key: value for key, value in model.items() if key not in {"class", "api_base", "model"}}, "model": model["model"], - "model_revision": os.environ.get( - "COSMOS_MODEL_REVISION", - MODEL_REVISION if model["api_base"] == "http://cosmos-endpoint:8080/v1/chat/completions" - and model["model"] == "nvidia/Cosmos3-Nano" else "unknown"), - "serving_image": (os.environ.get("VLLM_IMAGE", SERVING_IMAGE) - if model["api_base"] == "http://cosmos-endpoint:8080/v1/chat/completions" + "model_revision": os.environ.get("COSMOS_MODEL_REVISION", + MODEL_REVISION if local and model["model"] == "nvidia/Cosmos3-Nano" else "unknown"), + "serving_image": (os.environ.get("VLLM_IMAGE", SERVING_IMAGE) if local else os.environ.get("COSMOS_SERVING_IMAGE", "unknown")), - "provenance": source_provenance(), "predictions": None, "scores": None} + "provenance": source_provenance(), "predictions": None, "scores": None, + "inference_timing": {"started_utc": started}} + config_path.write_text(json.dumps(config, indent=2) + "\n") _write_state(work, state) command = [sys.executable, "-u", str(ROOT / "run.py"), "--config", str(config_path), "--mode", "infer", "--work-dir", str(work / "native"), @@ -186,82 +203,119 @@ def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): print(" ".join(command), flush=True) return state try: + endpoint(wait=1800) + model["model"] = os.environ.get("COSMOS_MODEL", model["model"]) + state["model"] = model["model"] + config_path.write_text(json.dumps(config, indent=2) + "\n") + _write_state(work, state) run_command(command, log_path=work / "run.log") path = verify_predictions(work / "native", prepared["selected_indices"]) state.update(status="inferred", predictions=str(path.relative_to(work))) + state["inference_timing"].update(ended_utc=utc_now(), elapsed_seconds=time.monotonic() - clock) _write_state(work, state) if mode == "all": score(data_root, state["scope"], work=work) else: results(work) - except BaseException: + except BaseException as exc: state = json.loads((work / "run.json").read_text()) - state["status"] = "failed" + state.update(status="failed", error={"type": type(exc).__name__, "message": str(exc)}) + if "ended_utc" not in state["inference_timing"]: + state["inference_timing"].update(ended_utc=utc_now(), elapsed_seconds=time.monotonic() - clock) _write_state(work, state) raise return json.loads((work / "run.json").read_text()) def score(data_root, scope, work=None, predictions=None, score_dir=None, nframe=16): - from vlmeval.mvbench_local import score_predictions + from vlmeval.perception_test_local import score_predictions if scope not in {"smoke", "full"}: raise ValueError("Scoring requires explicit smoke or full scope") if (work is None) == (predictions is None): raise ValueError("Supply exactly one of --work-dir or --predictions") - if nframe < 1: - raise ValueError("--nframe must be positive") + _protocol(nframe) state = None if work is not None: + work = Path(work) state = json.loads((work / "run.json").read_text()) - if (state.get("source") != "mvbench-local" or state.get("dry_run") - or state.get("status") not in {"inferred", "complete"}): - raise ValueError("Only successful MVBench inference runs can be scored") + if (state.get("source") != SOURCE or state.get("benchmark") != "PerceptionTest" + or state.get("dry_run") or state.get("status") not in {"inferred", "complete"}): + raise ValueError("Only successful PerceptionTest inference runs can be scored") if scope != state["scope"]: raise ValueError("Scoring scope differs from inference scope") nframe = state["nframe"] - predictions = work / state["predictions"] + _protocol(nframe) + predictions = (work / state["predictions"]).resolve() + if not predictions.is_relative_to(work.resolve()): + raise ValueError("Prediction artifact must remain within its inference work directory") score_dir = Path(score_dir) if score_dir is not None else work / "scores" elif score_dir is None: raise ValueError("--score-dir is required for existing native prediction files") score_dir = Path(score_dir) if score_dir.exists(): raise ValueError("Scoring output already exists; choose a fresh --score-dir") - prepared = prepare(data_root, smoke=scope == "smoke") + # Annotation/cut/scope fingerprints do not depend on video files or their hashes. + prepared = prepare(data_root, smoke=scope == "smoke", require_media=False) if state is not None and any(state[key] != prepared[key] for key in - ("dataset_revision", "manifest_sha256", "selected_indices")): - raise ValueError("Dataset identities or annotations changed since inference") + ("dataset_revision", "annotation_manifest_sha256", + "manifest_sha256", "selected_indices")): + raise ValueError("Dataset identities, annotations, or cuts changed since inference") predictions = verify_predictions(predictions, prepared["selected_indices"]) + started, clock = utc_now(), time.monotonic() if state is not None: - state["status"] = "scoring" + state.update(status="scoring", scoring_timing={"started_utc": started}) _write_state(work, state) try: summary = score_predictions(predictions, data_root, score_dir, smoke=scope == "smoke", nframe=nframe) + timing = {"started_utc": started, "ended_utc": utc_now(), + "elapsed_seconds": time.monotonic() - clock} + # Keep controller/model provenance beside the native trusted-label score artifact. + (score_dir / "controller.json").write_text(json.dumps({ + "source": SOURCE, "scope": scope, "timing": timing, + "frame_protocol": PROTOCOL, "provenance": source_provenance(), + "model": state.get("model", "unknown") if state is not None else "unknown", + "model_revision": state.get("model_revision", "unknown") if state is not None else "unknown", + "serving_image": state.get("serving_image", "unknown") if state is not None else "unknown", + "generation_parameters": state.get("generation_parameters") if state is not None else None, + "inference_run": str(work / "run.json") if work is not None else None, + }, indent=2) + "\n") if state is not None: - state.update(status="complete", scores={"summary": str(score_dir / "summary.json"), - "primary_metric": summary["primary_metric"], - "coverage": summary["coverage"]}) + state.update(status="complete", scoring_timing=timing, + scores={"summary": str(score_dir / "summary.json"), + "primary_metric": summary["primary_metric"], "coverage": summary["coverage"]}) _write_state(work, state) results(work) else: - print(json.dumps(summary, indent=2, default=str), flush=True) + show_summary(summary, scope) + print(f"Score artifact: {score_dir / 'summary.json'}", flush=True) return summary - except BaseException: + except BaseException as exc: if state is not None: - state["status"] = "failed" + state.update(status="failed", error={"type": type(exc).__name__, "message": str(exc)}) + state["scoring_timing"].update(ended_utc=utc_now(), elapsed_seconds=time.monotonic() - clock) _write_state(work, state) raise +def show_summary(summary, scope): + print(f"PerceptionTest validation MCQ — {scope}: {summary['status']}", flush=True) + print("Top1/counts: " + json.dumps(summary["primary_metric"]), flush=True) + print("Coverage: " + json.dumps(summary["coverage"]), flush=True) + print("Prototype 16-frame/aspect-preserving-768 protocol; no reference-score parity claim.", flush=True) + + def results(work): + work = Path(work) state = json.loads((work / "run.json").read_text()) - print(f"MVBench: {state['status']} ({state['scope']})", flush=True) - print(json.dumps({"coverage": state["coverage"], "scores": state.get("scores"), - "dataset_revision": state["dataset_revision"], - "model_revision": state["model_revision"], "provenance": state["provenance"]}, - indent=2), flush=True) - print("Smoke is not a full benchmark. No published-score or paper-parity claim.", flush=True) + if state.get("scores"): + show_summary({"status": state["status"], **state["scores"]}, state["scope"]) + print(f"Score artifact: {state['scores']['summary']}", flush=True) + else: + print(f"PerceptionTest: {state['status']} ({state['scope']}); coverage: " + + json.dumps(state["coverage"]), flush=True) + print(f"Run provenance: {work / 'run.json'}", flush=True) return state @@ -271,16 +325,17 @@ def main(): sub = parser.add_subparsers(dest="command", required=True) check = sub.add_parser("check", help="Wait for /v1/models and verify the requested model") check.add_argument("--wait", type=int, default=1800) - download = sub.add_parser("download", help="Download approved pinned public inputs, never removed NTU media") + download = sub.add_parser("download", help="Download pinned public validation inputs") download.add_argument("--scope", choices=["metadata", "smoke", "full"], default="smoke") - prep = sub.add_parser("prepare", help="Validate pinned annotations and complete selected media coverage") - runner = sub.add_parser("run", help="Run native inference and optionally native exact-match scoring") - scorer = sub.add_parser("score", help="Score existing predictions without a GPU or model endpoint") + prep = sub.add_parser("prepare", help="Validate selected annotations/cuts and media coverage") + runner = sub.add_parser("run", help="Run native inference and optionally local top1 scoring") + scorer = sub.add_parser("score", help="Score predictions without model, endpoint, or video files") for child in (download, prep, runner, scorer): child.add_argument("--data-root", type=Path, - default=Path(os.environ.get("MVBENCH_DATA_DIR", "/mvbench"))) + default=Path(os.environ.get("PERCEPTION_TEST_DATA_DIR", "/perception-test"))) for child in (prep, runner): - child.add_argument("--smoke", action="store_true", help="Fixed first five Egocentric Navigation IDs, not full MVBench") + child.add_argument("--smoke", action="store_true", help="Five fixed video_8241 validation questions") + prep.add_argument("--metadata-only", action="store_true", help="Prepare without video files (score-only)") runner.add_argument("--mode", choices=["infer", "all"], default="all") runner.add_argument("--api-nproc", type=int, default=1) runner.add_argument("--nframe", type=int, default=16) @@ -292,16 +347,16 @@ def main(): scorer.add_argument("--scope", choices=["smoke", "full"], required=True) scorer.add_argument("--score-dir", type=Path) scorer.add_argument("--nframe", type=int, default=16) - report = sub.add_parser("results", help="Show metric summaries, coverage, and provenance") + report = sub.add_parser("results", help="Show top1/counts, coverage, and artifact locations") report.add_argument("--work-dir", type=Path, required=True) args = parser.parse_args() if args.command == "check": endpoint(args.wait) elif args.command == "download": - from vlmeval.mvbench_local import fetch_dataset + from vlmeval.perception_test_local import fetch_dataset fetch_dataset(args.data_root.resolve(), scope=args.scope) elif args.command == "prepare": - prepare(args.data_root.resolve(), args.smoke) + prepare(args.data_root.resolve(), args.smoke, require_media=not args.metadata_only) elif args.command == "run": run(args.work_dir.resolve(), args.mode, args.smoke, args.dry_run, args.data_root.resolve(), args.api_nproc, args.nframe) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh index 9bc6a3c3d..3c9adc2b6 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -11,9 +11,9 @@ docker info >/dev/null export COSMOS_COMMIT COSMOS_COMMIT=$(git rev-parse HEAD) : "${LAUNCHABLE_STATE_DIR:?Set an absolute state directory outside the Framework checkout}" -export MVBENCH_DATA_DIR="${MVBENCH_DATA_DIR:-$LAUNCHABLE_STATE_DIR/mvbench}" +export PERCEPTION_TEST_DATA_DIR="${PERCEPTION_TEST_DATA_DIR:-$LAUNCHABLE_STATE_DIR/perception-test}" export LAUNCHABLE_STATE_DIR -python3 - "$LAUNCHABLE_STATE_DIR" "$MVBENCH_DATA_DIR" "$(git rev-parse --show-toplevel)" <<'PY' +python3 - "$LAUNCHABLE_STATE_DIR" "$PERCEPTION_TEST_DATA_DIR" "$(git rev-parse --show-toplevel)" <<'PY' import os from pathlib import Path import sys @@ -22,10 +22,10 @@ checkout = Path(sys.argv[3]).resolve() state, data = (Path(value) for value in sys.argv[1:3]) for path in (state, data): if not path.is_absolute() or path.resolve().is_relative_to(checkout): - raise SystemExit("State and MVBench data must use absolute paths outside the Framework checkout") + raise SystemExit("State and Perception Test data must use absolute paths outside the Framework checkout") if any(data.resolve().is_relative_to((state / name).resolve()) or (state / name).resolve().is_relative_to(data.resolve()) for name in ("cache", "results")): - raise SystemExit("MVBench data must be separate from writable cache/results mounts") + raise SystemExit("Perception Test data must be separate from writable cache/results mounts") for path in (state / "cache", state / "results", data): path.mkdir(parents=True, exist_ok=True) if not os.access(path, os.W_OK): @@ -77,7 +77,7 @@ PY ) chmod 600 .env docker compose config --quiet -docker compose build mvbench-eval +docker compose build perception-eval if [[ "$EVAL_SCORE_ONLY" == "0" && -z "${COSMOS_API_BASE:-}" ]]; then nvidia-smi export COSMOS_MODEL_REVISION="${COSMOS_MODEL_REVISION:-e59a53c25979a090fa8706c9acc0c254a6e89b92}" @@ -86,10 +86,10 @@ if [[ "$EVAL_SCORE_ONLY" == "0" && -z "${COSMOS_API_BASE:-}" ]]; then else # Existing-prediction scoring and BYOM never start a local model/GPU. export COSMOS_MODEL_REVISION="${COSMOS_MODEL_REVISION:-unknown}" - docker compose up -d mvbench-eval + docker compose up -d perception-eval fi printf 'Jupyter is starting on %s:8888. Open its authenticated secure link or SSH tunnel.\n' "${JUPYTER_BIND_ADDRESS:-127.0.0.1}" printf '%s\n' \ - 'Retrieve the notebook login token with: docker compose exec mvbench-eval jupyter server list' \ - 'Open Evaluate_Cosmos3.ipynb to download public MVBench data, prepare, infer, and score.' \ - 'Full evaluation also requires manually acquired authorized NTU RGB+D media; see the notebook.' + 'Retrieve the notebook login token with: docker compose exec perception-eval jupyter server list' \ + 'Open Evaluate_Cosmos3.ipynb to download public Perception Test validation data, prepare, infer, and score.' \ + 'Default smoke: five video_8241 questions; full 19,140-question validation is explicit opt-in.' diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py index 4793b5fa3..1232fbaae 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Controller behavior regressions; synthetic artifacts, no GPU or network.""" -import contextlib -import io +"""Controller boundary regressions; temporary artifacts, no GPU or network.""" import json from pathlib import Path import subprocess @@ -10,42 +8,65 @@ import tempfile import types import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch import launchable -def prepared(indices=(3400, 3401, 3402, 3403, 3404)): - return {"dataset_revision": "pinned-test-revision", "manifest_sha256": "trusted-hash", +def prepared(indices=(13404, 13405, 13406, 13407, 13408), media=True): + return {"dataset_revision": "pinned-test-revision", "manifest_sha256": "scope-and-cuts", + "annotation_manifest_sha256": "trusted-annotation-and-cuts", + "media_sha256": {"video_8241": "video-hash"} if media else {}, + "media_manifest_sha256": "media-hash" if media else None, "selected_indices": list(indices), "coverage": { - "expected_rows": 4000, "selected_rows": len(indices), - "available_rows": len(indices), "expected_tasks": 20, - "selected_tasks": 1, "subset": len(indices) != 4000}} + "expected_rows": 19140, "selected_rows": len(indices), + "available_rows": len(indices) if media else 0, + "expected_videos": 5260, "selected_videos": 1, "subset": True}} -def table(work, indices, values=None, stamp="T20261029"): +def table(work, indices, values=None, stamp="T20261030"): output = work / "cosmos" / stamp output.mkdir(parents=True) - path = output / "cosmos_MVBench.tsv" + path = output / "cosmos_PerceptionTest.tsv" values = values if values is not None else ["A"] * len(indices) path.write_text("index\tprediction\n" + "".join( f"{index}\t{value}\n" for index, value in zip(indices, values))) return path +def inference_state(root, selection=None, **updates): + state = {**(selection or prepared()), "source": launchable.SOURCE, + "benchmark": "PerceptionTest", "status": "inferred", "scope": "smoke", + "nframe": 16, "predictions": "native.tsv", "dry_run": False} + state.update(updates) + (root / "run.json").write_text(json.dumps(state)) + return state + + +def scorer_module(effect=None): + module = types.ModuleType("vlmeval.perception_test_local") + module.score_predictions = Mock(side_effect=effect) + return module + + class ArtifactTests(unittest.TestCase): def test_existing_output_rejected_before_network_or_overwrite(self): - with tempfile.TemporaryDirectory() as temp, patch.object(launchable, "prepare") as prep: + with tempfile.TemporaryDirectory() as temp, patch.object(launchable, "prepare") as prep, \ + patch.object(launchable, "endpoint") as endpoint: + marker = Path(temp) / "existing" + marker.write_text("keep") with self.assertRaisesRegex(ValueError, "fresh --work-dir"): launchable.run(Path(temp), "infer", True, False, Path(temp) / "data") prep.assert_not_called() + endpoint.assert_not_called() + self.assertEqual(marker.read_text(), "keep") - def test_nonpositive_parameters_rejected_before_output(self): - for workers, frames in ((0, 16), (-1, 16), (1, 0), (1, -1)): + def test_invalid_workers_and_frame_protocol_rejected_before_output(self): + for workers, frames in ((0, 16), (-1, 16), (1, 0), (1, 8), (1, 32)): with self.subTest(workers=workers, frames=frames), tempfile.TemporaryDirectory() as temp: output = Path(temp) / "fresh" with patch.object(launchable, "prepare") as prep: - with self.assertRaisesRegex(ValueError, "positive"): + with self.assertRaises(ValueError): launchable.run(output, "infer", False, False, Path(temp) / "missing", api_nproc=workers, nframe=frames) prep.assert_not_called() @@ -54,112 +75,203 @@ def test_nonpositive_parameters_rejected_before_output(self): def test_coverage_failure_precedes_endpoint_and_output(self): with tempfile.TemporaryDirectory() as temp: work = Path(temp) / "fresh" - with patch.object(launchable, "prepare", side_effect=ValueError("missing NTU")), \ + with patch.object(launchable, "prepare", side_effect=ValueError("missing video_8241")), \ patch.object(launchable, "endpoint") as endpoint: - with self.assertRaisesRegex(ValueError, "missing NTU"): - launchable.run(work, "all", False, False, Path(temp) / "data") + with self.assertRaisesRegex(ValueError, "missing video_8241"): + launchable.run(work, "all", True, False, Path(temp) / "data") endpoint.assert_not_called() self.assertFalse(work.exists()) - def test_predictions_reject_empty_and_failure_markers(self): + def test_predictions_reject_empty_and_transport_failure_markers(self): for value in (" ", "VIDEO_LOAD_FAILED", "prefix VIDEO_LOAD_FAILED", "Failed to obtain answer via API."): with self.subTest(value=value), tempfile.TemporaryDirectory() as temp: - path = table(Path(temp), [3400], [value]) + path = table(Path(temp), [13404], [value]) with self.assertRaisesRegex(RuntimeError, "failed/empty"): - launchable.verify_predictions(path, [3400]) + launchable.verify_predictions(path, [13404]) + + def test_nonempty_invalid_and_literal_nan_reach_local_scorer(self): + for value in ("nan", "ambiguous A or B", "unparseable text"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as temp: + path = table(Path(temp), [13404], [value]) + self.assertEqual(launchable.verify_predictions(path, [13404]), path) + + def test_json_null_is_missing_but_literal_nan_is_not(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "predictions.json" + path.write_text(json.dumps([{"index": 13404, "prediction": None}])) + with self.assertRaisesRegex(RuntimeError, "failed/empty"): + launchable.verify_predictions(path, [13404]) + path.write_text(json.dumps([{"index": 13404, "prediction": "nan"}])) + self.assertEqual(launchable.verify_predictions(path, [13404]), path) def test_selected_identities_exact_even_when_row_count_matches(self): - for indices in ([3400, 3400], [3400, 9999], [3400], [3400, 3401, 3402]): + for indices in ([13404, 13404], [13404, 9999], [13404], [13404, 13405, 13406]): with self.subTest(indices=indices), tempfile.TemporaryDirectory() as temp: path = table(Path(temp), indices) with self.assertRaisesRegex(RuntimeError, "identities|expected 2 predictions"): - launchable.verify_predictions(path, [3400, 3401]) - + launchable.verify_predictions(path, [13404, 13405]) def test_multiple_dated_prediction_tables_are_ambiguous(self): with tempfile.TemporaryDirectory() as temp: work = Path(temp) - table(work, [3400], stamp="T20261028") - table(work, [3400], stamp="T20261029") + table(work, [13404], stamp="T20261029") + table(work, [13404], stamp="T20261030") with self.assertRaisesRegex(RuntimeError, "expected one prediction table, found 2"): - launchable.verify_predictions(work, [3400]) + launchable.verify_predictions(work, [13404]) - def test_inference_failure_marks_state_and_keeps_artifacts(self): + def test_inference_execution_failure_retains_state_config_and_log(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) work = root / "fresh" def fail(command, log_path=None): - Path(log_path).write_text("private diagnostic\n") + Path(log_path).write_text("diagnostic retained\n") raise subprocess.CalledProcessError(3, command) with patch.object(launchable, "prepare", return_value=prepared()), \ patch.object(launchable, "endpoint"), \ patch.object(launchable, "run_command", side_effect=fail): with self.assertRaises(subprocess.CalledProcessError): launchable.run(work, "all", True, False, root / "data") - self.assertEqual(json.loads((work / "run.json").read_text())["status"], "failed") - self.assertEqual((work / "run.log").read_text(), "private diagnostic\n") + state = json.loads((work / "run.json").read_text()) + self.assertEqual(state["status"], "failed") + self.assertEqual(state["error"]["type"], "CalledProcessError") + self.assertIn("ended_utc", state["inference_timing"]) + self.assertGreaterEqual(state["inference_timing"]["elapsed_seconds"], 0) + self.assertEqual((work / "run.log").read_text(), "diagnostic retained\n") self.assertTrue((work / "config.json").is_file()) + def test_endpoint_failure_is_retained_and_never_executes_inference(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) / "fresh" + with patch.object(launchable, "prepare", return_value=prepared()), \ + patch.object(launchable, "endpoint", side_effect=RuntimeError("endpoint unavailable")), \ + patch.object(launchable, "run_command") as child: + with self.assertRaisesRegex(RuntimeError, "endpoint unavailable"): + launchable.run(work, "all", True, False, Path(temp) / "data") + child.assert_not_called() + self.assertEqual(json.loads((work / "run.json").read_text())["status"], "failed") - def test_score_output_and_scope_guards(self): + def test_incomplete_inference_predictions_never_enter_scorer(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) - native = types.ModuleType("vlmeval.mvbench_local") - native.score_predictions = unittest.mock.Mock() - with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}): + work = root / "fresh" + def incomplete(command, log_path=None): + table(work / "native", prepared()["selected_indices"], ["A"] * 4 + ["VIDEO_LOAD_FAILED"]) + with patch.object(launchable, "prepare", return_value=prepared()), \ + patch.object(launchable, "endpoint"), \ + patch.object(launchable, "run_command", side_effect=incomplete), \ + patch.object(launchable, "score") as score: + with self.assertRaisesRegex(RuntimeError, "failed/empty"): + launchable.run(work, "all", True, False, root / "data") + score.assert_not_called() + self.assertEqual(json.loads((work / "run.json").read_text())["status"], "failed") + self.assertFalse((work / "scores").exists()) + + def test_score_output_scope_and_source_guards(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + native = scorer_module() + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}): with self.assertRaisesRegex(ValueError, "explicit"): launchable.score(root, "subset", predictions=root / "predictions.tsv", score_dir=root / "new") with self.assertRaisesRegex(ValueError, "fresh --score-dir"): launchable.score(root, "smoke", predictions=root / "predictions.tsv", score_dir=root) + for source, status, dry_run in (("other-source", "inferred", False), + (launchable.SOURCE, "failed", False), + (launchable.SOURCE, "planned", True)): + inference_state(root, source=source, status=status, dry_run=dry_run) + with self.assertRaisesRegex(ValueError, "successful PerceptionTest"): + launchable.score(root, "smoke", work=root) native.score_predictions.assert_not_called() - def test_changed_annotations_or_scope_fail_before_native_scoring(self): - for change in ("scope", "manifest_sha256", "selected_indices"): + def test_changed_annotations_cuts_or_scope_fail_before_native_scoring(self): + for change in ("scope", "dataset_revision", "manifest_sha256", + "annotation_manifest_sha256", "selected_indices"): with self.subTest(change=change), tempfile.TemporaryDirectory() as temp: root = Path(temp) selection = prepared() - state = {**selection, "source": "mvbench-local", "status": "inferred", - "scope": "smoke", "nframe": 16, "predictions": "native.tsv"} - if change == "scope": - state["scope"] = "full" - elif change == "manifest_sha256": - state[change] = "different-hash" - else: - state[change] = [9999] - (root / "run.json").write_text(json.dumps(state)) - native = types.ModuleType("vlmeval.mvbench_local") - native.score_predictions = unittest.mock.Mock() - with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}), \ + updates = {change: "full" if change == "scope" else [9999] + if change == "selected_indices" else "changed"} + inference_state(root, **updates) + native = scorer_module() + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}), \ patch.object(launchable, "prepare", return_value=selection): with self.assertRaisesRegex(ValueError, "scope|changed"): launchable.score(root / "data", "smoke", work=root) native.score_predictions.assert_not_called() self.assertFalse((root / "scores").exists()) + def test_prediction_artifact_cannot_escape_inference_directory(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + inference_state(root, predictions="../outside.tsv") + native = scorer_module() + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}): + with self.assertRaisesRegex(ValueError, "within its inference work directory"): + launchable.score(root / "data", "smoke", work=root) + native.score_predictions.assert_not_called() + def test_failed_prediction_never_enters_native_scorer(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) - path = table(root / "native", [3400], ["VIDEO_LOAD_FAILED"]) - native = types.ModuleType("vlmeval.mvbench_local") - native.score_predictions = unittest.mock.Mock() - with patch.dict(sys.modules, {"vlmeval.mvbench_local": native}), \ - patch.object(launchable, "prepare", return_value=prepared([3400])): + path = table(root / "native", [13404], ["VIDEO_LOAD_FAILED"]) + native = scorer_module() + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}), \ + patch.object(launchable, "prepare", return_value=prepared([13404], media=False)): with self.assertRaisesRegex(RuntimeError, "failed/empty"): launchable.score(root / "data", "smoke", predictions=path, score_dir=root / "scores") native.score_predictions.assert_not_called() self.assertFalse((root / "scores").exists()) - def test_streamed_child_output_and_failure_log_retained(self): + def test_score_only_works_without_media_or_model_and_keeps_annotation_identity(self): with tempfile.TemporaryDirectory() as temp: - path = Path(temp) / "run.log" - captured = io.StringIO() - with contextlib.redirect_stdout(captured): - with self.assertRaises(subprocess.CalledProcessError): - launchable.run_command([sys.executable, "-u", "-c", - "print('first', flush=True); raise SystemExit(7)"], log_path=path) - self.assertEqual(captured.getvalue(), "first\n") - self.assertEqual(path.read_text(), "first\n") + root = Path(temp) + selection = prepared([13404]) + path = table(root / "native", [13404], ["nan"]) + inference_state(root, selection, predictions=str(path.relative_to(root))) + metadata = prepared([13404], media=False) + def score_artifact(predictions, data_root, output_dir, smoke, nframe): + self.assertEqual(predictions, path) + self.assertTrue(smoke) + self.assertFalse((data_root / "videos").exists()) + output_dir.mkdir() + summary = {"status": "complete", "primary_metric": { + "name": "top1", "value": 0.0, "correct": 0, "total": 1}, + "coverage": metadata["coverage"]} + (output_dir / "summary.json").write_text(json.dumps(summary)) + return summary + native = scorer_module(score_artifact) + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}), \ + patch.object(launchable, "prepare", return_value=metadata) as prep, \ + patch.object(launchable, "endpoint") as endpoint, \ + patch.object(launchable, "run_command") as child: + summary = launchable.score(root / "metadata-only", "smoke", work=root) + prep.assert_called_once_with(root / "metadata-only", smoke=True, require_media=False) + endpoint.assert_not_called() + child.assert_not_called() + self.assertEqual(summary["primary_metric"]["total"], 1) + self.assertEqual(json.loads((root / "run.json").read_text())["status"], "complete") + controller = json.loads((root / "scores" / "controller.json").read_text()) + self.assertIn("ended_utc", controller["timing"]) + + def test_scoring_execution_failure_is_retained_without_complete_status(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path = table(root / "native", [13404]) + inference_state(root, prepared([13404]), predictions=str(path.relative_to(root))) + def fail(predictions, data_root, output_dir, smoke, nframe): + output_dir.mkdir() + (output_dir / "diagnostic.txt").write_text("retain") + raise RuntimeError("scoring failed") + native = scorer_module(fail) + with patch.dict(sys.modules, {"vlmeval.perception_test_local": native}), \ + patch.object(launchable, "prepare", return_value=prepared([13404], media=False)): + with self.assertRaisesRegex(RuntimeError, "scoring failed"): + launchable.score(root / "data", "smoke", work=root) + state = json.loads((root / "run.json").read_text()) + self.assertEqual(state["status"], "failed") + self.assertIn("ended_utc", state["scoring_timing"]) + self.assertEqual((root / "scores" / "diagnostic.txt").read_text(), "retain") if __name__ == "__main__": diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py deleted file mode 100644 index 4c31c4373..000000000 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mvbench.py +++ /dev/null @@ -1,368 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Synthetic consumer-boundary tests; no network, GPU, or private media. - -The fixture trust root is patched to hashes of synthetic annotations, never to -bypass authentication. Counts, native task ordering, preparation, image reading, -native exact matching, and extraction all execute their real implementations. -""" -import contextlib -import io -import json -from pathlib import Path -import stat -import tempfile -import unittest -from unittest.mock import patch -import zipfile - -import pandas as pd -from PIL import Image - -from vlmeval import mvbench_local as local -from vlmeval.dataset.mvbench import MVBench -from vlmeval.smp import get_intermediate_file_path - - -def _write_json(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value), encoding='utf-8') - - -class MVBenchTests(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.root = Path(self.temporary.name) - self.data = self.root / 'data' - self.hashes = {} - self.sources = {} - for task, (name, prefix, data_type, bound) in MVBench.TASKS.items(): - rows = [] - for index in range(200): - row = { - 'video': 'fixture' if data_type == 'frame' else 'fixture.mp4', - 'question': f'Choose the shape in synthetic {task} question {index}.', - 'candidates': ['red square', 'blue circle', 'green triangle'], - 'answer': ['red square', 'blue circle', 'green triangle'][index % 3], - } - if bound: - row.update(start=0, end=2) - rows.append(row) - self.sources[name] = rows - path = self.data / 'json' / name - _write_json(path, rows) - self.hashes[f'json/{name}'] = local._sha256(path) - for name in ('README.md', 'video/MVBench_videos_ntu.txt'): - path = self.data / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text('Synthetic public-input fixture, no actual dataset media.\n') - self.hashes[name] = local._sha256(path) - self.trust = patch.dict(local.METADATA_SHA256, self.hashes, clear=True) - self.trust.start() - self.addCleanup(self.trust.stop) - self._media(local.SMOKE_TASK) - - def _media(self, task): - name, prefix, data_type, bound = MVBench.TASKS[task] - path = self.data / prefix.replace('your_data_path', 'video') / self.sources[name][0]['video'] - if data_type == 'frame': - path.mkdir(parents=True, exist_ok=True) - for index in range(1, 10): - color = (255, 0, 0) if index <= 3 else (0, 0, 255) - Image.new('RGB', (320, 240), color).save(path / f'{index:05d}.jpg') - else: - path.parent.mkdir(parents=True, exist_ok=True) - # Scoring/coverage fixtures need identity and nonempty media, not a - # decoder. The prompt test below uses real numbered JPEG inputs. - path.write_bytes(b'Synthetic scoring-only media identity; not decoded.') - return path - - def _repin(self, name): - path = self.data / 'json' / name - _write_json(path, self.sources[name]) - local.METADATA_SHA256[f'json/{name}'] = local._sha256(path) - - def _table(self): - prepared = local.prepare_dataset(self.data, smoke=True) - table = pd.read_csv(prepared['data_file'], sep='\t') - table['prediction'] = ['(A)', '(B)', '(C)', '(A)', '(B)'] - return table - - def _predictions(self, table=None, name='predictions.tsv'): - path = self.root / name - (self._table() if table is None else table).to_csv(path, sep='\t', index=False) - return path - - def _score(self, predictions=None, name='results'): - path = self._predictions() if predictions is None else predictions - with contextlib.redirect_stdout(io.StringIO()): - return local.score_predictions(path, self.data, self.root / name, smoke=True) - - def test_smoke_is_fixed_subset_and_never_a_full_score(self): - prepared = local.prepare_dataset(self.data, smoke=True) - self.assertEqual(prepared['selected_indices'], list(range(3400, 3405))) - self.assertEqual(prepared['coverage']['expected_rows'], 4000) - self.assertEqual(prepared['coverage']['available_rows'], 200) - self.assertEqual(prepared['coverage']['selected_rows'], 5) - self.assertEqual(prepared['coverage']['selected_tasks'], ['Egocentric Navigation']) - self.assertTrue(prepared['coverage']['subset']) - with self.assertRaises(local.MVBenchCoverageError) as caught: - local.prepare_dataset(self.data) - self.assertIn('4000', str(caught.exception)) - self.assertIn('video/nturgbd', str(caught.exception)) - self.assertIn('Fine-grained Pose', caught.exception.report['missing_media_by_task']) - self.assertTrue((self.data / '.mvbench/coverage-full.json').is_file()) - summary = self._score() - self.assertEqual(summary['status'], 'smoke_only') - self.assertEqual(summary['primary_metric']['value'], 1.0) - self.assertEqual(summary['primary_metric']['total'], 5) - self.assertFalse(summary['provenance']['paper_parity']) - self.assertEqual(json.loads((self.root / 'results/summary.json').read_text()), summary) - - def test_full_requires_manual_ntu_and_does_not_drop_its_rows(self): - for task in MVBench.TASKS: - if task != 'Fine-grained Pose': - self._media(task) - with self.assertRaises(local.MVBenchCoverageError) as caught: - local.prepare_dataset(self.data) - self.assertEqual(caught.exception.report['coverage']['available_rows'], 3800) - self.assertEqual(set(caught.exception.report['missing_media_by_task']), {'Fine-grained Pose'}) - self._media('Fine-grained Pose') - full = local.prepare_dataset(self.data) - self.assertEqual(full['coverage']['available_rows'], 4000) - self.assertEqual(full['coverage']['selected_rows'], 4000) - self.assertEqual(len(full['coverage']['selected_tasks']), 20) - self.assertFalse(full['coverage']['subset']) - - def test_source_metadata_and_prebuilt_tsv_cannot_replace_ground_truth(self): - prepared = local.prepare_dataset(self.data, smoke=True) - canonical = Path(prepared['data_file']) - canonical.write_text('index\tanswer\n3400\tforged\n') - repaired = local.prepare_dataset(self.data, smoke=True) - self.assertEqual(len(pd.read_csv(repaired['data_file'], sep='\t')), 5) - source = self.data / 'json/egocentric_navigation.json' - source.write_text(source.read_text().replace('red square', 'forged square')) - with self.assertRaisesRegex(ValueError, 'checksum mismatch'): - local.prepare_dataset(self.data, smoke=True) - - def test_media_content_mutation_changes_manifest_not_just_path(self): - first = local.prepare_dataset(self.data, smoke=True) - video = self.data / next(iter(first['media_sha256'])) - video.write_bytes(b'Changed synthetic media at the same path.') - second = local.prepare_dataset(self.data, smoke=True) - self.assertNotEqual(first['media_manifest_sha256'], second['media_manifest_sha256']) - self.assertNotEqual(first['manifest_sha256'], second['manifest_sha256']) - self.assertEqual(first['annotation_manifest_sha256'], second['annotation_manifest_sha256']) - - def test_missing_bad_answer_and_changed_official_row_count_are_rejected(self): - name = 'egocentric_navigation.json' - for answer in (None, 'not a candidate'): - with self.subTest(answer=answer): - self.sources[name][0]['answer'] = answer - self._repin(name) - with self.assertRaisesRegex(ValueError, 'Answer must identify'): - local.prepare_dataset(self.data, smoke=True) - self.sources[name][0]['answer'] = 'red square' - self.sources[name].pop() - self._repin(name) - with self.assertRaisesRegex(ValueError, 'exactly 200'): - local.prepare_dataset(self.data, smoke=True) - - def test_empty_and_missing_media_cannot_enter_scoring(self): - video = self._media(local.SMOKE_TASK) - for remove in (False, True): - with self.subTest(remove=remove): - if remove: - video.unlink() - else: - video.write_bytes(b'') - with self.assertRaises(local.MVBenchCoverageError): - local.prepare_dataset(self.data, smoke=True) - - def test_legitimate_one_and_five_candidate_questions_are_not_rewritten(self): - name = 'egocentric_navigation.json' - self.sources[name][0].update(candidates=['red square'], answer='red square') - self.sources[name][1].update( - candidates=['red square', 'blue circle', 'green triangle', 'yellow star', 'purple pentagon'], - answer='purple pentagon', - ) - self._repin(name) - table = self._table() - table.loc[1, 'prediction'] = '(E)' - summary = self._score(self._predictions(table)) - self.assertEqual(summary['primary_metric']['correct'], 5) - - def test_prediction_identity_coverage_is_exact_not_just_row_count(self): - original = self._table() - cases = { - 'missing': original.iloc[:-1].copy(), - 'duplicate': pd.concat([original.iloc[:-1], original.iloc[:1]], ignore_index=True), - 'extra': pd.concat([original, original.iloc[:1].assign(index=99999)], ignore_index=True), - 'substituted': original.assign(index=[3400, 3401, 3402, 3403, 99999]), - } - for name, table in cases.items(): - with self.subTest(name=name): - with self.assertRaisesRegex(ValueError, 'coverage must match'): - self._score(self._predictions(table, name + '.tsv'), name) - self.assertFalse((self.root / name / 'summary.json').exists()) - reordered = original.iloc[::-1] - summary = self._score(self._predictions(reordered, 'reordered.tsv')) - self.assertEqual(summary['primary_metric']['correct'], 5) - - def test_tampered_identity_labels_candidates_and_prompt_are_rejected(self): - marker = self.root / 'executed' - changes = { - 'answer': 'forged answer', 'video': '../elsewhere.mp4', - 'task_type': 'Action Sequence', 'question': 'forged question', - 'candidates': f'__import__("pathlib").Path({str(marker)!r}).touch()', - } - for column, value in changes.items(): - with self.subTest(column=column): - table = self._table() - table.loc[0, column] = value - with self.assertRaisesRegex(ValueError, 'Untrusted'): - self._score(self._predictions(table, f'{column}.tsv'), column) - self.assertFalse(marker.exists()) - # Only index/prediction is also valid; all answer identities are rebuilt. - summary = self._score(self._predictions(self._table()[['index', 'prediction']], 'minimal.tsv')) - self.assertEqual(summary['primary_metric']['correct'], 5) - - def test_failure_rows_never_receive_truthy_minus_one_credit(self): - for prediction in ('', ' ', 'Failed to obtain answer via API.', 'VIDEO_LOAD_FAILED', - 'Failed to obtain answer: RuntimeError synthetic failure'): - with self.subTest(prediction=prediction): - table = self._table() - table.loc[0, 'prediction'] = prediction - with self.assertRaisesRegex(ValueError, 'Missing or failed prediction'): - self._score(self._predictions(table), 'failed') - self.assertFalse((self.root / 'failed/summary.json').exists()) - - def test_existing_score_output_is_preserved(self): - output = self.root / 'existing' - output.mkdir() - sentinel = output / 'summary.json' - sentinel.write_text('previous result') - with self.assertRaisesRegex(ValueError, 'fresh output'): - local.score_predictions(self._predictions(), self.data, output, smoke=True) - self.assertEqual(sentinel.read_text(), 'previous result') - - def test_nonempty_malformed_predictions_are_native_wrong_answers_offline(self): - table = self._table() - # These strings must not become pandas NA or numeric scalars in native - # scoring. Native model=None deterministically gives unknown option Z. - table['prediction'] = ['nan', '1', '.', '(', ')'] - with patch('vlmeval.dataset.mvbench.build_judge', side_effect=AssertionError('network judge forbidden')): - summary = self._score(self._predictions(table)) - self.assertEqual(summary['primary_metric']['correct'], 0) - self.assertEqual(summary['primary_metric']['total'], 5) - self.assertFalse(summary['provenance']['network_judge']) - scores = pd.read_csv(self.root / 'results/scored_predictions.tsv', sep='\t') - self.assertEqual(scores['score'].tolist(), [0] * 5) - self.assertEqual(scores['index'].tolist(), list(range(3400, 3405))) - - def test_native_micro_accuracy_is_not_task_macro_for_unequal_sizes(self): - table = pd.DataFrame([ - {'index': index, 'task_type': 'large' if index < 3 else 'small', - 'question': 'Choose a shape.', 'answer': 'red square', - 'candidates': repr(['red square', 'blue circle']), - 'prediction': '(A)' if index < 3 else '(B)'} - for index in range(4) - ]) - native_file = str(self.root / 'unequal.tsv') - table.to_csv(native_file, sep='\t', index=False) - with contextlib.redirect_stdout(io.StringIO()): - rating = MVBench.evaluate(native_file, trusted_data=table, model='exact_matching') - summary = local._summarize_rating(rating) - self.assertEqual(summary['primary_metric']['value'], 0.75) - self.assertEqual(summary['task_macro_accuracy'], 0.5) - self.assertEqual(summary['per_task']['large']['total'], 3) - self.assertTrue(Path(get_intermediate_file_path(native_file, '_score')).exists()) - - def test_native_image_prompt_respects_clip_bounds_cache_and_reference_crop(self): - frames = self._media('Episodic Reasoning') - dataset = local.build_dataset(self.data, nframe=4, smoke=True) - line = dict(dataset.data.iloc[0]) - line.update(prefix=str(frames.parent.relative_to(self.data)), video=frames.name, - data_type='frame', bound=True, start=0, end=1) - first = dataset.build_prompt(line, video_llm=True) - first_images = [part['value'] for part in first if part['type'] == 'image'] - self.assertEqual(len(first_images), 4) - self.assertFalse(any(part['type'] == 'video' for part in first)) - self.assertIn('Question: Choose the shape', first[-3]['value']) - self.assertEqual(first[-1]['value'], 'Best option:(') - for path in first_images: - with Image.open(path) as image: - self.assertEqual(image.size, (224, 224)) - line.update(start=1, end=2) - second = dataset.build_prompt(line, video_llm=True) - second_images = [part['value'] for part in second if part['type'] == 'image'] - self.assertTrue(set(first_images).isdisjoint(second_images)) - with Image.open(first_images[-1]) as left, Image.open(second_images[-1]) as right: - self.assertNotEqual(left.getpixel((100, 100)), right.getpixel((100, 100))) - self.assertEqual(dataset.save_video_into_images(line), second_images) - key = dataset.clip_cache_key(line) - dataset.nframe = 8 - self.assertNotEqual(dataset.clip_cache_key(line), key) - dataset.nframe = 4 - # Mutate a numbered source frame, not the path or clip bounds. - Image.new('RGB', (320, 240), (0, 255, 0)).save(frames / '00005.jpg') - self.assertNotEqual(dataset.clip_cache_key(line), key) - changed_images = dataset.save_video_into_images(line) - self.assertTrue(set(changed_images).isdisjoint(second_images)) - alternate = frames.parent / 'different-source' / frames.name - alternate.mkdir(parents=True) - for frame in frames.glob('*.jpg'): - (alternate / frame.name).write_bytes(frame.read_bytes()) - line['prefix'] += '/different-source' - self.assertNotEqual(dataset.clip_cache_key(line), dataset.clip_cache_key({ - **line, 'prefix': str(frames.parent.relative_to(self.data)), - })) - - def test_extraction_validates_every_member_before_writing(self): - for bad_name in ('../escape.txt', '/absolute.txt', 'C:/drive.txt', 'folder\\escape.txt'): - with self.subTest(name=bad_name): - archive = self.root / 'unsafe.zip' - with zipfile.ZipFile(archive, 'w') as source: - source.writestr('innocent.txt', 'must not be extracted either') - source.writestr(bad_name, 'unsafe') - destination = self.root / 'extract' - with self.assertRaisesRegex(ValueError, 'Unsafe relative path'): - local._safe_extract(archive, destination) - self.assertFalse((destination / 'innocent.txt').exists()) - self.assertFalse((self.root / 'escape.txt').exists()) - - def test_zip_symlinks_and_preexisting_escape_links_are_rejected(self): - archive = self.root / 'symlink.zip' - link = zipfile.ZipInfo('link') - link.create_system = 3 - link.external_attr = (stat.S_IFLNK | 0o777) << 16 - with zipfile.ZipFile(archive, 'w') as source: - source.writestr(link, '../escape') - with self.assertRaisesRegex(ValueError, 'member type'): - local._safe_extract(archive, self.root / 'extract') - destination = self.root / 'extract' - outside = self.root / 'outside' - outside.mkdir() - (destination / 'linked').symlink_to(outside, target_is_directory=True) - with zipfile.ZipFile(archive, 'w') as source: - source.writestr('linked/escape.txt', 'unsafe') - with self.assertRaisesRegex(ValueError, 'escapes dataset root'): - local._safe_extract(archive, destination) - self.assertFalse((outside / 'escape.txt').exists()) - - def test_june_overlay_overwrites_only_its_unwrapped_media_path(self): - destination = self.root / 'video' - target = destination / 'scene_qa/video/fixture.mp4' - target.parent.mkdir(parents=True) - target.write_bytes(b'old') - archive = self.root / 'data0613.zip' - with zipfile.ZipFile(archive, 'w') as source: - source.writestr('data0613/scene_qa/video/fixture.mp4', b'corrected') - local._safe_extract(archive, destination, overlay=True) - self.assertEqual(target.read_bytes(), b'corrected') - self.assertFalse((destination / 'data0613').exists()) - - -if __name__ == '__main__': - unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_perception_test.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_perception_test.py new file mode 100644 index 000000000..6b43228c1 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_perception_test.py @@ -0,0 +1,462 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic trust-boundary and real-video tests; no network, judge or GPU.""" +import copy +import io +import json +from pathlib import Path +import shutil +import stat +import tempfile +import unittest +from unittest.mock import patch +import warnings +import zipfile + +import cv2 +import numpy as np +import pandas as pd +from PIL import Image + +from vlmeval import perception_test_local as local +from vlmeval.dataset.perception_test import PerceptionTestLocal + + +class PerceptionTestTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.data = self.root / 'data' + self.data.mkdir() + self.annotations = {} + for video, size in (('video_9000', 2), ('video_8241', 5), ('video_2', 2)): + questions = [] + for i in reversed(range(size)): + questions.append({'id': i, 'question': f'Which shape is shown in question {i}?', + 'options': ['red square', 'blue circle', 'green triangle'], + 'answer_id': i % 3, 'area': 'memory' if i % 2 == 0 else 'physics', + 'reasoning': 'descriptive', + 'tag': ['shape', 'motion'] if i % 2 == 0 else ['shape']}) + self.annotations[video] = {'metadata': {'video_id': video, 'split': 'valid', 'num_frames': 12}, + 'mc_question': questions} + self.cuts = {'video_2': -1, 'video_8241': 6, 'video_9000': -1} + self._json('mc_question_valid.json', self.annotations) + self._json('cut_frame_mapping_valid.json', self.cuts) + (self.data / 'README.md').write_text('Synthetic attribution fixture; not actual benchmark data.\n') + hashes = {name: local._sha256(self.data / name) for name in local.METADATA_SHA256} + for target, value in (('EXPECTED_ROWS', 9), ('EXPECTED_VIDEOS', 3), ('SMOKE_INDICES', list(range(2, 7)))): + patcher = patch.object(local, target, value) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch.dict(local.METADATA_SHA256, hashes, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + + def _json(self, name, value): + (self.data / name).write_text(json.dumps(value), encoding='utf-8') + + def _repin(self, name, value): + self._json(name, value) + local.METADATA_SHA256[name] = local._sha256(self.data / name) + + def _table(self): + result = local.prepare_dataset(self.data, smoke=True, require_media=False) + table = pd.read_csv(result['data_file'], sep='\t', keep_default_na=False) + table['prediction'] = ['(A)', 'B', 'Option C.', 'Answer: A', '(B) blue circle'] + return table + + def _predictions(self, table=None, name='predictions.tsv'): + path = self.root / name + (self._table() if table is None else table).to_csv(path, sep='\t', index=False) + return path + + def _score(self, table=None, name='score'): + return local.score_predictions(self._predictions(table), self.data, self.root / name, smoke=True) + + def _media(self, video='video_8241'): + path = self.data / 'videos' / f'{video}.mp4' + path.parent.mkdir(exist_ok=True) + path.write_bytes(b'Synthetic identity bytes, not a decodable video.') + return path + + def _real_video(self, red_after=6, size=(960, 480)): + path = self.data / 'videos' / 'video_8241.mp4' + path.parent.mkdir(exist_ok=True) + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*'mp4v'), 6, size) + if not writer.isOpened(): + self.fail('OpenCV mp4v encoding is required for real frame-boundary tests') + try: + for i in range(12): + # BGR: allowed frames green, forbidden tail red. + color = (0, 180, 0) if i < red_after else (0, 0, 220) + frame = np.empty((size[1], size[0], 3), dtype=np.uint8) + frame[:] = color + writer.write(frame) + finally: + writer.release() + return path + + def _dataset(self, nframe=4): + return PerceptionTestLocal(data_root=self.data, smoke=True, nframe=nframe) + + def test_small_real_frames_are_not_upscaled(self): + self._real_video(size=(320, 160)) + dataset = self._dataset() + for path in dataset.save_video_frames(dataset.data.iloc[0]): + with Image.open(path) as image: + self.assertEqual(image.size, (320, 160)) + + def test_choice_adapter_rejects_malformed_or_conflicting_choices(self): + options = ['red square', 'blue circle', 'green triangle'] + for text in ('(A', 'A)', 'A or B', '(A)(B)', 'Ared square', '???', 'nan'): + with self.subTest(text=text): + self.assertIsNone(local.parse_choice(text, options)) + self.assertEqual(local.parse_choice('blue circle', options), 1) + self.assertEqual(local.parse_choice('Answer: C. It is the triangle.', options), 2) + + def test_numeric_video_and_question_order_preserves_stable_selected_ids(self): + prepared = local.prepare_dataset(self.data, smoke=True, require_media=False) + self.assertEqual(prepared['selected_indices'], [2, 3, 4, 5, 6]) + table = pd.read_csv(prepared['data_file'], sep='\t') + self.assertEqual(table['question_id'].tolist(), [0, 1, 2, 3, 4]) + self.assertEqual(table['video_id'].unique().tolist(), ['video_8241']) + self.assertEqual(prepared['coverage']['expected_rows'], 9) + self.assertEqual(prepared['coverage']['selected_rows'], 5) + + def test_full_shape_preserves_fixed_absolute_smoke_identities(self): + annotations, cuts = {}, {} + template = self.annotations['video_8241']['mc_question'][0] + video_ids = list(range(1, 4002)) + [8241] + list(range(8242, 9500)) + for position, video_number in enumerate(video_ids): + video = f'video_{video_number}' + if position < 4001: + size = 4 if position < 1401 else 3 + elif video_number == 8241: + size = 5 + else: + size = 5 if position - 4002 < 699 else 4 + questions = [{**template, 'id': i} for i in reversed(range(size))] + annotations[video] = {'metadata': {'video_id': video, 'split': 'valid', 'num_frames': 12}, + 'mc_question': questions} + cuts[video] = -1 + self._repin('mc_question_valid.json', annotations) + self._repin('cut_frame_mapping_valid.json', cuts) + with patch.object(local, 'EXPECTED_ROWS', 19140), patch.object(local, 'EXPECTED_VIDEOS', 5260), \ + patch.object(local, 'SMOKE_INDICES', [13404, 13405, 13406, 13407, 13408]): + prepared = local.prepare_dataset(self.data, smoke=True, require_media=False) + self.assertEqual(prepared['selected_indices'], [13404, 13405, 13406, 13407, 13408]) + self.assertEqual(prepared['coverage']['expected_rows'], 19140) + self.assertEqual(prepared['coverage']['expected_videos'], 5260) + full = local.prepare_dataset(self.data, require_media=False) + self.assertEqual(full['selected_indices'], list(range(19140))) + self.assertEqual(full['coverage']['selected_videos'], 5260) + + def test_native_export_preserves_missing_value_like_option_text(self): + changed = copy.deepcopy(self.annotations) + question = next(q for q in changed['video_8241']['mc_question'] if q['id'] == 0) + question['options'] = ['nan', 'NA', 'null'] + self._repin('mc_question_valid.json', changed) + self._media() + dataset = self._dataset() + self.assertEqual(dataset.data.iloc[0][['A', 'B', 'C']].tolist(), ['nan', 'NA', 'null']) + + def test_media_independent_manifest_and_offline_scoring(self): + offline = local.prepare_dataset(self.data, smoke=True, require_media=False) + self._media() + online = local.prepare_dataset(self.data, smoke=True) + self.assertEqual(offline['manifest_sha256'], online['manifest_sha256']) + self.assertEqual(offline['annotation_manifest_sha256'], online['annotation_manifest_sha256']) + self.assertIsNone(offline['media_manifest_sha256']) + self.assertIn('videos/video_8241.mp4', online['media_sha256']) + shutil.rmtree(self.data / 'videos') + summary = self._score() + self.assertEqual(summary['primary_metric'], {'name': 'top1', 'value': 1., 'unit': 'fraction', + 'correct': 5, 'total': 5}) + self.assertEqual(summary['status'], 'smoke_only') + self.assertEqual(summary['invalid_count'], 0) + self.assertEqual(summary['provenance']['media_sha256'], {}) + + def test_missing_full_media_never_drops_rows(self): + self._media() + with self.assertRaises(local.PerceptionTestCoverageError) as caught: + local.prepare_dataset(self.data) + self.assertEqual(caught.exception.report['coverage']['available_rows'], 5) + self.assertEqual(caught.exception.report['coverage']['selected_rows'], 9) + self.assertEqual(caught.exception.report['missing_media'], ['videos/video_2.mp4', 'videos/video_9000.mp4']) + self._media('video_2') + self._media('video_9000') + prepared = local.prepare_dataset(self.data) + self.assertEqual(prepared['selected_indices'], list(range(9))) + self.assertEqual(prepared['coverage']['selected_rows'], 9) + + def test_prebuilt_tsv_cannot_replace_immutable_labels(self): + table = self._table() + prepared = local.prepare_dataset(self.data, smoke=True, require_media=False) + Path(prepared['data_file']).write_text('index\tanswer_id\n2\t2\n') + regenerated = local.prepare_dataset(self.data, smoke=True, require_media=False) + fresh = pd.read_csv(regenerated['data_file'], sep='\t') + self.assertEqual(fresh['answer_id'].tolist(), [0, 1, 2, 0, 1]) + self.assertEqual(self._score(table)['primary_metric']['correct'], 5) + + def test_annotation_labels_options_and_cuts_are_checksum_pinned(self): + original = (self.data / 'mc_question_valid.json').read_bytes() + for field, replacement in (('answer_id', 2), ('options', ['forged', 'b', 'c'])): + changed = copy.deepcopy(self.annotations) + changed['video_8241']['mc_question'][0][field] = replacement + self._json('mc_question_valid.json', changed) + with self.assertRaisesRegex(ValueError, 'checksum'): + local.prepare_dataset(self.data, smoke=True, require_media=False) + (self.data / 'mc_question_valid.json').write_bytes(original) + self._json('cut_frame_mapping_valid.json', {**self.cuts, 'video_8241': -1}) + with self.assertRaisesRegex(ValueError, 'checksum'): + local.prepare_dataset(self.data, smoke=True, require_media=False) + + def test_invalid_and_missing_trusted_cut_bounds_are_rejected(self): + for cut in (0, -2, 13, True, 1.5, None): + with self.subTest(cut=cut): + self._repin('cut_frame_mapping_valid.json', {**self.cuts, 'video_8241': cut}) + with self.assertRaisesRegex(ValueError, 'cut'): + local.prepare_dataset(self.data, smoke=True, require_media=False) + self._repin('cut_frame_mapping_valid.json', {'video_2': -1, 'video_9000': -1}) + with self.assertRaisesRegex(ValueError, 'cut'): + local.prepare_dataset(self.data, smoke=True, require_media=False) + + def test_duplicate_missing_extra_and_modified_prediction_identity_rejected(self): + table = self._table() + cases = [table.iloc[:-1].copy(), pd.concat([table, table.iloc[:1]], ignore_index=True)] + extra = table.copy() + extra.loc[0, 'index'] = 999 + cases.append(extra) + modified = table.copy() + modified.loc[0, 'video_id'] = 'video_2' + cases.append(modified) + modified = table.copy() + modified.loc[0, 'question_id'] = 99 + cases.append(modified) + modified = table.copy() + modified['index'] = modified['index'].astype(float) + modified.loc[0, 'index'] = 2.5 + cases.append(modified) + for i, case in enumerate(cases): + with self.subTest(case=i), self.assertRaises(ValueError): + self._score(case, name=f'bad-{i}') + self.assertFalse((self.root / f'bad-{i}').exists()) + + def test_prediction_ground_truth_options_question_and_cut_tampering_rejected(self): + for column, replacement in (('answer', 'C'), ('answer_id', 2), ('A', 'forged'), + ('options', '["forged", "blue circle", "green triangle"]'), + ('question', 'forged question'), ('cut_frame', -1), + ('num_frames', 99), ('tag', '["forged"]')): + table = self._table() + table.loc[0, column] = replacement + with self.subTest(column=column), self.assertRaisesRegex(ValueError, 'Untrusted'): + self._score(table, name=f'tampered-{column}') + + def test_minimal_shuffled_predictions_use_trusted_labels(self): + table = self._table()[['index', 'prediction']].iloc[::-1] + self.assertEqual(self._score(table)['primary_metric']['correct'], 5) + + def test_malformed_ambiguous_literal_nan_are_wrong_not_dropped(self): + table = self._table() + table['prediction'] = ['A or B', 'nan', 'unparseable model response', '(A)', '(C)'] + summary = self._score(table) + self.assertEqual(summary['primary_metric']['correct'], 1) + self.assertEqual(summary['primary_metric']['total'], 5) + self.assertEqual(summary['invalid_count'], 3) + scored = pd.read_csv(self.root / 'score/scored_predictions.tsv', sep='\t', keep_default_na=False) + self.assertEqual(scored['prediction'][1], 'nan') + self.assertEqual(scored['score'].tolist(), [0, 0, 0, 1, 0]) + + def test_blank_missing_and_failed_outputs_reject_incomplete_runs(self): + for i, failure in enumerate(('', ' ', None, 'Failed to obtain answer via API.', + 'VIDEO_LOAD_FAILED: truncated', 'transport_error: reset', 'API_ERROR')): + table = self._table() + table.loc[0, 'prediction'] = failure + with self.subTest(failure=failure), self.assertRaisesRegex(ValueError, 'Missing or failed'): + self._score(table, name=f'failure-{i}') + + def test_exact_counts_and_overlapping_tag_denominators(self): + table = self._table() + table['prediction'] = ['A', 'C', 'C', 'C', 'B'] + summary = self._score(table) + self.assertEqual(summary['primary_metric']['correct'], 3) + self.assertEqual(summary['primary_metric']['value'], .6) + self.assertEqual(summary['per_area']['memory'], {'correct': 3, 'total': 3, 'accuracy': 1.}) + self.assertEqual(summary['per_area']['physics'], {'correct': 0, 'total': 2, 'accuracy': 0.}) + self.assertEqual(summary['per_reasoning']['descriptive']['total'], 5) + self.assertEqual(summary['per_tag']['shape']['total'], 5) + self.assertEqual(summary['per_tag']['motion']['total'], 3) + self.assertEqual(sum(g['total'] for g in summary['per_tag'].values()), 8) + + def test_duplicate_option_text_preserves_official_option_id(self): + changed = copy.deepcopy(self.annotations) + question = next(q for q in changed['video_8241']['mc_question'] if q['id'] == 0) + question['options'] = ['same text', 'same text', 'other text'] + question['answer_id'] = 1 + self._repin('mc_question_valid.json', changed) + table = self._table() + table.loc[0, 'prediction'] = 'B' + self.assertEqual(self._score(table)['primary_metric']['correct'], 5) + table.loc[0, 'prediction'] = 'same text' + summary = self._score(table, name='ambiguous-text') + self.assertEqual(summary['invalid_count'], 1) + self.assertEqual(summary['primary_metric']['correct'], 4) + + def test_prediction_files_never_execute_pickle_or_expressions(self): + payload = self.root / 'payload.pkl' + payload.write_bytes(b'not a pickle') + with self.assertRaisesRegex(ValueError, 'pickle'): + local.score_predictions(payload, self.data, self.root / 'unsafe', smoke=True) + table = self._table() + table.loc[0, 'options'] = '__import__("os").system("false")' + with self.assertRaisesRegex(ValueError, 'Untrusted options'): + self._score(table) + + def test_fresh_output_required_and_stale_neighbor_scores_ignored(self): + table = self._table() + predictions = self._predictions(table) + (self.root / 'predictions_score.tsv').write_text('forged stale score\n') + summary = local.score_predictions(predictions, self.data, self.root / 'fresh', smoke=True) + self.assertEqual(summary['primary_metric']['correct'], 5) + before = (self.root / 'fresh/summary.json').read_bytes() + with self.assertRaisesRegex(ValueError, 'fresh'): + local.score_predictions(predictions, self.data, self.root / 'fresh', smoke=True) + self.assertEqual((self.root / 'fresh/summary.json').read_bytes(), before) + + def test_real_sampled_frames_enforce_exclusive_cut_and_aspect_resize(self): + self._real_video() + dataset = self._dataset() + prompt = dataset.build_prompt(0, video_llm=True) + images = [m['value'] for m in prompt if m['type'] == 'image'] + self.assertEqual(len(images), 4) + self.assertFalse(any(m['type'] == 'video' for m in prompt)) + for path in images: + with Image.open(path) as image: + self.assertEqual(image.size, (768, 384)) + red, green, blue = image.getpixel((300, 200)) + self.assertGreater(green, red + 100) + line = dataset.data.iloc[0].copy() + line['cut_frame'] = -1 + full_paths = dataset.save_video_frames(line) + with Image.open(full_paths[-1]) as image: + red, green, blue = image.getpixel((300, 200)) + self.assertGreater(red, green + 100) + self.assertNotEqual(images, full_paths) + self.assertEqual(PerceptionTestLocal.sample_indices(12, 6, 4), [0, 2, 3, 5]) + self.assertEqual(PerceptionTestLocal.sample_indices(1, -1, 4), [0, 0, 0, 0]) + + def test_real_frame_cache_changes_for_content_cut_nframe_and_protocol(self): + path = self._real_video() + dataset = self._dataset() + row = dataset.data.iloc[0].copy() + original = dataset.save_video_frames(row) + self.assertEqual(dataset.save_video_frames(row), original) + key = dataset.clip_cache_key(row) + changed = row.copy() + changed['cut_frame'] = 5 + self.assertNotEqual(key, dataset.clip_cache_key(changed)) + dataset.nframe = 3 + self.assertNotEqual(key, dataset.clip_cache_key(row)) + dataset.nframe = 4 + with patch('vlmeval.dataset.perception_test.PROTOCOL', 'synthetic-next-protocol'): + self.assertNotEqual(key, dataset.clip_cache_key(row)) + self._real_video(red_after=0) + self.assertNotEqual(key, dataset.clip_cache_key(row)) + new_paths = dataset.save_video_frames(row) + self.assertNotEqual(original, new_paths) + with Image.open(new_paths[0]) as image: + red, green, blue = image.getpixel((300, 200)) + self.assertGreater(red, green + 100) + self.assertTrue(path.is_file()) + + def test_invalid_decoded_bounds_fail_instead_of_clamping(self): + for cut in (0, -2, 13, True, 1.5): + with self.subTest(cut=cut), self.assertRaises(ValueError): + PerceptionTestLocal.sample_indices(12, cut, 4) + self._real_video() + dataset = self._dataset() + row = dataset.data.iloc[0].copy() + row['cut_frame'] = 13 + with self.assertRaisesRegex(ValueError, 'cut'): + dataset.save_video_frames(row) + + def test_prompt_never_uses_answer_metadata(self): + self._real_video() + dataset = self._dataset() + row = dataset.data.iloc[0].copy() + row['answer'] = 'SECRET_LABEL_SENTINEL' + row['answer_id'] = 'SECRET_LABEL_SENTINEL' + prompt = dataset.build_prompt(row) + text = '\n'.join(m['value'] for m in prompt if m['type'] == 'text') + self.assertNotIn('SECRET_LABEL_SENTINEL', text) + self.assertIn('(A) red square', text) + self.assertIn('(B) blue circle', text) + self.assertIn('(C) green triangle', text) + + def test_safe_extraction_rejects_traversal_absolute_symlink_before_writing(self): + for i, name in enumerate(('../escape', '/absolute', 'videos/../../escape', 'C:\\escape')): + archive = self.root / f'bad-{i}.zip' + with zipfile.ZipFile(archive, 'w') as stream: + stream.writestr('valid.txt', 'must not be extracted') + stream.writestr(name, 'bad') + destination = self.root / f'extract-{i}' + with self.assertRaises(ValueError): + local._safe_extract(archive, destination) + self.assertFalse((destination / 'valid.txt').exists()) + archive = self.root / 'symlink.zip' + with zipfile.ZipFile(archive, 'w') as stream: + info = zipfile.ZipInfo('videos/link') + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + stream.writestr(info, '../../outside') + with self.assertRaisesRegex(ValueError, 'member type'): + local._safe_extract(archive, self.root / 'symlink-out') + + def test_existing_symlink_parent_and_duplicate_members_rejected(self): + outside = self.root / 'outside' + outside.mkdir() + destination = self.root / 'extract' + destination.mkdir() + (destination / 'videos').symlink_to(outside, target_is_directory=True) + archive = self.root / 'parent-link.zip' + with zipfile.ZipFile(archive, 'w') as stream: + stream.writestr('videos/video_8241.mp4', 'bad') + with self.assertRaisesRegex(ValueError, 'symlink'): + local._safe_extract(archive, destination) + self.assertEqual(list(outside.iterdir()), []) + archive = self.root / 'duplicate.zip' + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + with zipfile.ZipFile(archive, 'w') as stream: + stream.writestr('same.txt', 'first') + stream.writestr('same.txt', 'second') + with self.assertRaisesRegex(ValueError, 'Duplicate'): + local._safe_extract(archive, self.root / 'duplicate-out') + self.assertFalse((self.root / 'duplicate-out/same.txt').exists()) + + def test_download_verifies_generation_bytes_and_sha_before_install(self): + content = b'synthetic public pinned bytes' + digest = __import__('hashlib').sha256(content).hexdigest() + pin = {'object': 'synthetic', 'generation': '42', 'bytes': len(content), 'sha256': digest} + class Response(io.BytesIO): + headers = {'x-goog-generation': '42'} + path = self.root / 'download' + with patch.object(local.urllib.request, 'urlopen', return_value=Response(content)): + local._download(path, pin) + self.assertEqual(path.read_bytes(), content) + with self.assertRaisesRegex(ValueError, 'checksum'): + local._download(path, {**pin, 'sha256': '0' * 64}) + path.unlink() + with patch.object(local.urllib.request, 'urlopen', return_value=Response(content + b'corrupt')): + with self.assertRaisesRegex(ValueError, 'verification'): + local._download(path, pin) + self.assertFalse(path.exists()) + with patch.object(local.urllib.request, 'urlopen', return_value=Response(content)): + with self.assertRaisesRegex(ValueError, 'generation'): + local._download(path, {**pin, 'generation': '43'}) + self.assertFalse(path.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py index 9f8e713d5..0ef65a0bd 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/__init__.py @@ -125,7 +125,8 @@ from .moat import MOAT from .moviechat1k import MovieChat1k from .mssbench import MSSBenchDataset -from .mvbench import MVBench, MVBenchLocal, MVBench_MP4 +from .mvbench import MVBench, MVBench_MP4 +from .perception_test import PerceptionTestLocal from .mvpbench import MVPBench from .mvu_eval import MVUEval from .NPMM import NPMM @@ -363,7 +364,7 @@ def evaluate(self, eval_file, **judge_kwargs): ] VIDEO_DATASET = [ - MMBenchVideo, VideoMME, MVBench, MVBench_MP4, MVTamperBench, + MMBenchVideo, VideoMME, MVBench, MVBench_MP4, PerceptionTestLocal, MVTamperBench, LongVideoBench, WorldSense, VDC, MovieChat1k, MEGABench, MLVU, MLVU_MCQ, MLVU_OpenEnded, TempCompass, TempCompass_MCQ, TempCompass_Captioning, TempCompass_YorN, diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py index 778cec3a4..44b96041a 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/mvbench.py @@ -38,7 +38,8 @@ class MVBench(VideoBaseDataset): TYPE = 'Video-MCQ' DEFAULT_JUDGE = ['chatgpt-0125', 'gpt-4-0125'] - TASKS = { + def __init__(self, dataset='MVBench', nframe=0, fps=-1): + self.type_data_list = { 'Action Sequence': ('action_sequence.json', 'your_data_path/star/Charades_v1_480/', 'video', True), # has start & end 'Action Prediction': ('action_prediction.json', @@ -79,10 +80,7 @@ class MVBench(VideoBaseDataset): 'your_data_path/tvqa/frames_fps3_hq/', 'frame', True), # has start & end, read frame 'Counterfactual Inference': ('counterfactual_inference.json', 'your_data_path/clevrer/video_validation/', 'video', False), - } - - def __init__(self, dataset='MVBench', nframe=0, fps=-1): - self.type_data_list = self.TASKS + } super().__init__(dataset=dataset, nframe=nframe, fps=fps) @classmethod @@ -371,7 +369,7 @@ def build_prompt(self, line, video_llm): return message @classmethod - def evaluate(self, eval_file, *, trusted_data=None, **judge_kwargs): + def evaluate(self, eval_file, **judge_kwargs): assert get_file_extension(eval_file) in ['xlsx', 'json', 'tsv'], 'data file should be an supported format (xlsx/json/tsv) file' # noqa: E501 @@ -393,9 +391,7 @@ def evaluate(self, eval_file, *, trusted_data=None, **judge_kwargs): res = {} if not osp.exists(tmp_file) else load(tmp_file) res = {k: v for k, v in res.items() if FAIL_MSG not in v} - # Local scoring supplies an authenticated table directly so pandas - # cannot reinterpret literal model strings (e.g. "nan" or "1"). - data = load(eval_file) if trusted_data is None else trusted_data.copy() + data = load(eval_file) data_un = data[~pd.isna(data['prediction'])] for idx in data_un['index']: @@ -415,11 +411,6 @@ def evaluate(self, eval_file, *, trusted_data=None, **judge_kwargs): if FAIL_MSG in pred: data.loc[idx, 'score'] = -1 - elif trusted_data is not None and not any( - character.isalnum() for character in pred.lower().strip().split(' ')[0]): - # Punctuation alone is not an option: native substring - # matching otherwise grants credit for ".", "(" and ")". - data.loc[idx, 'score'] = 0 else: data.loc[idx, 'score'] = int(check_ans_with_model( pred, ans, model, @@ -441,113 +432,6 @@ def evaluate(self, eval_file, *, trusted_data=None, **judge_kwargs): dump(rating, tgt_file) return rating -class MVBenchLocal(MVBench): - """Pinned local MVBench, preserving native sampling and image prompts. - - Unlike the generic downloader, local preparation never switches revisions or - drops missing NTU rows. Frame caches are clip-specific rather than keyed by - video basename alone. Native video conversion is deliberately disabled: - Cosmos receives exactly nframe sampled images and the native MCQ prompt. - """ - - def __init__(self, dataset='MVBench', data_root=None, nframe=16, smoke=False, fps=-1): - from pathlib import Path - - if dataset != 'MVBench': - raise ValueError('MVBenchLocal supports dataset="MVBench" only') - if data_root is None: - raise ValueError('MVBenchLocal requires an explicit data_root') - if isinstance(nframe, bool) or not isinstance(nframe, int) or nframe <= 0: - raise ValueError('nframe must be a positive integer') - if fps > 0: - raise ValueError('MVBenchLocal uses native fixed-frame sampling, not fps sampling') - self._local_root = Path(data_root).expanduser().resolve() - self._local_smoke = bool(smoke) - super().__init__(dataset=dataset, nframe=nframe, fps=fps) - self.frame_root = str(self._local_root / '.mvbench' / 'frames') - os.makedirs(self.frame_root, exist_ok=True) - - def prepare_dataset(self, dataset_name='MVBench', **kwargs): - from vlmeval.mvbench_local import FRAME_RESOLUTION, prepare_dataset - from .utils.mvbench import GroupCenterCrop, GroupScale - - prepared = prepare_dataset(self._local_root, smoke=self._local_smoke) - self.local_provenance = prepared - self.decord_method = { - 'video': self.read_video, 'gif': self.read_gif, 'frame': self.read_frame, - } - self.frame_fps = 3 - # Reference sampling/cropping, but not VideoChat2 tensor normalization: - # the Cosmos server owns its model-specific image normalization. - self.transform = T.Compose([ - GroupScale(FRAME_RESOLUTION, interpolation=Image.BICUBIC), - GroupCenterCrop(FRAME_RESOLUTION), - Stack(), ToTorchFormatTensor(), - ]) - return prepared - - def qa_template(self, data): - import ast - - candidates = data['candidates'] - if isinstance(candidates, str): - candidates = ast.literal_eval(candidates) - if (not isinstance(candidates, list) - or not all(isinstance(option, str) for option in candidates) - or candidates.count(data['answer']) != 1): - raise ValueError('MVBench question has invalid candidates or answer') - question = f"Question: {data['question']}\nOptions:\n" - question += '\n'.join(f'({chr(ord("A") + index)}) {option}' - for index, option in enumerate(candidates)) - answer_index = candidates.index(data['answer']) - return question.rstrip(), f'({chr(ord("A") + answer_index)}) {data["answer"]}' - - def clip_cache_key(self, line): - import hashlib - from vlmeval.mvbench_local import ( - DATASET_REVISION, FRAME_PREPROCESSING, FRAME_RESOLUTION, _media_content_sha256, - ) - - identity = { - 'dataset_revision': DATASET_REVISION, - 'dataset_root': str(self._local_root), - 'media': str(os.path.join(line['prefix'], line['video'])), - 'data_type': line['data_type'], - 'bounds': [float(line['start']), float(line['end'])] if line['bound'] else None, - 'nframe': self.nframe, - 'sampling': 'native-MVBench.get_index-v1', - 'resolution': FRAME_RESOLUTION, 'preprocessing': FRAME_PREPROCESSING, - 'media_sha256': _media_content_sha256(self._local_root, line), - } - return hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() - - def save_video_into_images(self, line): - key = self.clip_cache_key(line) - frame_paths = self.frame_paths(key) - if all(osp.isfile(path) and osp.getsize(path) > 0 for path in frame_paths): - return frame_paths - bound = (line['start'], line['end']) if line['bound'] else None - video_path = osp.join(self.data_root, line['prefix'], line['video']) - self.num_segments = self.nframe - images = self.decord_method[line['data_type']](video_path, bound) - return self.save_video_frames(images, key, self.nframe) - - def build_prompt(self, line, video_llm=False): - return super().build_prompt(line, video_llm=False) - - def evaluate(self, eval_file, **judge_kwargs): - from pathlib import Path - from vlmeval.mvbench_local import score_predictions - - if judge_kwargs.get('model', 'exact_matching') != 'exact_matching': - raise ValueError('MVBenchLocal only permits the network-free exact_matching judge') - path = Path(eval_file) - return score_predictions( - path, self._local_root, path.parent / f'{path.stem}_mvbench', - smoke=self._local_smoke, nframe=self.nframe, - ) - - class MVBench_MP4(VideoBaseDataset): DEFAULT_JUDGE = ['chatgpt-0125', 'gpt-4-0125'] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/perception_test.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/perception_test.py new file mode 100644 index 000000000..427be52bc --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/dataset/perception_test.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Native video harness integration for public Perception Test MCQ validation.""" +import json +from numbers import Integral +import os +from pathlib import Path +import tempfile + +from PIL import Image +import portalocker + +from vlmeval.perception_test_local import ( + DATASET_REVISION, FRAME_PREPROCESSING, MAX_FRAME_SIDE, PROTOCOL, SAMPLING, + _fingerprint, _inside, _integer, _media_content_sha256, prepare_dataset, score_predictions, +) +from .video_base import VideoBaseDataset + + +class PerceptionTestLocal(VideoBaseDataset): + TYPE = 'Video-MCQ' + DEFAULT_JUDGE = 'exact_matching' + + def __init__(self, dataset='PerceptionTest', data_root=None, nframe=16, smoke=False, + fps=-1, pack=False, total_pixels=None, **kwargs): + import pandas as pd + + if dataset != 'PerceptionTest': + raise ValueError('PerceptionTestLocal supports dataset="PerceptionTest" only') + _integer(nframe, 'nframe', 1) + if fps > 0 or pack or total_pixels is not None: + raise ValueError('PerceptionTestLocal requires fixed-frame unpacked images with maxside768 preprocessing') + root = data_root or os.environ.get('PERCEPTION_TEST_DATA_DIR') + if not root: + raise ValueError('Set PERCEPTION_TEST_DATA_DIR or provide data_root; fetch the public dataset first') + self._local_root = Path(root).expanduser().resolve() + self._local_smoke = bool(smoke) + super().__init__(dataset=dataset, nframe=nframe, fps=fps, **kwargs) + # The generic loader recognizes strings such as \"NA\" as missing values; + # official option text must survive native inference/export unchanged. + self.data = pd.read_csv(self.data_file, sep='\t', keep_default_na=False) + self.frame_root = str(self._local_root / '.perception-test' / 'frames') + Path(self.frame_root).mkdir(parents=True, exist_ok=True) + + @classmethod + def supported_datasets(cls): + return ['PerceptionTest'] + + def prepare_dataset(self, dataset='PerceptionTest'): + self.local_provenance = prepare_dataset(self._local_root, smoke=self._local_smoke, require_media=True) + return self.local_provenance + + @staticmethod + def _bounds(line): + cut, declared = line['cut_frame'], line['num_frames'] + if (isinstance(cut, bool) or not isinstance(cut, Integral) + or isinstance(declared, bool) or not isinstance(declared, Integral) + or declared <= 0 or (cut != -1 and not 0 < cut <= declared)): + raise ValueError(f'Invalid exclusive cut {cut!r} for declared {declared!r} frames; never clamp') + return int(cut), int(declared) + + def clip_cache_key(self, line): + cut, declared = self._bounds(line) + return _fingerprint({ + 'dataset_revision': DATASET_REVISION, 'media': line['video'], + 'media_sha256': _media_content_sha256(self._local_root, line), + 'cut_frame': cut, 'metadata_num_frames': declared, + 'nframe': self.nframe, 'protocol': PROTOCOL, 'sampling': SAMPLING, + 'max_frame_side': MAX_FRAME_SIDE, 'preprocessing': FRAME_PREPROCESSING, + }) + + @staticmethod + def sample_indices(total_frames, cut_frame, nframe): + _integer(total_frames, 'decoded frame count', 1) + _integer(nframe, 'nframe', 1) + if (isinstance(cut_frame, bool) or not isinstance(cut_frame, int) + or (cut_frame != -1 and not 0 < cut_frame <= total_frames)): + raise ValueError(f'Invalid exclusive frame cut {cut_frame!r} for {total_frames} decoded frames; never clamp') + stop = total_frames if cut_frame == -1 else cut_frame + return [(2 * i + 1) * stop // (2 * nframe) for i in range(nframe)] + + def save_video_frames(self, line): + """Decode only allowed source indices; cache content+cut+protocol identity.""" + import cv2 + + if isinstance(line, str): + matches = self.data[self.data['video'] == line] + if matches.empty: + raise ValueError(f'Unknown canonical video: {line}') + line = matches.iloc[0] + key = self.clip_cache_key(line) + directory = Path(self.frame_root) / key + paths = [directory / f'frame-{i + 1}-of-{self.nframe}.jpg' for i in range(self.nframe)] + directory.mkdir(parents=True, exist_ok=True) + with portalocker.Lock(str(directory) + '.lock', 'w', timeout=300): + if all(path.is_file() and path.stat().st_size > 0 for path in paths): + return [str(path) for path in paths] + source = _inside(self._local_root, line['video']) + reader = cv2.VideoCapture(str(source)) + try: + if not reader.isOpened(): + raise ValueError(f'Cannot decode Perception Test media: {source}') + count = int(reader.get(cv2.CAP_PROP_FRAME_COUNT)) + cut, declared = self._bounds(line) + indices = self.sample_indices(count, cut, self.nframe) + for path, index in zip(paths, indices): + if not reader.set(cv2.CAP_PROP_POS_FRAMES, index): + raise ValueError(f'Cannot seek source frame {index}: {source}') + ok, frame = reader.read() + if not ok or frame is None: + raise ValueError(f'Cannot decode required source frame {index}: {source}') + image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + image.thumbnail((MAX_FRAME_SIDE, MAX_FRAME_SIDE), Image.Resampling.LANCZOS) + with tempfile.NamedTemporaryFile(dir=directory, suffix='.jpg', delete=False) as stream: + temporary = Path(stream.name) + try: + image.save(temporary, format='JPEG') + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + finally: + reader.release() + if self.clip_cache_key(line) != key: + raise ValueError(f'Media changed during frame extraction: {source}; retry preparation') + return [str(path) for path in paths] + + def build_prompt(self, line, video_llm=False): + # VIDEO_LLM=True must not switch to raw video or bypass the official cut. + if isinstance(line, int): + line = self.data.iloc[line] + options = line['options'] + options = json.loads(options) if isinstance(options, str) else options + if not isinstance(options, list) or len(options) != 3 or not all(isinstance(o, str) for o in options): + raise ValueError('Perception Test prompt requires three ordered options') + text = ('Watch the sampled video frames and choose the best answer.\n' + f'Question: {line["question"]}\nOptions:\n' + + '\n'.join(f'({chr(65 + i)}) {option}' for i, option in enumerate(options)) + + '\nRespond with exactly one option letter: A, B, or C.') + return ([{'type': 'image', 'value': path} for path in self.save_video_frames(line)] + + [{'type': 'text', 'value': text}]) + + def evaluate(self, eval_file, **judge_kwargs): + if judge_kwargs.get('model', 'exact_matching') != 'exact_matching': + raise ValueError('PerceptionTestLocal uses local option-ID top1 only; remote judges are not allowed') + path = Path(eval_file) + return score_predictions(path, self._local_root, path.parent / f'{path.stem}_perception_test', + smoke=self._local_smoke, nframe=self.nframe) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py deleted file mode 100644 index 09964a65a..000000000 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/mvbench_local.py +++ /dev/null @@ -1,525 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Pinned, local MVBench inputs and native, network-free exact-match scoring. - -Video rights are not conferred by the dataset card's MIT metadata license. -The source card restricts videos to academic research and requires respecting -upstream agreements. NTU RGB+D acquisition is manual, never an old-revision or -mirror fallback. This module deliberately does not claim paper-score parity. -""" -import ast -import hashlib -import json -import math -import os -from pathlib import Path, PurePosixPath -import shutil -import stat -import tempfile -import warnings -import zipfile - -DATASET_REPO = 'OpenGVLab/MVBench' -DATASET_REVISION = '230a2d4fac8900333c61754641c7a13e069ac9c6' -EXPECTED_ROWS = 4000 -ROWS_PER_TASK = 200 -FRAME_RESOLUTION = 224 -FRAME_PREPROCESSING = 'GroupScale(224,bicubic)+GroupCenterCrop(224)+Stack(roll=False)+ToTorchFormatTensor(div=True)+JPEG' -SMOKE_TASK = 'Egocentric Navigation' -SMOKE_ROWS = 5 -RIGHTS_NOTICE = ( - 'MVBench video copyrights belong to their original creators/platforms; ' - 'the dataset card permits academic research use only and requires compliance ' - 'with all source agreements. Review the downloaded README.md before use. ' - 'The card describes 320 NTU RGB+D videos; this pinned revision lists 200 ' - 'required RGB AVI files for Fine-grained Pose. Obtain authorized access at ' - 'https://rose1.ntu.edu.sg/dataset/actionRecognition/ and use the exact pinned ' - 'video/MVBench_videos_ntu.txt list. Place authorized media in ' - '/video/nturgbd. No removed media are downloaded.' -) -# SHA-256 of the exact public bytes at DATASET_REVISION (not a user manifest). -METADATA_SHA256 = { - 'json/action_antonym.json': '0ebbec1f540f598a82bf27f569e7377fdc5c4b6e62d53d3847a88abb785f5513', - 'json/action_count.json': '5e3389d3f8a81744096825d735d45d04b1842fb49da7f5b9a70d4c2224505117', - 'json/action_localization.json': '164290bb5f447b6cdff26eb51a2470bcf4ce31f84d092ec4337f7bd4ce558eec', - 'json/action_prediction.json': '15601d39ca8604dbb642149c097f416518f4816b0c0bec57a6202d95f8bdac79', - 'json/action_sequence.json': 'd5f8c063a5ae5a433c3fb8ff78a03acbf0513a01928b384ed90a1e044adf230e', - 'json/character_order.json': '72c20ad5749da31759c92f4ed83ed117a52f70963ce1d85d5dfd1582accdead4', - 'json/counterfactual_inference.json': '6fabbbaba68b10b48bfb714c51b87569266bc65303530e5ab9c39cab2924bdfa', - 'json/egocentric_navigation.json': '4a5f80cca3471a15efe6d7d4ded192218871992cab32fa3a3eede93ea7fb3972', - 'json/episodic_reasoning.json': '39ca717f721c274a20835835e4ca417f58d5fb2d17ffebb7042d373561c871c3', - 'json/fine_grained_action.json': '21fb61328ab02e399170a22a2a4080a05dff18e8423de709e49b9fa2180fde58', - 'json/fine_grained_pose.json': 'f0cebc0a498474c03618aac20f67810cc41197ac97534a51f930c217dc13f091', - 'json/moving_attribute.json': 'b760214e4320b2ceed4580cc0febfc8c8b084487c7efcaef712b704adb850548', - 'json/moving_count.json': 'a561538a69b2722c96d2b608af488c04db6a0d803b03b49661b8834e9425d245', - 'json/moving_direction.json': '81379646d91fb078a8b38de897f40977eeeebd946ca1c6ef0f58e4bde9d21209', - 'json/object_existence.json': '2bb290258243c7dafe57042768d491e6a239d30d0ff74d0f163a989c60f69620', - 'json/object_interaction.json': '9624cf2fef5ad4ec882edf955ce1d10c806e71e4e8c6a880fd867b224914302b', - 'json/object_shuffle.json': 'f9e1bd17a9bbab1d9f5be71411ca619cf9743ca47bb808985b202138b7928118', - 'json/scene_transition.json': 'edab8a7992851061cac170f7e50782961ec21826f99e70ee72ae73a9dafb79f9', - 'json/state_change.json': '21fa00a1e4fdbd3a0fda4d04f1b4dfd8ee62a1e3f3663fdb0f6e40799c36027f', - 'json/unexpected_action.json': '76b6f440ccb1ed007c67236c0ec7a39ee7cfbfb04197e9cc4a303b9230ca958b', - 'README.md': '0e9520eb913e0825e8cff14eb7a9821a23587312ed2ee97cb86a4b2c2335ec99', - 'video/MVBench_videos_ntu.txt': '878766d10973e1c9e030fca19abb4417f2a8182803199724ef6a9b0b64ec906e', -} -ARCHIVE_SHA256 = { - 'video/FunQA_test.zip': 'd6ef954652c2a1b48fb772d52c97da35dc7f8b54e5bdf4b43ba4f773ed726723', - 'video/Moments_in_Time_Raw.zip': '16add0d03cbb7c6b76974551879f7dc8a7ce985b89699a2231136b81d3ba9967', - 'video/clevrer.zip': '9040ad663440611bd8580fc0fc3da31bd1a4bfef877631db7b570a7d7d41de1c', - 'video/data0613.zip': 'f045863b2ccf7f32ada437d72529fea1c026fc015a7df548f703c22b71a24962', - 'video/perception.zip': '98fc197a90c875fb24f34fe89cb91e2788e1e7ac2459bf860ed62d9f07104b34', - 'video/scene_qa.zip': '26b3dc385e5a8f7b8fd9aab32a3522488fdfe7bbe06f0d98206d43647c77cce9', - 'video/ssv2_video.zip': '8493fe3762bdc205b587c4d02e016f9ccbb314b523ed0bf2c5bdbe367456d1e9', - 'video/sta.zip': '4c27cbf62bbc1b0217b5183871f298e7a76b2a498dd4482365c617f25338de9a', - 'video/star.zip': 'bdb5f192a98f78e79e51da6ae801f8ba79d1d09c0d8c19b6e63105251cb17585', - 'video/tvqa.zip': '6c5bccf237164286123a7afcc1bd7e8323487d7f2fd0dc5f43609a34064f5d73', - 'video/vlnqa.zip': '578f9edf2973560ac0d79990d9a2961ccae8edcdb1cb0be2c3bf15045f04428a', -} -NATIVE_SCORER = 'VLMEvalKit MVBench.evaluate(model=exact_matching), vendored utils/mvbench.py' - - -class MVBenchCoverageError(ValueError): - """Incomplete media coverage; ``report`` gives exact missing paths by task.""" - - def __init__(self, message, report): - super().__init__(message) - self.report = report - - -def _sha256(path): - digest = hashlib.sha256() - with Path(path).open('rb') as stream: - for block in iter(lambda: stream.read(1024 * 1024), b''): - digest.update(block) - return digest.hexdigest() - - -def _json_write(path, value): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - # Atomic replacement avoids half-written canonical inputs when interrupted. - with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as stream: - temporary = Path(stream.name) - json.dump(value, stream, indent=2, allow_nan=False) - stream.write('\n') - os.replace(temporary, path) - - -def _inside(root, relative): - name = str(relative) - parts = PurePosixPath(name) - if not name or '\\' in name or '\x00' in name or ':' in name or parts.is_absolute() or '..' in parts.parts: - raise ValueError(f'Unsafe relative path: {name!r}') - target = Path(root).joinpath(*parts.parts) - if not target.resolve().is_relative_to(Path(root).resolve()): - raise ValueError(f'Path escapes dataset root: {name!r}') - return target - - -def _safe_extract(archive, destination, overlay=False): - """Extract regular ZIP members only; validate *all* names before writing. - - data0613 is the official June-2024 correction overlay. Strip that single - archive's data0613/ wrapper and overwrite base media, after base extraction. - """ - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(archive) as source: - members = [] - for info in source.infolist(): - _inside(destination, info.filename) - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode) or (stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR)): - raise ValueError(f'Unsupported ZIP member type: {info.filename!r}') - parts = PurePosixPath(info.filename).parts - if overlay: - if not parts or parts[0] != 'data0613': - raise ValueError(f'Unexpected correction-overlay path: {info.filename!r}') - parts = parts[1:] - if not parts: - continue - target = _inside(destination, PurePosixPath(*parts)) - if target.is_symlink(): - raise ValueError(f'Refusing ZIP symlink target: {target}') - members.append((info, target)) - for info, target in members: - if info.is_dir(): - target.mkdir(parents=True, exist_ok=True) - continue - target.parent.mkdir(parents=True, exist_ok=True) - with source.open(info) as reader, tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as writer: - temporary = Path(writer.name) - try: - shutil.copyfileobj(reader, writer) - except BaseException: - temporary.unlink(missing_ok=True) - raise - os.replace(temporary, target) - - -def fetch_dataset(root: Path, scope='smoke'): - """Download only approved pinned public inputs, without login or tokens. - - ``metadata`` gets all official annotations and rights notices. ``smoke`` - adds vlnqa.zip. ``full`` adds the current public archives (never NTU). - Fetch is not a coverage assertion: call prepare_dataset before inference. - """ - from huggingface_hub import hf_hub_download - - if scope not in ('metadata', 'smoke', 'full'): - raise ValueError('scope must be metadata, smoke, or full') - root = Path(root).expanduser().resolve() - root.mkdir(parents=True, exist_ok=True) - warnings.warn(RIGHTS_NOTICE, UserWarning, stacklevel=2) - archives = [] if scope == 'metadata' else ['video/vlnqa.zip'] - if scope == 'full': - archives = [name for name in ARCHIVE_SHA256 if name != 'video/data0613.zip'] + ['video/data0613.zip'] - requested = list(METADATA_SHA256) + archives - for name in requested: - _inside(root, name) - path = Path(hf_hub_download( - repo_id=DATASET_REPO, repo_type='dataset', revision=DATASET_REVISION, - filename=name, local_dir=str(root), token=False, - )) - expected = METADATA_SHA256.get(name, ARCHIVE_SHA256.get(name)) - if _sha256(path) != expected: - raise ValueError(f'Pinned file checksum mismatch: {name}. Remove the damaged local file and fetch again.') - if name in archives: - _safe_extract(path, root / 'video', overlay=name == 'video/data0613.zip') - notice = root / '.mvbench' / 'RIGHTS_NOTICE.txt' - notice.parent.mkdir(parents=True, exist_ok=True) - notice.write_text(RIGHTS_NOTICE + '\n', encoding='utf-8') - return { - 'root': str(root), 'dataset_revision': DATASET_REVISION, 'scope': scope, - 'downloaded_files': requested, 'rights_notice': RIGHTS_NOTICE, - 'manual_ntu_required_for_full': True, - } - - -def _task_definitions(): - from vlmeval.dataset.mvbench import MVBench - return MVBench.TASKS - - -def _read_annotations(root): - """Authenticate every source byte before deriving identities or labels.""" - for name, expected in METADATA_SHA256.items(): - path = _inside(root, name) - if not path.is_file(): - raise ValueError(f'Missing pinned metadata {name}; run fetch_dataset(root, scope="metadata").') - if _sha256(path) != expected: - raise ValueError(f'Pinned metadata checksum mismatch: {name}; local annotations are not trusted.') - rows = [] - for task, (name, prefix, data_type, bound) in _task_definitions().items(): - source = json.loads((root / 'json' / name).read_text(encoding='utf-8')) - if not isinstance(source, list) or len(source) != ROWS_PER_TASK: - raise ValueError(f'{name} must contain exactly {ROWS_PER_TASK} questions') - for position, item in enumerate(source): - candidates = item.get('candidates') - if (not isinstance(candidates, list) or not 1 <= len(candidates) <= 26 - or not all(isinstance(option, str) and option.strip() for option in candidates) - or len(set(candidates)) != len(candidates)): - raise ValueError(f'Invalid candidates in {name}:{position}') - answer = item.get('answer') - if not isinstance(answer, str) or candidates.count(answer) != 1: - raise ValueError(f'Answer must identify exactly one candidate in {name}:{position}') - if not isinstance(item.get('question'), str) or not item['question'].strip(): - raise ValueError(f'Missing question in {name}:{position}') - if not isinstance(item.get('video'), str) or not item['video'].strip(): - raise ValueError(f'Missing video in {name}:{position}') - media_prefix = prefix.replace('your_data_path', 'video') - _inside(root, str(PurePosixPath(media_prefix) / item['video'])) - start, end = item.get('start'), item.get('end') - if bound and (not all(isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) - for v in (start, end)) or start < 0 or end <= start): - raise ValueError(f'Invalid temporal bounds in {name}:{position}') - rows.append({ - 'index': len(rows), 'task_type': task, 'prefix': media_prefix, - 'data_type': data_type, 'bound': bound, 'start': start, 'end': end, - 'video': item['video'], 'question': item['question'], 'answer': answer, - # repr is exclusively of authenticated, validated string lists; - # upstream evaluate's eval never receives incoming candidates. - 'candidates': repr(candidates), - }) - if len(rows) != EXPECTED_ROWS or len(_task_definitions()) != 20: - raise ValueError('MVBench must contain 4000 questions across all 20 native tasks') - return rows - - -def _media_available(root, row): - path = _inside(root, str(PurePosixPath(row['prefix']) / row['video'])) - if row['data_type'] != 'frame': - return path.is_file() and path.stat().st_size > 0 - if not path.is_dir(): - return False - frames = sorted(path.glob('*.jpg')) - return bool(frames) and all( - frame.name == f'{position:05d}.jpg' and frame.stat().st_size > 0 - for position, frame in enumerate(frames, 1) - ) - - -_MEDIA_HASH_CACHE = {} - - -def _media_content_sha256(root, row): - """Hash media bytes, memoizing only while file identities/stats are stable.""" - path = _inside(root, str(PurePosixPath(row['prefix']) / row['video'])) - files = sorted(path.glob('*.jpg')) if row['data_type'] == 'frame' else [path] - if not files: - raise ValueError(f'Media has no frames: {path}') - - def signature(): - result = [] - for file in files: - _inside(root, file.relative_to(root).as_posix()) - info = file.stat() - result.append((file.name, info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)) - return tuple(result) - - before = signature() - key = str(path) - cached = _MEDIA_HASH_CACHE.get(key) - if cached is not None and cached[0] == before: - return cached[1] - if row['data_type'] == 'frame': - digest = hashlib.sha256() - for file in files: - digest.update(file.name.encode()) - digest.update(b'\0') - digest.update(bytes.fromhex(_sha256(file))) - value = digest.hexdigest() - else: - value = _sha256(path) - if signature() != before: - raise ValueError(f'Media changed while hashing: {path}') - _MEDIA_HASH_CACHE[key] = (before, value) - return value - - -def _manifest_sha256(): - payload = {'repo': DATASET_REPO, 'revision': DATASET_REVISION, 'metadata': METADATA_SHA256, - 'archives': ARCHIVE_SHA256, 'tasks': _task_definitions(), - 'task_order': list(_task_definitions())} - return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() - - -def prepare_dataset(root: Path, smoke=False): - """Rebuild canonical TSV from pinned annotations; never silently drop rows. - - Full requires all 4000 rows/media; smoke selects ONLY the first five official - Egocentric Navigation identities. Coverage always reports the full universe. - """ - import pandas as pd - - root = Path(root).expanduser().resolve() - rows = _read_annotations(root) - selected = [row for row in rows if row['task_type'] == SMOKE_TASK][:SMOKE_ROWS] if smoke else rows - available = {} - missing = {} - for row in rows: - key = (row['prefix'], row['video'], row['data_type']) - if key not in available: - available[key] = _media_available(root, row) - if not available[key]: - missing.setdefault(row['task_type'], set()).add(str(PurePosixPath(row['prefix']) / row['video'])) - coverage = { - 'expected_rows': EXPECTED_ROWS, - 'available_rows': sum(available[(r['prefix'], r['video'], r['data_type'])] for r in rows), - 'selected_rows': len(selected), 'expected_tasks': 20, - 'selected_tasks': list(dict.fromkeys(r['task_type'] for r in selected)), - 'subset': bool(smoke), - } - report = { - 'dataset_revision': DATASET_REVISION, 'coverage': coverage, - 'missing_media_by_task': {task: sorted(paths) for task, paths in missing.items()}, - 'rights_notice': RIGHTS_NOTICE, - } - scope = 'smoke' if smoke else 'full' - report_file = root / '.mvbench' / f'coverage-{scope}.json' - _json_write(report_file, report) - missing_selected = [r['index'] for r in selected if not available[(r['prefix'], r['video'], r['data_type'])]] - if missing_selected: - detail = ' Full evaluation requires all 4000 questions and all 20 tasks.' if not smoke else ' Smoke requires its fixed five navigation videos.' - raise MVBenchCoverageError( - f'MVBench media coverage incomplete: {coverage["available_rows"]}/4000 available; ' - f'{len(missing_selected)} selected rows lack usable media.{detail} ' - f'See {report_file}. {RIGHTS_NOTICE}', report, - ) - media_hashes = {} - for row in selected: - relative = str(PurePosixPath(row['prefix']) / row['video']) - if relative not in media_hashes: - media_hashes[relative] = _media_content_sha256(root, row) - annotation_manifest = _manifest_sha256() - media_manifest = hashlib.sha256(json.dumps(media_hashes, sort_keys=True).encode()).hexdigest() - manifest = hashlib.sha256(f'{annotation_manifest}:{media_manifest}'.encode()).hexdigest() - _json_write(root / '.mvbench' / f'media-manifest-{scope}.json', { - 'dataset_revision': DATASET_REVISION, 'sha256': media_manifest, 'media': media_hashes, - }) - data_file = root / '.mvbench' / f'MVBench-{scope}-{manifest[:16]}.tsv' - with tempfile.NamedTemporaryFile(mode='w', dir=data_file.parent, delete=False) as stream: - temporary = Path(stream.name) - pd.DataFrame(selected).to_csv(stream, sep='\t', index=False) - os.replace(temporary, data_file) - return { - 'root': str(root), 'data_file': str(data_file), 'dataset_revision': DATASET_REVISION, - 'manifest_sha256': manifest, 'coverage': coverage, - 'annotation_manifest_sha256': annotation_manifest, - 'media_manifest_sha256': media_manifest, 'media_sha256': media_hashes, - 'selected_indices': [row['index'] for row in selected], - } - - -def build_dataset(root: Path, nframe=16, smoke=False): - from vlmeval.dataset.mvbench import MVBenchLocal - return MVBenchLocal(data_root=str(root), nframe=nframe, smoke=smoke) - - -def _load_predictions(path): - import pandas as pd - - path = Path(path) - if path.suffix == '.tsv': - return pd.read_csv(path, sep='\t', keep_default_na=False, dtype={'prediction': str}) - if path.suffix == '.xlsx': - return pd.read_excel(path, keep_default_na=False, dtype={'prediction': str}) - if path.suffix == '.json': - return pd.DataFrame(json.loads(path.read_text(encoding='utf-8'))) - raise ValueError('Predictions must be TSV, XLSX, or JSON; pickle inputs are not accepted') - - -def _verified_predictions(incoming, trusted): - """Only strings cross the trust boundary; identities and labels are fixed.""" - if not {'index', 'prediction'}.issubset(incoming.columns): - raise ValueError('Predictions require index and prediction columns') - indices = [] - for value in incoming['index']: - try: - number = float(value) - except (TypeError, ValueError): - raise ValueError(f'Invalid prediction identity: {value!r}') from None - if isinstance(value, bool) or not math.isfinite(number) or number != int(number): - raise ValueError(f'Invalid prediction identity: {value!r}') - indices.append(int(number)) - expected = list(trusted['index']) - if len(indices) != len(set(indices)) or set(indices) != set(expected): - raise ValueError('Prediction coverage must match selected stable indices exactly: no missing, duplicate, or extra rows') - incoming = incoming.copy() - incoming['index'] = indices - incoming = incoming.set_index('index').loc[expected] - for position, row in trusted.reset_index(drop=True).iterrows(): - supplied = incoming.loc[row['index']] - for column in trusted.columns: - if column == 'index' or column not in incoming.columns: - continue - actual, wanted = supplied[column], row[column] - if column == 'candidates': - try: - actual = ast.literal_eval(actual) if isinstance(actual, str) else actual - wanted = ast.literal_eval(wanted) - except (ValueError, SyntaxError, TypeError): - raise ValueError(f'Untrusted candidates for index {row["index"]}') from None - elif column in ('start', 'end'): - if wanted is None or (isinstance(wanted, float) and math.isnan(wanted)): - if actual == '' or actual is None or (isinstance(actual, float) and math.isnan(actual)): - continue - try: - actual = float(actual) - except (ValueError, TypeError): - pass - elif column == 'bound' and isinstance(actual, str): - actual = {'True': True, 'False': False}.get(actual, actual) - if actual != wanted: - raise ValueError(f'Untrusted {column} for index {row["index"]}') - predictions = incoming['prediction'].tolist() - for index, prediction in zip(expected, predictions): - if (not isinstance(prediction, str) or not prediction.strip() - or 'failed to obtain answer' in prediction.lower() - or 'video_load_failed' in prediction.lower()): - raise ValueError(f'Missing or failed prediction for index {index}; inference must complete before scoring') - return predictions - - -def _summarize_rating(rating): - per_task = {} - for task, counts in rating.items(): - if task == 'overall': - continue - correct, total = counts[:2] - if not (0 <= correct <= total and total > 0): - raise ValueError(f'Invalid native score counts for {task}') - per_task[task] = {'correct': int(correct), 'total': int(total), 'accuracy': correct / total} - correct = sum(v['correct'] for v in per_task.values()) - total = sum(v['total'] for v in per_task.values()) - if not total or list(rating['overall'][:2]) != [correct, total]: - raise ValueError('Native overall counts disagree with task counts') - return { - 'primary_metric': {'name': 'accuracy', 'value': correct / total, 'unit': 'fraction', - 'correct': correct, 'total': total}, - 'task_macro_accuracy': sum(v['accuracy'] for v in per_task.values()) / len(per_task), - 'per_task': per_task, - } - - -def score_predictions(predictions: Path, root: Path, output_dir: Path, smoke=False, nframe=16): - """Native MVBench exact matching on authenticated labels, with no judge. - - Missing/failure predictions are rejected before native rating sees them: - upstream get_dimension_rating treats -1 as truthy. Malformed nonempty - strings still receive upstream wrong-answer semantics, not a fallback score. - Native pandas row positions are made contiguous for its .loc assignment; - original stable identities are restored in the saved scored_predictions.tsv. - """ - import pandas as pd - from vlmeval.dataset.mvbench import MVBench - from vlmeval.smp import get_intermediate_file_path, load - - if isinstance(nframe, bool) or not isinstance(nframe, int) or nframe <= 0: - raise ValueError('nframe must be a positive integer') - output_dir = Path(output_dir).expanduser().resolve() - if output_dir.exists(): - raise ValueError('Scoring requires a fresh output directory') - prepared = prepare_dataset(root, smoke=smoke) - trusted = pd.read_csv(prepared['data_file'], sep='\t') - trusted['prediction'] = _verified_predictions(_load_predictions(predictions), trusted) - official_indices = trusted['index'].tolist() - trusted['index'] = range(len(trusted)) - output_dir.mkdir(parents=True, exist_ok=False) - # Never consume a stale native _score cache supplied alongside predictions. - with tempfile.TemporaryDirectory(prefix='mvbench-native-', dir=output_dir) as temporary: - native_file = Path(temporary) / 'trusted.tsv' - trusted.to_csv(native_file, sep='\t', index=False) - rating = MVBench.evaluate(str(native_file), trusted_data=trusted, model='exact_matching') - scored = load(get_intermediate_file_path(str(native_file), '_score')) - if len(scored) != len(trusted) or not scored['score'].isin([0, 1]).all(): - raise ValueError('Native scorer returned missing, failed, or nonbinary scores; no result published') - scored['index'] = official_indices - scored.to_csv(output_dir / 'scored_predictions.tsv', sep='\t', index=False) - summary = { - 'status': 'smoke_only' if smoke else 'complete', 'dataset': 'MVBench', - 'coverage': prepared['coverage'], **_summarize_rating(rating), - 'native_rating': rating, - 'provenance': { - 'dataset_repo': DATASET_REPO, 'dataset_revision': DATASET_REVISION, - 'manifest_sha256': prepared['manifest_sha256'], 'selected_indices': official_indices, - 'annotation_manifest_sha256': prepared['annotation_manifest_sha256'], - 'media_manifest_sha256': prepared['media_manifest_sha256'], - 'media_sha256': prepared['media_sha256'], - 'prediction_sha256': _sha256(predictions), 'scorer': NATIVE_SCORER, - 'scorer_sha256': _sha256(Path(__file__).parent / 'dataset' / 'mvbench.py'), - 'scorer_utils_sha256': _sha256(Path(__file__).parent / 'dataset' / 'utils' / 'mvbench.py'), - 'judge': 'exact_matching', 'network_judge': False, 'nframe': nframe, - 'local_scoring_guards': [ - 'reject missing/failure predictions before native rating', - 'punctuation-only first option token scores zero, never substring credit', - ], - 'prompt_mode': 'native sampled images', 'rights_notice': RIGHTS_NOTICE, - 'frame_resolution': FRAME_RESOLUTION, 'frame_preprocessing': FRAME_PREPROCESSING, - 'reference_recipe': {'nframe': 16, 'resolution': 224}, - 'model_normalization': 'Cosmos serving model, not VideoChat2 tensor normalization', - 'paper_parity': False, - }, - } - _json_write(output_dir / 'summary.json', summary) - return summary diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/perception_test_local.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/perception_test_local.py new file mode 100644 index 000000000..6d43d8bcc --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/perception_test_local.py @@ -0,0 +1,494 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Public Perception Test validation inputs and offline MCQ top-1 scoring. + +The metric is official option-ID accuracy. The conservative text-to-option +adapter below is a launchable protocol, not an official letter-parser claim. +Images use a prototype 16-frame/aspect-preserving 768-maxside protocol, not a +published Cosmos3-Nano score recipe. Labels never enter inference prompts. +""" +import csv +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import tempfile +import urllib.request +import warnings +import zipfile + +DATASET_REPO = 'https://github.com/google-deepmind/perception_test' +DATASET_REVISION = '3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf' +EXPECTED_ROWS = 19140 +EXPECTED_VIDEOS = 5260 +SMOKE_VIDEO = 'video_8241' +SMOKE_INDICES = [13404, 13405, 13406, 13407, 13408] +MAX_FRAME_SIDE = 768 +PROTOCOL = 'perception-test-local-images-v1' +FRAME_PREPROCESSING = 'aspect-preserving downscale only; no crop' +SAMPLING = 'uniform segment midpoints in [0, exclusive_stop); duplicate samples allowed for short clips' +RIGHTS_NOTICE = ( + 'Perception Test by DeepMind Technologies Limited (2022), Patraucean et al. (NeurIPS 2023). ' + 'Software: Apache-2.0; other materials: CC BY 4.0 ' + '(https://creativecommons.org/licenses/by/4.0/). Retain attribution and consult the pinned README.md. ' + 'https://github.com/google-deepmind/perception_test' +) +METADATA_SHA256 = { + 'mc_question_valid.json': '174d3fa7ffa0958ae1d61311f85747dd5d0a9ae3d1e21c504dd4a47e723f78c3', + 'cut_frame_mapping_valid.json': '3cd9ae2d56f28ba70fa80ccca04aca6481ebab1bf1099f3aabe92afad89a67f7', + 'README.md': '9f739a07616e3f60af0723800d813ca87953496ba9e288fc62b4bfd7578afa85', +} +PUBLIC_ASSETS = { + 'downloads/mc_question_valid_annotations.zip': { + 'object': 'zip_data/mc_question_valid_annotations.zip', 'generation': '1686575054970958', + 'bytes': 204076, 'sha256': '3f3ebb388b042e35d5c07fa6e2404d78feb63e4e477740948fc92dfc83c480ad'}, + 'cut_frame_mapping_valid.json': { + 'object': 'misc/cut_frame_mapping_valid.json', 'generation': '1692694363800279', + 'bytes': 106874, 'sha256': METADATA_SHA256['cut_frame_mapping_valid.json']}, + 'README.md': { + 'url': f'https://raw.githubusercontent.com/google-deepmind/perception_test/{DATASET_REVISION}/README.md', + 'bytes': 30386, 'sha256': METADATA_SHA256['README.md']}, + 'downloads/sample_videos.zip': { + 'object': 'zip_data/sample_videos.zip', 'generation': '1685097626197809', + 'bytes': 225370639, 'sha256': '68e5cca8c3064859f273ec7a52c15d5eb4a337032d12e6d2f379079825a72537'}, + 'downloads/valid_videos.zip': { + 'object': 'zip_data/valid_videos.zip', 'generation': '1685479088600941', + 'bytes': 75375143328, 'sha256': '553c06c6f525af4d8ebb0c31b2868e21ed517ffa239f43aef3b82ed3793cd377'}, +} +_MEDIA_HASH_CACHE = {} + + +class PerceptionTestCoverageError(ValueError): + """Selected media are incomplete; report contains every missing video.""" + + def __init__(self, message, report): + super().__init__(message) + self.report = report + + +def _sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def _fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), + ensure_ascii=False, allow_nan=False).encode()).hexdigest() + + +def _json_write(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=path.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, indent=2, ensure_ascii=False, allow_nan=False) + stream.write('\n') + os.replace(temporary, path) + + +def _inside(root, relative): + name = str(relative) + parts = PurePosixPath(name) + if (not name or '\\' in name or '\x00' in name or ':' in name + or parts.is_absolute() or '..' in parts.parts): + raise ValueError(f'Unsafe relative path: {name!r}') + root = Path(root).resolve() + target = root.joinpath(*parts.parts) + current = target + while current != root: + if current.is_symlink(): + raise ValueError(f'Refusing symlink dataset path: {current}') + current = current.parent + if not target.resolve().is_relative_to(root): + raise ValueError(f'Path escapes dataset root: {name!r}') + return target + + +def _safe_extract(archive, destination, allowed=None): + """Validate every ZIP member before extracting regular files atomically.""" + destination = Path(destination).resolve() + destination.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive) as source: + members, seen = [], set() + for info in source.infolist(): + target = _inside(destination, info.filename) + mode = info.external_attr >> 16 + if stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR): + raise ValueError(f'Unsupported ZIP member type: {info.filename!r}') + name = PurePosixPath(info.filename).as_posix().rstrip('/') + if name in seen: + raise ValueError(f'Duplicate ZIP member: {name!r}') + seen.add(name) + if allowed is not None and not allowed(name, info.is_dir()): + raise ValueError(f'Unexpected ZIP member: {name!r}') + members.append((info, target)) + for info, target in members: + if info.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with source.open(info) as reader, tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as writer: + temporary = Path(writer.name) + try: + shutil.copyfileobj(reader, writer, length=8 * 1024 * 1024) + except BaseException: + temporary.unlink(missing_ok=True) + raise + os.replace(temporary, target) + + +def _download(path, pin): + if path.exists(): + if path.stat().st_size != pin['bytes'] or _sha256(path) != pin['sha256']: + raise ValueError(f'Pinned file checksum/size mismatch: {path}; remove damaged file and fetch again') + return + path.parent.mkdir(parents=True, exist_ok=True) + url = pin.get('url') or ('https://storage.googleapis.com/dm-perception-test/' + + pin['object'] + '?generation=' + pin['generation']) + temporary = None + try: + with urllib.request.urlopen(url, timeout=180) as reader, tempfile.NamedTemporaryFile( + dir=path.parent, delete=False) as writer: + temporary = Path(writer.name) + generation = reader.headers.get('x-goog-generation') + if pin.get('generation') and generation != pin['generation']: + raise ValueError(f'Public object generation mismatch: {url}') + digest, count = hashlib.sha256(), 0 + while block := reader.read(8 * 1024 * 1024): + writer.write(block) + digest.update(block) + count += len(block) + if count != pin['bytes'] or digest.hexdigest() != pin['sha256']: + raise ValueError(f'Public download failed pinned SHA-256/size verification: {url}') + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def fetch_dataset(root, scope='smoke'): + """Fetch generation/SHA-pinned public metadata, smoke sample, or full videos. + + No authentication, mirrors, internal hosts, or private dataset prerequisites. + Full requires sufficient storage for both the 75.4GB ZIP and extracted media. + """ + if scope not in ('metadata', 'smoke', 'full'): + raise ValueError('scope must be metadata, smoke, or full') + root = Path(root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + warnings.warn(RIGHTS_NOTICE, UserWarning, stacklevel=2) + requested = ['downloads/mc_question_valid_annotations.zip', 'cut_frame_mapping_valid.json', 'README.md'] + if scope != 'metadata': + requested.append('downloads/sample_videos.zip' if scope == 'smoke' else 'downloads/valid_videos.zip') + for name in requested: + pin = PUBLIC_ASSETS[name] + path = _inside(root, name) + _download(path, pin) + if name.endswith('_annotations.zip'): + _safe_extract(path, root, lambda n, directory: n == 'mc_question_valid.json' and not directory) + elif name.endswith('_videos.zip'): + _safe_extract(path, root, lambda n, directory: (directory and n == 'videos') or + (not directory and re.fullmatch(r'videos/video_\d+\.mp4', n) is not None)) + _read_annotations(root) + notice = root / '.perception-test' / 'RIGHTS_NOTICE.txt' + notice.parent.mkdir(parents=True, exist_ok=True) + notice.write_text(RIGHTS_NOTICE + '\n', encoding='utf-8') + return {'root': str(root), 'dataset_revision': DATASET_REVISION, 'scope': scope, + 'downloaded_files': requested, 'rights_notice': RIGHTS_NOTICE, + 'public_assets': {name: PUBLIC_ASSETS[name] for name in requested}} + + +def _integer(value, name, minimum=0): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f'{name} must be an integer >= {minimum}: {value!r}') + return value + + +def _read_annotations(root): + """Authenticate all label, option, identity, cutoff and rights source bytes.""" + for name, expected in METADATA_SHA256.items(): + path = _inside(root, name) + if not path.is_file(): + raise ValueError(f'Missing pinned metadata {name}; run fetch_dataset(root, scope="metadata")') + if _sha256(path) != expected: + raise ValueError(f'Pinned metadata checksum mismatch: {name}; annotations are not trusted') + annotations = json.loads((root / 'mc_question_valid.json').read_text(encoding='utf-8')) + cuts = json.loads((root / 'cut_frame_mapping_valid.json').read_text(encoding='utf-8')) + if len(annotations) != EXPECTED_VIDEOS or any(re.fullmatch(r'video_\d+', v) is None for v in annotations): + raise ValueError(f'Expected exactly {EXPECTED_VIDEOS} canonical video identities') + rows = [] + for video in sorted(annotations, key=lambda v: int(v.split('_')[1])): + source = annotations[video] + meta = source['metadata'] + num_frames = _integer(meta['num_frames'], f'{video} num_frames', 1) + if meta['video_id'] != video or meta['split'] != 'valid': + raise ValueError(f'Metadata identity/split mismatch: {video}') + cut = cuts.get(video) + if isinstance(cut, bool) or not isinstance(cut, int) or (cut != -1 and not 0 < cut <= num_frames): + raise ValueError(f'Missing or invalid exclusive frame cut: {video}: {cut!r}') + questions = sorted(source['mc_question'], key=lambda q: q['id']) + if [q['id'] for q in questions] != list(range(len(questions))) or not questions: + raise ValueError(f'Invalid or duplicate question identities: {video}') + for question in questions: + _integer(question['id'], f'{video} question ID') + options = question['options'] + answer = _integer(question['answer_id'], 'answer_id') + if (not isinstance(options, list) or len(options) != 3 + or not all(isinstance(o, str) and o.strip() for o in options) or answer >= 3): + raise ValueError(f'Invalid official MCQ options/answer ID: {video}/{question["id"]}') + tags = question['tag'] + if (not isinstance(tags, list) or not all(isinstance(t, str) and t for t in tags) + or len(tags) != len(set(tags))): + raise ValueError(f'Invalid diagnostic tags: {video}/{question["id"]}') + for field in ('question', 'area', 'reasoning'): + if not isinstance(question[field], str) or not question[field].strip(): + raise ValueError(f'Invalid {field}: {video}/{question["id"]}') + rows.append({'index': len(rows), 'video_id': video, 'question_id': question['id'], + 'video': f'videos/{video}.mp4', 'question': question['question'], + 'options': json.dumps(options, ensure_ascii=False), 'answer_id': answer, + 'answer': chr(65 + answer), 'A': options[0], 'B': options[1], 'C': options[2], + 'area': question['area'], 'reasoning': question['reasoning'], + 'tag': json.dumps(tags, ensure_ascii=False), 'cut_frame': cut, 'num_frames': num_frames}) + if len(rows) != EXPECTED_ROWS: + raise ValueError(f'Expected exactly {EXPECTED_ROWS} official questions, found {len(rows)}') + return rows + + +def _selected(rows, smoke): + selected = [r for r in rows if r['video_id'] == SMOKE_VIDEO and r['question_id'] in range(5)] if smoke else rows + if smoke and [r['index'] for r in selected] != SMOKE_INDICES: + raise ValueError('Pinned smoke identities do not match canonical stable indices') + return selected + + +def _media_content_sha256(root, row): + path = _inside(root, row['video']) + def signature(): + info = path.stat() + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns + before = signature() + cached = _MEDIA_HASH_CACHE.get(str(path)) + if cached is not None and cached[0] == before: + return cached[1] + value = _sha256(path) + if signature() != before: + raise ValueError(f'Media changed while hashing: {path}') + _MEDIA_HASH_CACHE[str(path)] = before, value + return value + + +def prepare_dataset(root, smoke=False, require_media=True): + """Rebuild trusted TSV; metadata/scope identity never depends on media. + + Media-free scoring skips content hashing; no unobserved media identity is + inferred. Coverage counts present nonempty files, not decoder validation. + """ + root = Path(root).expanduser().resolve() + rows = _read_annotations(root) + selected = _selected(rows, smoke) + available = {} + for row in rows: + if row['video'] not in available: + path = _inside(root, row['video']) + available[row['video']] = path.is_file() and path.stat().st_size > 0 + missing = sorted(video for video, present in available.items() if not present) + coverage = {'expected_rows': EXPECTED_ROWS, 'expected_videos': EXPECTED_VIDEOS, + 'available_rows': sum(available[r['video']] for r in rows), + 'available_videos': sum(available.values()), 'selected_rows': len(selected), + 'selected_videos': len({r['video_id'] for r in selected}), + 'selected_available_rows': sum(available[r['video']] for r in selected), 'subset': bool(smoke)} + scope = 'smoke' if smoke else 'full' + annotation_manifest = _fingerprint({'repo': DATASET_REPO, 'revision': DATASET_REVISION, + 'metadata_sha256': METADATA_SHA256, 'canonical_rows': rows}) + manifest = _fingerprint({'annotation_manifest_sha256': annotation_manifest, + 'scope': scope, 'selected_indices': [r['index'] for r in selected]}) + report = {'dataset_revision': DATASET_REVISION, 'coverage': coverage, 'missing_media': missing, + 'manifest_sha256': manifest, 'rights_notice': RIGHTS_NOTICE} + state = root / '.perception-test' + _json_write(state / f'coverage-{scope}.json', report) + if require_media and any(not available[r['video']] for r in selected): + raise PerceptionTestCoverageError( + f'Perception Test media incomplete: {coverage["selected_available_rows"]}/{len(selected)} selected ' + f'questions have media. Full requires all {EXPECTED_ROWS} questions/{EXPECTED_VIDEOS} videos; ' + f'run fetch_dataset(root, scope="{scope}"). See {state / ("coverage-" + scope + ".json")}', report) + media = {} + if require_media: + for row in selected: + if row['video'] not in media: + media[row['video']] = _media_content_sha256(root, row) + media_manifest = _fingerprint(media) if require_media else None + data_file = state / f'PerceptionTest-{scope}-{manifest[:16]}.tsv' + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=state, delete=False, newline='') as stream: + temporary = Path(stream.name) + writer = csv.DictWriter(stream, fieldnames=list(selected[0]), delimiter='\t') + writer.writeheader() + writer.writerows(selected) + os.replace(temporary, data_file) + if require_media: + _json_write(state / f'media-manifest-{scope}.json', {'sha256': media_manifest, 'media': media}) + return {'root': str(root), 'data_file': str(data_file), 'dataset_revision': DATASET_REVISION, + 'manifest_sha256': manifest, 'annotation_manifest_sha256': annotation_manifest, + 'coverage': coverage, 'selected_indices': [r['index'] for r in selected], + 'media_sha256': media, 'media_manifest_sha256': media_manifest} + + +def build_dataset(root, nframe=16, smoke=False): + from vlmeval.dataset.perception_test import PerceptionTestLocal + return PerceptionTestLocal(data_root=str(root), nframe=nframe, smoke=smoke) + + +def _load_predictions(path): + import pandas as pd + path = Path(path) + if path.suffix.lower() == '.tsv': + return pd.read_csv(path, sep='\t', keep_default_na=False, dtype={'prediction': str}) + if path.suffix.lower() == '.xlsx': + return pd.read_excel(path, keep_default_na=False, dtype={'prediction': str}) + if path.suffix.lower() == '.json': + return pd.DataFrame(json.loads(path.read_text(encoding='utf-8'))) + raise ValueError('Predictions must be TSV, XLSX or JSON; eval and pickle inputs are not accepted') + + +def _verified_predictions(incoming, trusted): + if not {'index', 'prediction'}.issubset(incoming.columns) or incoming.columns.duplicated().any(): + raise ValueError('Predictions require unique index and prediction columns') + indices = [] + for value in incoming['index']: + try: + numeric = float(value) + except (TypeError, ValueError): + raise ValueError(f'Invalid prediction identity: {value!r}') from None + if isinstance(value, bool) or not math.isfinite(numeric) or numeric != int(numeric): + raise ValueError(f'Invalid prediction identity: {value!r}') + indices.append(int(numeric)) + expected = [r['index'] for r in trusted] + if len(indices) != len(set(indices)) or set(indices) != set(expected): + raise ValueError('Prediction coverage must match selected stable indices exactly: no missing, duplicate, or extra rows') + incoming = incoming.copy() + incoming['index'] = indices + incoming = incoming.set_index('index').loc[expected] + predictions = [] + for row in trusted: + supplied = incoming.loc[row['index']] + for column, wanted in row.items(): + if column == 'index' or column not in incoming.columns: + continue + actual = supplied[column] + if column in ('options', 'tag'): + try: + actual = json.loads(actual) if isinstance(actual, str) else actual + except (TypeError, ValueError): + raise ValueError(f'Untrusted {column} for index {row["index"]}') from None + wanted = json.loads(wanted) + if actual != wanted: + raise ValueError(f'Untrusted {column} for index {row["index"]}') + prediction = supplied['prediction'] + if (not isinstance(prediction, str) or not prediction.strip() + or any(marker in prediction.lower() for marker in ( + 'failed to obtain answer', 'video_load_failed', 'image_load_failed', + 'transport_error', 'api_error'))): + raise ValueError(f'Missing or failed prediction for index {row["index"]}; inference must complete before scoring') + predictions.append(prediction) + return predictions + + +def parse_choice(prediction, options): + """Conservative adapter: exact option text or one A/B/C choice expression. + + A/B/C, (A), A., option A, answer: A, and a leading A followed by its exact + option text are accepted. Explanations may follow an explicit answer marker, + but any conflicting standalone choice letter makes the output ambiguous. + Duplicate option text is never resolved to an arbitrary option ID. + """ + text = prediction.strip() + if re.fullmatch(r'(?:[ABC]|\([ABC]\))[.!]?', text): + return ord(re.search(r'[ABC]', text).group()) - 65 + exact = [i for i, option in enumerate(options) if text.casefold() == option.strip().casefold()] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + return None + standalone = set(re.findall(r'(? Date: Tue, 3 Nov 2026 18:18:29 -0600 Subject: [PATCH 6/6] Default the Brev launchable to MMSI-Video-Bench and add customer-data evaluation - Add MMSI-Video-Bench (default full reference run) and VideoPhy2 notebooks alongside Perception Test, sharing benchmarks.py and notebook_ui.py. - Ship pinned expected full-run results in baselines.json, with paper references labeled separately. - Add a labeled customer-video MCQ path: CUSTOM_DATA_RECIPE.md, Customer_Data.ipynb and custom_eval.py (validate/run, local strict scoring, same data across models). - Record requested settings and observed visual-token evidence per response (vlmeval/execution_evidence.py); fields the endpoint does not report stay explicitly unknown. - Add Kaizen UI presentation for notebook results and optional design tooling. Signed-off-by: Jon Patterson --- .../vlmevalkit/EXTERNAL_CONFIGURATION.md | 11 +- .../cosmos3/reasoner/vlmevalkit/README.md | 66 +- .../launchable/CUSTOM_DATA_RECIPE.md | 177 +++ .../vlmevalkit/launchable/Customer_Data.ipynb | 172 +++ .../reasoner/vlmevalkit/launchable/Dockerfile | 2 +- .../launchable/Evaluate_Cosmos3.ipynb | 190 +-- .../launchable/MMSI_Video_Bench.ipynb | 217 +++ .../launchable/Perception_Test.ipynb | 218 +++ .../reasoner/vlmevalkit/launchable/README.md | 257 +++- .../vlmevalkit/launchable/VideoPhy2.ipynb | 217 +++ .../vlmevalkit/launchable/baselines.json | 48 + .../vlmevalkit/launchable/benchmarks.py | 152 ++ .../vlmevalkit/launchable/compose.yaml | 5 + .../vlmevalkit/launchable/custom_eval.py | 340 +++++ .../vlmevalkit/launchable/launchable.py | 28 +- .../vlmevalkit/launchable/mmsivideo.py | 489 +++++++ .../vlmevalkit/launchable/notebook_ui.py | 260 ++++ .../reasoner/vlmevalkit/launchable/setup.sh | 26 +- .../vlmevalkit/launchable/test_custom_eval.py | 183 +++ .../launchable/test_execution_evidence.py | 131 ++ .../vlmevalkit/launchable/test_mmsivideo.py | 111 ++ .../vlmevalkit/launchable/test_notebook_ui.py | 36 + .../vlmevalkit/launchable/test_videophy2.py | 133 ++ .../vlmevalkit/launchable/videophy2.py | 522 +++++++ .../vlmevalkit/vlmeval/api/cosmos_reason.py | 53 + .../vlmevalkit/vlmeval/execution_evidence.py | 224 +++ .agents/skills/kaizen-ui/SKILL.md | 220 +++ .../kaizen-ui/assets/color-primitives-hex.md | 141 ++ .../skills/kaizen-ui/assets/icon-catalog.md | 884 ++++++++++++ .agents/skills/kaizen-ui/references/ICONS.md | 98 ++ .../references/components/Accordion.md | 131 ++ .../kaizen-ui/references/components/Anchor.md | 96 ++ .../references/components/AnimatedChevron.md | 66 + .../kaizen-ui/references/components/AppBar.md | 102 ++ .../kaizen-ui/references/components/Avatar.md | 101 ++ .../kaizen-ui/references/components/Badge.md | 131 ++ .../kaizen-ui/references/components/Banner.md | 149 ++ .../kaizen-ui/references/components/Block.md | 58 + .../references/components/Breadcrumbs.md | 85 ++ .../kaizen-ui/references/components/Button.md | 189 +++ .../references/components/ButtonGroup.md | 93 ++ .../kaizen-ui/references/components/Card.md | 193 +++ .../references/components/Carousel.md | 166 +++ .../references/components/ChatBubble.md | 115 ++ .../references/components/Checkbox.md | 125 ++ .../references/components/CodeSnippet.md | 151 ++ .../references/components/Collapsible.md | 120 ++ .../references/components/Combobox.md | 233 ++++ .../references/components/Command.md | 198 +++ .../references/components/DatePicker.md | 254 ++++ .../references/components/Divider.md | 74 + .../references/components/Dropdown.md | 213 +++ .../kaizen-ui/references/components/Flex.md | 89 ++ .../references/components/FormField.md | 182 +++ .../kaizen-ui/references/components/Grid.md | 129 ++ .../kaizen-ui/references/components/Group.md | 111 ++ .../kaizen-ui/references/components/Halo.md | 59 + .../kaizen-ui/references/components/Hero.md | 154 +++ .../references/components/HorizontalNav.md | 160 +++ .../kaizen-ui/references/components/Inline.md | 47 + .../references/components/InputShell.md | 201 +++ .../kaizen-ui/references/components/Kbd.md | 176 +++ .../references/components/KeyValueGroup.md | 150 ++ .../kaizen-ui/references/components/Label.md | 80 ++ .../kaizen-ui/references/components/Legend.md | 83 ++ .../kaizen-ui/references/components/List.md | 78 ++ .../kaizen-ui/references/components/Menu.md | 239 ++++ .../kaizen-ui/references/components/Meter.md | 102 ++ .../kaizen-ui/references/components/Modal.md | 257 ++++ .../references/components/Notification.md | 166 +++ .../references/components/NumberInput.md | 77 ++ .../references/components/OtpInput.md | 118 ++ .../references/components/PageHeader.md | 159 +++ .../references/components/Pagination.md | 217 +++ .../kaizen-ui/references/components/Panel.md | 125 ++ .../references/components/Popover.md | 188 +++ .../references/components/PortalProvider.md | 39 + .../references/components/ProgressBar.md | 77 ++ .../references/components/RadioGroup.md | 238 ++++ .../references/components/RangeSlider.md | 125 ++ .../kaizen-ui/references/components/Rating.md | 107 ++ .../references/components/Resizable.md | 243 ++++ .../references/components/SegmentedControl.md | 156 +++ .../references/components/SegmentedMeter.md | 138 ++ .../kaizen-ui/references/components/Select.md | 240 ++++ .../references/components/Severity.md | 78 ++ .../references/components/SidePanel.md | 249 ++++ .../references/components/Skeleton.md | 120 ++ .../kaizen-ui/references/components/Slider.md | 180 +++ .../references/components/Spinner.md | 80 ++ .../kaizen-ui/references/components/Stack.md | 96 ++ .../references/components/StatusIndicator.md | 92 ++ .../references/components/StatusMessage.md | 89 ++ .../references/components/Stepper.md | 234 ++++ .../kaizen-ui/references/components/Switch.md | 112 ++ .../kaizen-ui/references/components/Table.md | 264 ++++ .../kaizen-ui/references/components/Tabs.md | 207 +++ .../kaizen-ui/references/components/Tag.md | 178 +++ .../kaizen-ui/references/components/Text.md | 95 ++ .../references/components/TextArea.md | 193 +++ .../references/components/TextInput.md | 201 +++ .../references/components/ThemeProvider.md | 66 + .../references/components/TimeInput.md | 77 ++ .../kaizen-ui/references/components/Toast.md | 136 ++ .../references/components/Toaster.md | 128 ++ .../references/components/ToggleButton.md | 97 ++ .../components/ToggleButtonGroup.md | 121 ++ .../references/components/Tooltip.md | 148 ++ .../references/components/TreeNav.md | 134 ++ .../kaizen-ui/references/components/Upload.md | 237 ++++ .../references/components/VerticalNav.md | 320 +++++ .../design-tokens/CHART_DATA_COLORS.md | 54 + .../references/design-tokens/COLORS.md | 148 ++ .../design-tokens/COLOR_PRIMITIVES.md | 31 + .../design-tokens/EFFECTS_AND_LAYOUT.md | 50 + .../references/design-tokens/SPACING_SCALE.md | 36 + .../references/design-tokens/TYPOGRAPHY.md | 47 + .../references/guides/COMPOSITION.md | 30 + .../references/guides/CONFIGURATION.md | 50 + .../references/layouts/browse-page.md | 92 ++ .../kaizen-ui/references/layouts/dashboard.md | 274 ++++ .../references/layouts/detail-page.md | 248 ++++ .../references/layouts/detail-placement.md | 68 + .../references/layouts/error-page.md | 70 + .../kaizen-ui/references/layouts/form-page.md | 180 +++ .../references/layouts/layout-principles.md | 152 ++ .../kaizen-ui/references/layouts/list-view.md | 95 ++ .../references/layouts/page-modification.md | 33 + .../references/layouts/page-shell.md | 136 ++ .../references/layouts/workspace-page.md | 375 +++++ .../references/patterns/activity-feed.md | 129 ++ .../references/patterns/confirmations.md | 133 ++ .../references/patterns/data-display.md | 130 ++ .../references/patterns/data-visualization.md | 73 + .../references/patterns/element-visibility.md | 83 ++ .../references/patterns/empty-states.md | 116 ++ .../references/patterns/entity-cards.md | 173 +++ .../references/patterns/error-states.md | 163 +++ .../kaizen-ui/references/patterns/feedback.md | 79 ++ .../kaizen-ui/references/patterns/filters.md | 167 +++ .../kaizen-ui/references/patterns/forms.md | 211 +++ .../references/patterns/key-value-display.md | 153 +++ .../kaizen-ui/references/patterns/loading.md | 108 ++ .../references/patterns/navigation.md | 154 +++ .../references/patterns/ranked-lists.md | 60 + .../kaizen-ui/references/patterns/search.md | 46 + .../kaizen-ui/references/patterns/settings.md | 106 ++ .../references/patterns/stat-blocks.md | 71 + .dockerignore | 1 + .gitignore | 3 + package-lock.json | 1223 +++++++++++++++++ package.json | 13 + 152 files changed, 22616 insertions(+), 244 deletions(-) create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/CUSTOM_DATA_RECIPE.md create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Customer_Data.ipynb create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/MMSI_Video_Bench.ipynb create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Perception_Test.ipynb create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/VideoPhy2.ipynb create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/baselines.json create mode 100755 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/benchmarks.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/custom_eval.py create mode 100755 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/mmsivideo.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/notebook_ui.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_custom_eval.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_execution_evidence.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mmsivideo.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_notebook_ui.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_videophy2.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/videophy2.py create mode 100644 .agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/execution_evidence.py create mode 100644 .agents/skills/kaizen-ui/SKILL.md create mode 100644 .agents/skills/kaizen-ui/assets/color-primitives-hex.md create mode 100644 .agents/skills/kaizen-ui/assets/icon-catalog.md create mode 100644 .agents/skills/kaizen-ui/references/ICONS.md create mode 100644 .agents/skills/kaizen-ui/references/components/Accordion.md create mode 100644 .agents/skills/kaizen-ui/references/components/Anchor.md create mode 100644 .agents/skills/kaizen-ui/references/components/AnimatedChevron.md create mode 100644 .agents/skills/kaizen-ui/references/components/AppBar.md create mode 100644 .agents/skills/kaizen-ui/references/components/Avatar.md create mode 100644 .agents/skills/kaizen-ui/references/components/Badge.md create mode 100644 .agents/skills/kaizen-ui/references/components/Banner.md create mode 100644 .agents/skills/kaizen-ui/references/components/Block.md create mode 100644 .agents/skills/kaizen-ui/references/components/Breadcrumbs.md create mode 100644 .agents/skills/kaizen-ui/references/components/Button.md create mode 100644 .agents/skills/kaizen-ui/references/components/ButtonGroup.md create mode 100644 .agents/skills/kaizen-ui/references/components/Card.md create mode 100644 .agents/skills/kaizen-ui/references/components/Carousel.md create mode 100644 .agents/skills/kaizen-ui/references/components/ChatBubble.md create mode 100644 .agents/skills/kaizen-ui/references/components/Checkbox.md create mode 100644 .agents/skills/kaizen-ui/references/components/CodeSnippet.md create mode 100644 .agents/skills/kaizen-ui/references/components/Collapsible.md create mode 100644 .agents/skills/kaizen-ui/references/components/Combobox.md create mode 100644 .agents/skills/kaizen-ui/references/components/Command.md create mode 100644 .agents/skills/kaizen-ui/references/components/DatePicker.md create mode 100644 .agents/skills/kaizen-ui/references/components/Divider.md create mode 100644 .agents/skills/kaizen-ui/references/components/Dropdown.md create mode 100644 .agents/skills/kaizen-ui/references/components/Flex.md create mode 100644 .agents/skills/kaizen-ui/references/components/FormField.md create mode 100644 .agents/skills/kaizen-ui/references/components/Grid.md create mode 100644 .agents/skills/kaizen-ui/references/components/Group.md create mode 100644 .agents/skills/kaizen-ui/references/components/Halo.md create mode 100644 .agents/skills/kaizen-ui/references/components/Hero.md create mode 100644 .agents/skills/kaizen-ui/references/components/HorizontalNav.md create mode 100644 .agents/skills/kaizen-ui/references/components/Inline.md create mode 100644 .agents/skills/kaizen-ui/references/components/InputShell.md create mode 100644 .agents/skills/kaizen-ui/references/components/Kbd.md create mode 100644 .agents/skills/kaizen-ui/references/components/KeyValueGroup.md create mode 100644 .agents/skills/kaizen-ui/references/components/Label.md create mode 100644 .agents/skills/kaizen-ui/references/components/Legend.md create mode 100644 .agents/skills/kaizen-ui/references/components/List.md create mode 100644 .agents/skills/kaizen-ui/references/components/Menu.md create mode 100644 .agents/skills/kaizen-ui/references/components/Meter.md create mode 100644 .agents/skills/kaizen-ui/references/components/Modal.md create mode 100644 .agents/skills/kaizen-ui/references/components/Notification.md create mode 100644 .agents/skills/kaizen-ui/references/components/NumberInput.md create mode 100644 .agents/skills/kaizen-ui/references/components/OtpInput.md create mode 100644 .agents/skills/kaizen-ui/references/components/PageHeader.md create mode 100644 .agents/skills/kaizen-ui/references/components/Pagination.md create mode 100644 .agents/skills/kaizen-ui/references/components/Panel.md create mode 100644 .agents/skills/kaizen-ui/references/components/Popover.md create mode 100644 .agents/skills/kaizen-ui/references/components/PortalProvider.md create mode 100644 .agents/skills/kaizen-ui/references/components/ProgressBar.md create mode 100644 .agents/skills/kaizen-ui/references/components/RadioGroup.md create mode 100644 .agents/skills/kaizen-ui/references/components/RangeSlider.md create mode 100644 .agents/skills/kaizen-ui/references/components/Rating.md create mode 100644 .agents/skills/kaizen-ui/references/components/Resizable.md create mode 100644 .agents/skills/kaizen-ui/references/components/SegmentedControl.md create mode 100644 .agents/skills/kaizen-ui/references/components/SegmentedMeter.md create mode 100644 .agents/skills/kaizen-ui/references/components/Select.md create mode 100644 .agents/skills/kaizen-ui/references/components/Severity.md create mode 100644 .agents/skills/kaizen-ui/references/components/SidePanel.md create mode 100644 .agents/skills/kaizen-ui/references/components/Skeleton.md create mode 100644 .agents/skills/kaizen-ui/references/components/Slider.md create mode 100644 .agents/skills/kaizen-ui/references/components/Spinner.md create mode 100644 .agents/skills/kaizen-ui/references/components/Stack.md create mode 100644 .agents/skills/kaizen-ui/references/components/StatusIndicator.md create mode 100644 .agents/skills/kaizen-ui/references/components/StatusMessage.md create mode 100644 .agents/skills/kaizen-ui/references/components/Stepper.md create mode 100644 .agents/skills/kaizen-ui/references/components/Switch.md create mode 100644 .agents/skills/kaizen-ui/references/components/Table.md create mode 100644 .agents/skills/kaizen-ui/references/components/Tabs.md create mode 100644 .agents/skills/kaizen-ui/references/components/Tag.md create mode 100644 .agents/skills/kaizen-ui/references/components/Text.md create mode 100644 .agents/skills/kaizen-ui/references/components/TextArea.md create mode 100644 .agents/skills/kaizen-ui/references/components/TextInput.md create mode 100644 .agents/skills/kaizen-ui/references/components/ThemeProvider.md create mode 100644 .agents/skills/kaizen-ui/references/components/TimeInput.md create mode 100644 .agents/skills/kaizen-ui/references/components/Toast.md create mode 100644 .agents/skills/kaizen-ui/references/components/Toaster.md create mode 100644 .agents/skills/kaizen-ui/references/components/ToggleButton.md create mode 100644 .agents/skills/kaizen-ui/references/components/ToggleButtonGroup.md create mode 100644 .agents/skills/kaizen-ui/references/components/Tooltip.md create mode 100644 .agents/skills/kaizen-ui/references/components/TreeNav.md create mode 100644 .agents/skills/kaizen-ui/references/components/Upload.md create mode 100644 .agents/skills/kaizen-ui/references/components/VerticalNav.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/CHART_DATA_COLORS.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/COLORS.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/COLOR_PRIMITIVES.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/EFFECTS_AND_LAYOUT.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/SPACING_SCALE.md create mode 100644 .agents/skills/kaizen-ui/references/design-tokens/TYPOGRAPHY.md create mode 100644 .agents/skills/kaizen-ui/references/guides/COMPOSITION.md create mode 100644 .agents/skills/kaizen-ui/references/guides/CONFIGURATION.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/browse-page.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/dashboard.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/detail-page.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/detail-placement.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/error-page.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/form-page.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/layout-principles.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/list-view.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/page-modification.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/page-shell.md create mode 100644 .agents/skills/kaizen-ui/references/layouts/workspace-page.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/activity-feed.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/confirmations.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/data-display.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/data-visualization.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/element-visibility.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/empty-states.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/entity-cards.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/error-states.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/feedback.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/filters.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/forms.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/key-value-display.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/loading.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/navigation.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/ranked-lists.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/search.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/settings.md create mode 100644 .agents/skills/kaizen-ui/references/patterns/stat-blocks.md create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md index 7454e4706..a67f6826f 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/EXTERNAL_CONFIGURATION.md @@ -108,5 +108,12 @@ The notebook shows concise metrics/coverage and artifact links. BYOM uses complete `COSMOS_API_BASE`, exact `COSMOS_MODEL`, optional environment-only `COSMOS_API_KEY`, and explicit `COSMOS_MODEL_REVISION`; setup then skips local serving. Score-only setup uses `bash setup.sh --score-only`. -This launchable does not implement a customer-ingestion evaluation agent or alter -other model deployment tracks. Source changes are not hardware or accuracy proof. +For your own labeled video MCQ data, follow the +[customer-data recipe](launchable/CUSTOM_DATA_RECIPE.md) and +[Customer_Data.ipynb](launchable/Customer_Data.ipynb). This is a separate +canonical JSONL path: official benchmark identity/checksum gates remain intact. +Questions, answer options and actual ground-truth labels are required before +GPU use; footage alone cannot produce an accuracy score. Existing agents can +perform the one-off conversion described by the recipe. No customer-ingestion +agent, benchmark builder, automatic converter or other model deployment track +is added. diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md index 0ff62e995..9a05f4816 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Cosmos3 evaluation: Perception Test launchable and native benchmark recipes +# Cosmos3 evaluation: public benchmark launchable and native recipes This directory is an **NVIDIA-customized [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)** maintained in Cosmos Framework. It contains reference settings for NVIDIA @@ -14,17 +14,22 @@ datasets publicly reproducible or establish official score parity. For user-hosted endpoints and local dataset inputs, see [Deployment and dataset configuration](EXTERNAL_CONFIGURATION.md). -For the Nano-only Brev notebook, use the -[Perception Test launchable](launchable/README.md): 19,140 public validation -multiple-choice questions across 5,260 videos, pinned public downloads, native -inference, and local top1 scoring without an API judge. Default smoke selects -five questions from `video_8241`; full validation is explicit opt-in, not the -held-out test split. Public dataset materials are CC BY 4.0 with attribution to -the Perception Test authors; code is Apache 2.0. The prototype uses 16 sampled -images within official source cutpoints, aspect-preserving downscale to a -maximum side of 768, and makes no reference/published-score parity claim. -Data and state stay outside the checkout. This is not a customer-ingestion -evaluation agent. Existing native recipes below remain independent and unchanged. +For the Brev notebook workflow, use the +[public benchmark launchable](launchable/README.md): MMSI-Video-Bench is the +default full reference run, with Perception Test and VideoPhy2 also +available. Public inputs are downloaded by the customer; local scoring does +not use an external judge or private dataset credentials. Expected pinned +measurements and their comparison limits are included. + +For your own labeled video multiple-choice data, follow the +[standalone customer-data recipe](launchable/CUSTOM_DATA_RECIPE.md), then use +[Customer_Data.ipynb](launchable/Customer_Data.ipynb). Videos alone do not +provide an accuracy score: actual questions, options and ground-truth labels +are required. Existing agents perform one-off schema conversion; no benchmark +builder, packaging workflow or automatic ingestion agent is included. +Data and state stay outside the checkout. The launchable does not claim +identical published-reference protocols. Existing native recipes below remain +independent and unchanged. The kit ships, for each benchmark, reference `run.py --config` settings — split into a shared dataset layer (`cosmos_eval/data/`) and a per-family model layer @@ -97,7 +102,7 @@ Serve the model behind an OpenAI-compatible `/v1/chat/completions` endpoint (see **[Reasoner with vLLM](../../../../cookbooks/cosmos3/reasoner/README.md#run-with-vllm)**): ```bash -vllm serve nvidia/Cosmos3-Nano \ +VLLM_USE_FLASHINFER_SAMPLER=0 vllm serve nvidia/Cosmos3-Nano \ --async-scheduling \ --allowed-local-media-path / \ --port 8080 \ @@ -116,6 +121,41 @@ configs request, and `--allowed-local-media-path /` lets it read local media. (R hosted NIM, …) as long as it answers `POST /v1/chat/completions` and `GET /v1/models`; the wrapper health-checks the endpoint and auto-detects the served model id from `/v1/models`. +The launchable pins `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` +and model revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`. Its inspected +vLLM version is `0.23.1rc1.dev1306+gbfee6a802`. This build does **not** expose +`--trust-request-mm-kwargs` in `vllm serve --help=all`: the supported equivalent +is its unconditional request override path. `renderers/online_renderer.py` +passes request `mm_processor_kwargs` into `RenderParams`; `renderers/hf.py` +forwards those settings into the multimodal prompt/processor. Do not add the +unsupported flag or replace this digest merely to obtain it. Other runtime +versions must be checked for their own trust/override policy. The sampler +environment override above also applies to the launchable's serving container. + +Public benchmark runs save redacted requested/observed execution evidence per +response and an `execution-summary.json` aggregate, referenced by their +summary/provenance. Native `CosmosReason2` collection is opt-in via +`execution_evidence_dir` and `execution_processor_profile`; disabled collection +does not change its payload or return API. Evidence requests raw token IDs but +retains only visual groups, actual response model, usage and finish reasons—not +prompts, text token IDs, labels, media, endpoint URLs or credentials. + +Nano's explicit processor profile comes from the pinned public +[`config.json`](https://huggingface.co/nvidia/Cosmos3-Nano/resolve/e59a53c25979a090fa8706c9acc0c254a6e89b92/config.json) +and [`preprocessor_config.json`](https://huggingface.co/nvidia/Cosmos3-Nano/resolve/e59a53c25979a090fa8706c9acc0c254a6e89b92/preprocessor_config.json): +patch 16, spatial merge 2, temporal patch 2. Each bounded consecutive visual +token group yields spatial grid area `tokens × (16 × 2)²` **pixels**; video +groups yield **frame slots** `groups × 2`, not decoded or unique frame counts. +These derivations assume that processor profile and unpruned placeholder +expansion, not a server-attested revision. Per-group geometry is checked, not +just total tokens. Width/height, decoder FPS, decoded frame counts and effective +generation settings remain explicitly unknown when the response cannot expose +them. Non-32-aligned image inputs can be resized by the processor, so original +input dimensions are not relabeled as effective pixel geometry. Requested +generation/revision and historical scientific provenance remain requests and +provenance; they are not newly verified by a successful completion. + + --- ## 3. Environment variables diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/CUSTOM_DATA_RECIPE.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/CUSTOM_DATA_RECIPE.md new file mode 100644 index 000000000..ee143aa16 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/CUSTOM_DATA_RECIPE.md @@ -0,0 +1,177 @@ +# Evaluate your labeled video multiple-choice data + +This is a local **customer-video MCQ** evaluation, not an official benchmark protocol, published Cosmos score, or leaderboard submission. It does not convert arbitrary datasets automatically, build benchmarks, use an external judge, or obtain private labels. Each question has 2–26 options, one known correct answer, and optional category and temporal bounds. There is no requirement to use MVBench's tasks or three options. + +## 1. Labels and permissions first — before provisioning a GPU + +Obtain the selected videos and their actual ground-truth answers, confirm permission to process them, and confirm that every selected question has exactly one labeled answer. An unlabeled test split, a caption-only collection, or a dataset whose answers require inaccessible credentials is **not ready**. Stop before GPU provisioning if labels or videos are missing. Never infer, invent, or ask the evaluated model to generate missing ground truth. Record the split, source revision, license, inclusion rules, and any deliberate subset. A small subset is a customer subset, not the full source benchmark. + +Choose an external persistent directory, for example `/mnt/customer-eval/inspection-v1/`. Keep inputs and outputs outside the source checkout and outside source control. Do not paste private data or credentials into shared notebooks, commits, or conversion-agent prompts without authorization. Conversion agents should work with authorized local files only. + +Use exactly this layout (video filenames are examples, not supplied assets): + +```text +/mnt/customer-eval/inspection-v1/ + rows.jsonl + config-nano.json + config-qwen.json + media/ + invented_clip_001.mp4 + invented_clip_002.mp4 +``` + +No annotations or videos are included in this recipe. The examples below are invented metadata to explain the schema, not evaluation data or labels for any real collection. + +## 2. Map your source columns into canonical `rows.jsonl` + +Have an existing Codex or Claude coding agent inspect the source schema, source documentation, and a few authorized rows. Give it the schema and mapping instructions below. Ask it to write canonical JSONL directly into the external directory; do not create a reusable converter product or alter the public benchmark loaders. + +Before conversion, require the agent to state a concrete mapping table, including these decisions: + +| Canonical field | Mapping decision to document | +| --- | --- | +| `id` | Stable source question ID, including split/video/question identity if source IDs are not globally unique. Never a model-dependent ID. | +| `video` | Existing local video path relative to `media_root`; map source video IDs/filenames explicitly. Do not guess by caption similarity. | +| `question` | Actual question text, not an answer-bearing explanation, caption, rationale, or task label. | +| `candidates` | Ordered source options as an array of full strings. Preserve order and exact text; 2–26 unique nonempty entries. | +| `answer` | Full text of the one correct candidate. Resolve a source letter/index/option-column through the documented option order and index base. Never treat `0` and `1` index conventions interchangeably. | +| `category` | Optional existing source task/category or a documented, fixed grouping. Do not invent a new taxonomy per model. | +| `start`, `end` | Optional paired numeric seconds into the source video, with source units converted explicitly. If the source already provides trimmed clips, use the clip's own timeline, not original-video timestamps. | + +Suitable conversion-agent instruction: + +> Read CUSTOM_DATA_RECIPE.md and inspect the authorized local dataset. Report the source-to-canonical mapping, answer index base, split, selected count, video mapping, timestamp units and category meanings first. Produce only rows.jsonl and a config JSON in my external data directory using the exact schema below. Preserve source question IDs, option order, actual labels and video content. Resolve each source label to exactly one complete candidate string. Do not generate missing labels, download untrusted replacements, change the public benchmark loaders, add a converter tool, or request a GPU. Validate every selected row and list missing/ambiguous inputs rather than silently skipping them. Run custom_eval.py validate and resolve all validation failures before inference. Record the inclusion rules and source revision in dataset.source/revision. Do not include credentials in any artifact. + +Canonical `rows.jsonl` is UTF-8, one complete JSON object per line, no blank lines. These two invented rows illustrate different option counts and category/temporal settings: + +```jsonl +{"id":"inspection-v1:invented-001:q1","video":"invented_clip_001.mp4","question":"Which direction does the box move?","candidates":["Left","Right","It stays still","It leaves the frame"],"answer":"Right","category":"object-motion","start":0.0,"end":2.0} +{"id":"inspection-v1:invented-002:q1","video":"invented_clip_002.mp4","question":"Does the person place the cup on the table?","candidates":["Yes","No"],"answer":"Yes","category":"action-completion"} +``` + +### Content and temporal semantics + +- All rows are selected; there is no hidden sampling or automatic row dropping. Make a separate fixed manifest and dataset revision for an intentional subset. +- IDs are nonempty strings, globally unique within the manifest. Multiple questions can refer to one video but must have different IDs. +- `video` uses a safe relative POSIX path within `media_root`; absolute paths, URLs, `..`, backslashes, colon-containing paths and symlink escapes are rejected. Copy authorized media into the root or use paths that resolve within it. Missing or empty files fail validation. +- `question`, candidate strings, ID and category cannot be blank or have surrounding whitespace. Unknown fields are rejected so source labels/rationales are not accidentally sent to inference. Candidate duplicates are rejected. Keep semantically duplicate options out of the source selection too; validation cannot determine semantic equivalence. +- `answer` must match exactly one candidate string, including case and punctuation. The runner sends *all candidate texts* but never sends which one is correct, the answer field, IDs, categories or source labels. The source text itself must not disclose the answer in a rationale or annotation; inspect this during mapping. +- `start` and `end` must both be omitted for a full clip, or both supplied as finite numeric seconds with `0 <= start < end <= duration`. Booleans, NaN and infinity are invalid. Questions must be answerable from the selected segment; validation checks bounds, not semantic relevance. Native decoding performs temporal selection before sampling. +- Categories describe the question, not the predicted answer. Missing category is reported as `uncategorized`. Category accuracy is a question-weighted percentage within that category; overall accuracy is question-weighted across every selected row, not an average of category percentages. If a category is literally named `uncategorized`, it shares the missing-category bucket. + +## 3. Write `config-nano.json` + +Paths resolve relative to the **config file**, never the current working directory. Absolute external input paths also work. Adapt the invented dataset identity, source, model revision and pixel/frame settings to your data and deployed model: + +```json +{ + "dataset": { + "name": "customer-inspection-mcq", + "revision": "inspection-v1-labeled-fixed-selection", + "source": "Authorized local inspection export; source revision example-r1; fixed selected split; mapping documented by data owner" + }, + "manifest": "rows.jsonl", + "media_root": "media", + "nframes": 32, + "min_pixels": 50176, + "max_pixels": 50176, + "image_patch_size": 16, + "model": "nvidia/Cosmos3-Nano", + "model_revision": "unknown", + "generation": { + "temperature": 0, + "seed": 42, + "max_tokens": 32, + "enable_thinking": false + } +} +``` + +Customization points: + +- `dataset.name/revision/source` are required nonempty strings. Set revision to the real fixed export/selection identity; record transformations and licensing/source context in `source`. Do not put secrets or credential-bearing URLs here. +- `nframes` is an even integer at least 4, requested of the existing native video utility. Choose it to fit the segment durations and model capacity. Native decoding can reject clips that cannot satisfy the request; the runner pre-decodes every selected clip before any endpoint request. +- `min_pixels` and `max_pixels` are positive per-frame spatial pixel budgets (`min <= max`), not a total video token limit. `image_patch_size` must match the deployed processor; the example is not a universal model profile. Native shape rounding may change actual dimensions. Keep budgets fixed for like-for-like comparisons unless explicitly studying budgets. +- `model` and `model_revision` identify the deployed model. If omitted, use `COSMOS_MODEL` and `COSMOS_MODEL_REVISION`; model must be supplied in config or environment, and absent revision is honestly `unknown`. A config value takes precedence. The runner does not select/deploy a different endpoint based on the model field; point the endpoint to that actual deployment. +- Optional `processor_profile` is an evidence object supplied only from verified deployment/processor provenance, with `patch_size`, `spatial_merge_size`, `temporal_patch_size`, `image_token_id`, `video_token_id`, `provenance`, and the bounded-group markers `vision_start_token_id`/`vision_end_token_id` as supported by `vlmeval.execution_evidence`. When omitted for the exact `nvidia/Cosmos3-Nano` model name, the shared helper uses its pinned, provenance-recorded Nano processor profile; other models do not inherit that profile. Do not copy Nano values to another model or invent a profile to obtain a verified status. Missing runtime observability remains unknown; requested budgets alone are not proof of effective execution. Even verified bounded token-group counts do not attest decoded frame count, decoder FPS, generation settings or effective model revision. +- `generation` supports only `temperature`, nonnegative integer `seed`, positive integer `max_tokens`, and boolean `enable_thinking`. Defaults are the example values. The prompt requires one uppercase letter; a very small token cap or thinking mode can cause truncation, which counts wrong. Do not retry malformed answers selectively. +- No judge, scoring URL, label URL, automatic converter, output overwrite, or resume settings exist. Scoring is always local strict MCQ accuracy. Output location is passed separately with a fresh `--work-dir`. + +## 4. Validate all inputs before using an endpoint + +The toolkit is `.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit` in this repository. In a launchable installation, use its toolkit root containing `vlmeval/` and `launchable/`. Use the installed launchable Python environment (including decord, ffmpeg and the native Cosmos video dependencies). From that root: + +```bash +python launchable/custom_eval.py validate --config /mnt/customer-eval/inspection-v1/config-nano.json +``` + +Validation is CPU-side and sends no endpoint requests. It validates all labels, IDs, candidate counts, safe media paths, media duration and temporal ranges, probes media frames, and hashes every selected media file. Resolve every failure; never remove inconvenient rows just to obtain a completed score. An intentional subset needs a new documented identity before model comparison. Keep the returned `dataset_identity`, expected count and `manifest_sha256`. The semantic data identity includes dataset metadata, canonical rows and video hashes/metadata; reformatting JSON does not change it, but changing labels, options, selection, categories, temporal bounds or media bytes does. The raw manifest hash can change on harmless JSON reformatting. + +## 5. Run from notebook or CLI + +Open `launchable/Customer_Data.ipynb` from the `launchable/Evaluate_Cosmos3.ipynb` overview. Set its visible `CONFIG_FILE` to your absolute external config path; the notebook invokes the same validate/run CLI and renders `summary.json`. Use the overview/launchable endpoint workflow for deployment. The same Python API can also be called from a notebook cell after making the toolkit root and `launchable/` importable: + +```python +from pathlib import Path +import sys +sys.path.insert(0, str(Path("launchable").resolve())) +import custom_eval + +config = Path("/mnt/customer-eval/inspection-v1/config-nano.json") +validated = custom_eval.validate(config) +summary = custom_eval.run(config, Path("/mnt/customer-eval/results/nano-inspection-v1-run01"), api_nproc=1) +summary +``` + +This cell assumes the notebook working directory is the toolkit root; adjust the import path to the actual toolkit installation if not. Endpoint startup and model deployment remain the notebook/launchable's responsibility. Do not provision an endpoint until validation succeeds. + +For CLI, export the actual complete chat-completions URL; keep a real API key only in the environment: + +```bash +export COSMOS_API_BASE=http://cosmos-endpoint:8080/v1/chat/completions +export COSMOS_MODEL=nvidia/Cosmos3-Nano +export COSMOS_MODEL_REVISION=unknown +export COSMOS_API_KEY=EMPTY +python launchable/custom_eval.py run \ + --config /mnt/customer-eval/inspection-v1/config-nano.json \ + --work-dir /mnt/customer-eval/results/nano-inspection-v1-run01 \ + --api-nproc 1 +``` + +`EMPTY` is only for a local endpoint that does not require authentication. Use your authorized secret environment for authenticated endpoints; never store keys in config. `COSMOS_TIMEOUT` optionally sets positive finite request timeout seconds (default 300). Concurrency is 1–4; use 1 first. No endpoint URL or key is deliberately persisted. Dataset/source strings are user metadata, so keep them credential-free. + +Choose an external output parent that the evaluation process owns or can write to. Inputs may remain read-only. If an operator-provided output directory has the wrong owner, correct that directory's ownership or choose a writable output location before running; do not broaden raw-data permissions or alter the mapping to bypass a filesystem error. + +A result directory **must not already exist**, even if empty. Each run validates inputs again, pre-decodes every selected clip with the existing native video processor, and only then issues requests. Temporary transport files can contain encoded customer video while running and are removed when the run exits normally or through handled exceptions; keep work storage private and remove abandoned temporary files after an unclean process kill. + +## 6. Inspect local scoring and compare models + +Outputs under the fresh work directory: + +- `validated.json`: fixed dataset identity, source metadata, media SHA256/durations, selected count, requested sampling/model/generation. +- `requested-execution.json`: requested model/generation/processor budget/profile, not proof of runtime execution. +- `responses/000000.json`, etc.: per-request HTTP status, shared requested/observed execution evidence, strict parsed answer outcome, or redacted exception type. No inference prompt or video base64 is retained in response evidence. Arbitrary malformed model text is not retained. +- `execution-summary.json`: shared aggregate of recorded requested settings, observed bounded visual-group shapes, verification counts and unknown fields. Also embedded in `summary.json` as `execution_evidence`; incomplete evidence coverage or matching token geometry does not establish endpoint/model attestation or all effective media settings. +- `results.jsonl`: one result per processed row, stable ID, category, parsed single-letter prediction (or null), validity, correctness, finish reason and response-evidence pointer; transport errors have an `error` instead of a correctness value. +- `summary.json`: `benchmark=customer-video-mcq`, `scope=custom`, dataset/model identity, `status`, expected/scored coverage, elapsed seconds, execution pointers, overall accuracy and category breakdown. + +A valid answer is exactly one uppercase candidate letter after surrounding whitespace is removed. Full option text, lowercase letters, multiple choices, explanations, blanks and truncated (`finish_reason=length`) answers are **wrong** and remain in the denominator. Ground-truth labels are stored locally in your input manifest, never sent as an answer field. HTTP/connection/JSON transport failures are **not** wrong answers: the run fails, exits nonzero, and does not report a completed accuracy. Successful HTTP JSON responses with missing/malformed answer choices count as invalid wrong answers, while their execution evidence remains unverified. Inspect `status=completed`, `coverage.expected == coverage.scored`, and evidence availability before quoting any score. A preflight failure also fails the run without issuing requests. Never compare a failed/partial result as if complete. + +For Qwen or another compatible deployment, copy `config-nano.json` to `config-qwen.json` and change only the actual model/revision and provenance-grounded processor settings needed for that deployment. Point `COSMOS_API_BASE` to its endpoint. Do not regenerate the manifest, relabel options, alter selection, or replace media between runs. Use a different fresh work directory, for example `/mnt/customer-eval/results/qwen-inspection-v1-run01`. Use the exact model-supported patch size/profile; document any unavoidable processor budget differences. + +Before reporting comparison, require both runs completed, equal expected/scored counts, and identical `dataset_identity`; verify manifest/media identities if transferring between machines. Compare `primary_metric.value` (percent) with `correct/total`, per-category `accuracy_percent`, elapsed seconds and requested/observed execution evidence. Identify model revision, any budget differences, unknown runtime facts and transport/deployment environment. Preserve first-run results rather than overwriting or selectively regenerating wrong answers. The result measures this fixed customer selection under these settings, not official benchmark parity or leaderboard rank. + +### Fields reported as unknown + +Every run lists the same five fields in `execution_evidence.unknown_fields`. A standard chat-completions response does not report these values, so they are left unknown rather than filled in from the requested settings. + +| Field | Meaning | +| --- | --- | +| `decoded_video_frames` | Frames actually decoded and the source frame indices used. Frame slots are reported separately and can include repeated or padded frames. | +| `processor_width_height` | Width, height and crop used by the processor. Grid area is reported, but area alone does not determine dimensions. | +| `effective_decoder_fps` | Sampling FPS the decoder actually used, as opposed to the requested or source FPS. | +| `effective_generation_settings` | Generation parameters the server actually applied, as opposed to the requested configuration. | +| `effective_model_revision` | Checkpoint revision the server actually loaded, as opposed to the configured revision or returned model name. | + +For image inputs, the three video-decoder fields do not apply. What is reported: requested settings, the model name returned by the server, token usage, and visual-token counts. Frame slots and grid areas are derived from those token counts using the processor profile. + diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Customer_Data.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Customer_Data.ipynb new file mode 100644 index 000000000..a94812d4f --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Customer_Data.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "id": "welcome", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[← All benchmarks](Evaluate_Cosmos3.ipynb)\n", + "\n", + "# Your labeled video data\n", + "\n", + "**Labels first:** footage alone cannot produce an accuracy score. You need questions, answer options, and actual ground-truth answers. Confirm data permissions before using a GPU.\n", + "\n", + "Follow the [standalone customer-data recipe](CUSTOM_DATA_RECIPE.md) to map your source files into canonical JSONL and a config JSON. This is a customer-defined evaluation, not an official benchmark or leaderboard submission.\n" + ] + }, + { + "id": "helpers", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "scrolled": true + }, + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from notebook_ui import invoke, show_customer_results\n", + "\n", + "CLI = [sys.executable, \"-u\", str(Path.cwd() / \"custom_eval.py\")]\n" + ] + }, + { + "id": "configure", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 01 / Point to your configuration\n", + "\n", + "Set the absolute path to your external config file. It names your manifest, video directory, model, frame/pixel budget, and generation settings. Paths inside that file resolve relative to the config—not this notebook. Keep inputs fixed when comparing models.\n" + ] + }, + { + "id": "settings", + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "CONFIG_FILE = Path(\"/datasets/customer-eval/config-nano.json\")\n", + "API_NPROC = 1 # 1–4; run one benchmark notebook at a time.\n", + "\n", + "RESULTS_ROOT = Path(os.environ.get(\"EVAL_RESULTS_DIR\", \"/results\"))\n", + "STAMP = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%S%fZ\")\n", + "WORK = RESULTS_ROOT / \"customer-video-mcq\" / f\"custom-{STAMP}\"\n", + "LOG = WORK.with_suffix(\".log\")\n", + "if not CONFIG_FILE.is_absolute():\n", + " raise ValueError(\"Set CONFIG_FILE to your absolute external configuration path.\")\n", + "print(f\"Configuration: {CONFIG_FILE}\\nNew result directory: {WORK}\")\n" + ] + }, + { + "id": "validate", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 02 / Validate labels and media\n", + "\n", + "This CPU-only step checks every selected question, label, ID, option list, video, and temporal bound before any model request. Missing or ambiguous inputs stop the run; they are not silently skipped. Review the reported count and dataset identity.\n" + ] + }, + { + "id": "validate-inputs", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "scrolled": true + }, + "execution_count": null, + "outputs": [], + "source": [ + "invoke(CLI + [\"validate\", \"--config\", str(CONFIG_FILE)], WORK.with_suffix(\".validation.log\"))\n" + ] + }, + { + "id": "run", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 03 / Run and score locally\n", + "\n", + "The endpoint must already serve the model named in your configuration. Only the question and all options are sent—not the correct-answer field. Invalid or truncated answers count wrong; transport failures cannot become a completed score. No external judge or manual benchmark command.\n" + ] + }, + { + "id": "infer-and-score", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "scrolled": true + }, + "execution_count": null, + "outputs": [], + "source": [ + "invoke(CLI + [\"run\", \"--config\", str(CONFIG_FILE), \"--work-dir\", str(WORK),\n", + " \"--api-nproc\", str(API_NPROC)], LOG)\n" + ] + }, + { + "id": "review", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 04 / Inspect the score and execution evidence\n", + "\n", + "Check complete coverage, category scores, dataset identity, and requested versus observable effective input settings. An unknown effective setting is not a verified setting. Retain the model revision and any budget differences.\n" + ] + }, + { + "id": "results", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "scrolled": false + }, + "execution_count": null, + "outputs": [], + "source": [ + "summary = json.loads((WORK / \"summary.json\").read_text())\n", + "show_customer_results(summary, WORK, LOG)\n" + ] + }, + { + "id": "compare", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare another model\n", + "\n", + "Deploy or select the second endpoint using the [recipe](CUSTOM_DATA_RECIPE.md#6-inspect-local-scoring-and-compare-models). Change only the model/revision and any required processor profile in a copy of your config. Keep the manifest and videos unchanged; rerun this notebook for a fresh result directory. Require equal dataset identities and coverage before comparing scores. Closing a notebook tab does not stop a running kernel or Brev VM billing.\n", + "\n", + "Share reviewed aggregate scores and code—not labels, videos, raw predictions, credentials, or executed notebooks.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "title": "Your labeled video data" + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile index 1f3623987..6d8912f72 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile @@ -16,6 +16,6 @@ RUN pip install --no-cache-dir --no-deps -e . \ ARG COSMOS_COMMIT=unknown ENV COSMOS_COMMIT=${COSMOS_COMMIT} PYTHONUNBUFFERED=1 HF_HOME=/cache/huggingface \ AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null \ - AWS_EC2_METADATA_DISABLED=true PERCEPTION_TEST_DATA_DIR=/perception-test LMUData=/cache/lmudata + AWS_EC2_METADATA_DISABLED=true PERCEPTION_TEST_DATA_DIR=/perception-test BENCHMARK_DATA_DIR=/datasets LMUData=/cache/lmudata WORKDIR /opt/vlmevalkit/launchable CMD ["python", "jupyter_config.py"] diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index 9c91b7be9..fb6bc877b 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -2,179 +2,51 @@ "cells": [ { "cell_type": "markdown", + "id": "overview", "metadata": {}, "source": [ - "# Evaluate Cosmos3-Nano on Perception Test validation MCQ\n", + "
\n", + "

NVIDIA Cosmos · Evaluation workspace

\n", + "

Evaluate video understanding

\n", + "

Run the default reference benchmark, compare another public benchmark, or evaluate your own labeled videos. All scoring is local.

\n", + "
\n", "\n", - "**Public download → validate coverage → native inference → local top1.** Default smoke selects five questions (IDs 0–4) from `video_8241`, stable indices 13404–13408. Full validation is explicit opt-in: 19,140 questions / 5,260 MCQ videos, **not the held-out test split**.\n", + "**Start here:** [Run MMSI-Video-Bench](MMSI_Video_Bench.ipynb), the default full reference run. [Expected pinned results and limits](baselines.json) are included with the launchable.\n", "\n", - "Dataset attribution: [Perception Test authors / Google DeepMind](https://github.com/google-deepmind/perception_test), [paper](https://arxiv.org/abs/2305.13786). Their code is Apache 2.0 and other materials [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); retain attribution/license link and identify adaptations. This launchable is an adaptation, not author endorsement or a customer-ingestion evaluation agent.\n", + "\n", "\n", - "The public Nano checkpoint is pinned to `e59a53c25979a090fa8706c9acc0c254a6e89b92`. This committed notebook is unexecuted; no empirical validation or reference-score parity is asserted." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1 · Choose scope and execution mode\n", - "\n", - "The external `PERCEPTION_TEST_DATA_DIR` is mounted at `/perception-test`; cache and results are separate. No model/data credentials, Docker socket or dataset artifacts are embedded. `RUN_FULL = True` selects full validation directly without rerunning smoke.\n", - "\n", - "For **score-only**, choose explicit `SCORE_SCOPE` and exactly one of `PREDICTIONS_FILE` or `EXISTING_WORK`. Only pinned metadata is needed: no video, model, GPU or endpoint. Set `DOWNLOAD_PUBLIC = False` for already-staged offline scoring." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "from pathlib import Path\n", - "import subprocess\n", - "import sys\n", - "from datetime import datetime, timezone\n", - "from IPython.display import FileLink, display\n", - "\n", - "SCORE_ONLY = os.environ.get('EVAL_SCORE_ONLY') == '1'\n", - "PREDICTIONS_FILE = None\n", - "EXISTING_WORK = None\n", - "SCORE_SCOPE = 'smoke' # Choose 'full' explicitly for 19,140 existing predictions.\n", - "RUN_FULL = False\n", - "DOWNLOAD_PUBLIC = True\n", - "NFRAME = 16 # Fixed launchable prototype protocol.\n", - "API_NPROC = 1\n", - "DATA = Path(os.environ.get('PERCEPTION_TEST_DATA_DIR', '/perception-test'))\n", - "SCOPE = SCORE_SCOPE if SCORE_ONLY else ('full' if RUN_FULL else 'smoke')\n", - "STAMP = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')\n", - "WORK = Path('/results') / f'perception-test-{SCOPE}-{STAMP}'\n", - "SCORE_DIR = Path('/results') / f'perception-test-score-{SCOPE}-{STAMP}'\n", - "CLI = [sys.executable, '-u', '/opt/vlmevalkit/launchable/launchable.py']\n", - "\n", - "def invoke(command):\n", - " # Stream each line into Jupyter, not a buffer held until the run ends.\n", - " with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,\n", - " text=True, bufsize=1) as process:\n", - " for line in process.stdout:\n", - " print(line, end='', flush=True)\n", - " if process.wait():\n", - " raise subprocess.CalledProcessError(process.returncode, command)\n", - "\n", - "if SCOPE not in {'smoke', 'full'}:\n", - " raise ValueError('Choose explicit smoke or full scope.')\n", - "if SCORE_ONLY and (PREDICTIONS_FILE is None) == (EXISTING_WORK is None):\n", - " raise ValueError('Set exactly one of PREDICTIONS_FILE or EXISTING_WORK.')\n", - "print(('Score existing predictions' if SCORE_ONLY else 'Infer and score'), SCOPE)\n", - "print('Data:', DATA, '| frames:', NFRAME, '| API workers:', API_NPROC)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2 · Download and prepare the selected public scope\n", - "\n", - "The helper uses direct official public URLs, pinned SHA-256, and safe extraction. Source repository revision: `3938d2f1ba3a6b502025741cea4cd73c7b3bdfaf`. Metadata comprises `mc_question_valid.json`, `cut_frame_mapping_valid.json`, and the attribution README. Smoke downloads the approximately 225 MB official sample archive; full downloads the 75,375,143,328-byte official validation archive, plus extraction space.\n", - "\n", - "Inference preparation requires every selected video before contacting the model. Score-only downloads metadata only and prepares with `require_media=False`. Canonical annotation/cut/scope fingerprints are independent of video presence, with media hashes recorded separately." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if DOWNLOAD_PUBLIC:\n", - " invoke(CLI + ['download', '--data-root', str(DATA),\n", - " '--scope', 'metadata' if SCORE_ONLY else SCOPE])\n", - "else:\n", - " print('Using staged pinned inputs; no download.')\n", - "scope_flag = ['--smoke'] if SCOPE == 'smoke' else []\n", - "invoke(CLI + ['prepare', '--data-root', str(DATA)] + scope_flag\n", - " + (['--metadata-only'] if SCORE_ONLY else []))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3 · Infer and score locally, or rescore existing predictions\n", - "\n", - "**Prototype protocol:** exactly 16 uniform frames within official source cutpoints; `-1` means all frames, positive cuts are exclusive stop frames. Bad cuts fail rather than clamp. Aspect-preserving downscale only to maximum side 768, no crop. Sampled images go directly to the endpoint, never re-encoded video. This is not a reference-score or official letter-parser parity claim.\n", + "### Your own data\n", "\n", - "Native `CosmosReason2` generation uses temperature 0, seed 1, no thinking, max 8,192 output tokens. BYOM must accept sampled-image messages; configure endpoint/model/revision before host setup and keep API keys environment-only.\n", + "**You need labels, not just footage:** questions, answer options, and correct answers, with permission to process the selected videos. Use the [standalone recipe](CUSTOM_DATA_RECIPE.md), then open [Your labeled video data](Customer_Data.ipynb). An existing coding agent can convert your authorized data into the documented shape; no benchmark-builder or automated conversion product is required.\n", "\n", - "Progress streams here and into `run.log`. Fresh output directories prevent stale reuse. Exact selected IDs are required. Blank, missing, duplicate, unexpected or transport-failure rows reject incomplete runs before scoring. Nonempty malformed/ambiguous choices count wrong and invalid, not dropped; literal text `nan` remains model text. Trusted labels come only from pinned annotations, never prediction-supplied answers. No API judge, dummy answer or partial-full-score fallback." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if SCORE_ONLY:\n", - " source = (['--work-dir', str(EXISTING_WORK)] if EXISTING_WORK is not None\n", - " else ['--predictions', str(PREDICTIONS_FILE)])\n", - " invoke(CLI + ['score', '--data-root', str(DATA), '--scope', SCOPE,\n", - " '--score-dir', str(SCORE_DIR), '--nframe', str(NFRAME)] + source)\n", - " SUMMARY = SCORE_DIR / 'summary.json'\n", - "else:\n", - " # Controller validates coverage before checking the model endpoint.\n", - " invoke(CLI + ['run', '--data-root', str(DATA), '--mode', 'all',\n", - " '--nframe', str(NFRAME), '--api-nproc', str(API_NPROC),\n", - " '--work-dir', str(WORK)] + scope_flag)\n", - " SUMMARY = WORK / 'scores' / 'summary.json'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4 · Human result and durable artifacts\n", + "### Run a public benchmark\n", "\n", - "Top1 is correct option ID / **all selected questions**, with per-area, per-reasoning and per-tag diagnostics in the summary. Smoke is not a full benchmark. Artifact provenance records annotation/cut identities, separate media hashes, prediction hash, source/model revisions and frame protocol. Controller artifacts include useful UTC start/end and measured elapsed time, not invented cost or peak memory." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "summary = json.loads(SUMMARY.read_text())\n", - "metric = summary['primary_metric']\n", - "print(f\"Perception Test validation — {SCOPE}: {summary['status']}\")\n", - "print(f\"Top1: {metric['value']:.2%} ({metric['correct']}/{metric['total']})\")\n", - "print('Coverage:', json.dumps(summary['coverage']))\n", - "print('Prototype protocol; no reference-score parity claim.')\n", - "artifacts = [SUMMARY, SUMMARY.parent / 'controller.json']\n", - "if not SCORE_ONLY:\n", - " artifacts += [WORK / 'run.json', WORK / 'run.log', WORK / 'config.json']\n", - "for artifact in artifacts:\n", - " if artifact.is_file():\n", - " display(FileLink(str(artifact), url_prefix='/files'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Stop and retain results\n", + "1. Open a benchmark notebook above.\n", + "2. MMSI-Video-Bench defaults to `\"full\"`; Perception Test and VideoPhy2 default to `\"smoke\"`. Change `SCOPE` explicitly for another scope.\n", + "3. Select **Run → Run All Cells**. No terminal Docker command is needed to execute the benchmark.\n", "\n", - "In the **host terminal**, `docker compose --profile local stop` preserves mounted state. `bash setup.sh --score-only` starts only authenticated Jupyter. Evaluator service: `perception-eval`; Compose project: `cosmos3-perception-test`. There is no Docker socket or host model port. Stopping containers does not stop Brev VM billing; stop the instance separately.\n", + "**Run one benchmark at a time.** The notebooks share the existing model endpoint and Python runners; opening a page does not provision another model instance. Each run writes a fresh folder under `/results//`.\n", "\n", - "Do not commit credentials, videos, labels, predictions or answer-bearing artifacts. Clear outputs before sharing. The supported deployment runtime is native Linux x86-64. A source-only change does not demonstrate notebook/GPU execution or full validation reproduction." + "Smoke checks are not full benchmark scores. Accuracy and correlation are different metrics—do not average scores across these benchmarks. Check each notebook’s download budget and attribution before running.\n" ] } ], "metadata": { - "kernelspec": {"display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3"}, - "language_info": {"name": "python", "version": "3.11"} + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "title": "Cosmos3 · Benchmark overview" }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/MMSI_Video_Bench.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/MMSI_Video_Bench.ipynb new file mode 100644 index 000000000..71fb5cc83 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/MMSI_Video_Bench.ipynb @@ -0,0 +1,217 @@ +{ + "cells": [ + { + "id": "welcome", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[← All benchmarks](Evaluate_Cosmos3.ipynb)\n", + "\n", + "# MMSI-Video-Bench\n", + "\n", + "Answer spatial-reasoning questions using canonical video frames and reference views.\n", + "\n", + "This is the default reference run. It evaluates the full public benchmark by default; choose smoke only for a pipeline check. Select **Run → Run All Cells**.\n" + ] + }, + { + "id": "shared-helpers", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "from notebook_ui import show_plan, show_results, invoke\n", + "\n", + "# This page always runs MMSI-Video-Bench. Use All benchmarks to switch.\n", + "BENCHMARK = \"mmsivideo-bench\"\n", + "SCORE_ONLY = False\n", + "EXISTING_WORK = None\n", + "PREDICTIONS_FILE = None\n" + ] + }, + { + "id": "configure", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 01 / Configure the run\n", + "\n", + "`SCOPE = \"full\"` evaluates all 1,106 questions against the published expected launchable baseline. Choose `\"smoke\"` for five questions only—not a performance measurement. Each attempt gets a fresh timestamped results folder. Review the download budget before running.\n" + ] + }, + { + "id": "settings", + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from benchmarks import validate_selection\n", + "\n", + "SCOPE = os.environ.get(\"EVAL_SCOPE\", \"full\") # \"full\" reference run or \"smoke\" pipeline check\n", + "DOWNLOAD_PUBLIC = True # False only when the selected pinned inputs are staged.\n", + "API_NPROC = 1 # 1–4 simultaneous API workers; do not run concurrent notebooks.\n", + "\n", + "\n", + "# Persistent mounts are separate from source. Override these for another runtime.\n", + "DATA_ROOT = Path(os.environ.get(\"BENCHMARK_DATA_DIR\", \"/datasets\"))\n", + "RESULTS_ROOT = Path(os.environ.get(\"EVAL_RESULTS_DIR\", \"/results\"))\n", + "DATA = (Path(os.environ.get(\"PERCEPTION_TEST_DATA_DIR\", \"/perception-test\"))\n", + " if BENCHMARK == \"perception-test\" else DATA_ROOT / BENCHMARK)\n", + "LAUNCHABLE = Path.cwd()\n", + "CLI = [sys.executable, \"-u\", str(LAUNCHABLE / \"benchmarks.py\")]\n", + "STAMP = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%S%fZ\")\n", + "WORK = RESULTS_ROOT / BENCHMARK / f\"{SCOPE}-{'score-' if SCORE_ONLY else ''}{STAMP}\"\n", + "LOG = WORK.with_suffix(\".log\")\n", + "\n", + "validate_selection(BENCHMARK, SCOPE, SCORE_ONLY)\n", + "if not 1 <= API_NPROC <= 4:\n", + " raise ValueError(\"Choose API_NPROC between 1 and 4.\")\n", + "if SCORE_ONLY and (EXISTING_WORK is None) == (PREDICTIONS_FILE is None):\n", + " raise ValueError(\"Set exactly one of EXISTING_WORK or PREDICTIONS_FILE.\")\n", + "if not (LAUNCHABLE / \"benchmarks.py\").is_file():\n", + " raise ValueError(\"Run this notebook from its launchable directory.\")\n", + "show_plan(BENCHMARK, SCOPE, DATA, WORK, API_NPROC, SCORE_ONLY)\n" + ] + }, + { + "id": "download", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 02 / Download the public inputs\n", + "\n", + "Downloads use pinned publisher URLs and checksums, then safe extraction. Existing verified assets are reused. **Check the download budget above before running:** a small smoke can still require a large source archive, especially MMSI-Video-Bench. No Hugging Face token or private NVIDIA dataset is required.\n", + "\n", + "Inputs stay on your instance. Keep publisher attribution and applicable license terms; public access is not permission to redistribute every underlying video.\n" + ] + }, + { + "id": "fetch", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "if DOWNLOAD_PUBLIC:\n", + " command = CLI + [\"download\", \"--benchmark\", BENCHMARK,\n", + " \"--scope\", SCOPE, \"--data-root\", str(DATA)]\n", + " if SCORE_ONLY:\n", + " command += [\"--metadata-only\"]\n", + " invoke(command, WORK.with_suffix(\".download.log\"))\n", + "else:\n", + " print(\"Downloads skipped. The runner will validate the selected staged inputs before inference.\")\n" + ] + }, + { + "id": "execute", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 03 / Run inference and score locally\n", + "\n", + "This cell checks selected media/annotation coverage before model requests, streams real progress below, and writes a durable log. It then applies the selected benchmark’s local scorer. **No external scoring submission, external judge, or manual benchmark command.**\n", + "\n", + "**MMSI-Video-Bench protocol:** Uniform-50 public annotation frames plus references; the pinned official answer extractor is primary. The maintained no-judge extractor is diagnostic, not interchangeable. Truncated or invalid outputs remain counted and clearly reported; transport failures cannot become a completed full score.\n", + "\n", + "Run one benchmark at a time. Full runs can take hours. Interrupting leaves diagnostics; rerun the configuration cell before a new attempt to get a fresh output folder.\n" + ] + }, + { + "id": "infer-and-score", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "command = CLI + [\"run\", \"--benchmark\", BENCHMARK, \"--scope\", SCOPE,\n", + " \"--data-root\", str(DATA), \"--work-dir\", str(WORK),\n", + " \"--api-nproc\", str(API_NPROC)]\n", + "if SCORE_ONLY:\n", + " command += [\"--score-only\"]\n", + " command += ([\"--existing-work\", str(EXISTING_WORK)] if EXISTING_WORK is not None\n", + " else [\"--predictions\", str(PREDICTIONS_FILE)])\n", + "invoke(command, LOG)\n" + ] + }, + { + "id": "review", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 04 / Read the result, retain the evidence\n", + "\n", + "The heading below is checked against the result artifact’s benchmark and scope. Accuracy and correlation are not the same metric; do not combine scores across benchmarks. **Smoke results are not full benchmark results.**\n", + "\n", + "Expand the protocol record to inspect the exact setting. JSON reports, prediction evidence, source/dataset revisions, and logs remain in the benchmark’s own result folder. Links below are private runtime artifacts—not files to commit or publish.\n" + ] + }, + { + "id": "results", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "SUMMARY = WORK / (\"notebook-summary.json\" if SCORE_ONLY else \"summary.json\")\n", + "summary = json.loads(SUMMARY.read_text())\n", + "show_results(summary, BENCHMARK, SCOPE, WORK, LOG)\n" + ] + }, + { + "id": "handoff", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reviewer checklist / Before sharing with Dan\n", + "\n", + "- Confirm the selected dataset, split, frame/pixel budget, generation settings, and primary scorer are the intended protocol.\n", + "- Inspect coverage, invalid/truncated answers, source/data/model revisions, and the retained logs. A matching score alone does not establish reproduction of a published result.\n", + "- To switch benchmarks, use [← All benchmarks](Evaluate_Cosmos3.ipynb) and open another notebook. Finish or interrupt this run first; closing its tab does not stop its kernel. For a new attempt here, use **Restart Kernel and Run All Cells**. Verified inputs are cached; new results are isolated.\n", + "- Share code and a reviewed aggregate result, not videos, labels, raw predictions, endpoint credentials, or executed notebook outputs. Clear all outputs before committing.\n", + "\n", + "### Stop safely\n", + "\n", + "In the **host terminal**, `docker compose --profile local stop` preserves mounted data/results. The existing evaluator service is still named `perception-eval` and the Compose project `cosmos3-perception-test`; both serve all three benchmark notebooks. Stopping containers does **not** stop Brev VM billing—stop the instance separately when finished. This notebook targets native Linux x86-64 on Brev, with the pinned Cosmos3-Nano endpoint or a configured compatible endpoint.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "title": "MMSI-Video-Bench" + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Perception_Test.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Perception_Test.ipynb new file mode 100644 index 000000000..d51d4210f --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Perception_Test.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "id": "welcome", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[← All benchmarks](Evaluate_Cosmos3.ipynb)\n", + "\n", + "# Perception Test\n", + "\n", + "Understand actions, objects, and events using the public validation questions and local labels.\n", + "\n", + "Choose smoke or full scope below, then select **Run → Run All Cells**.\n" + ] + }, + { + "id": "shared-helpers", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "from notebook_ui import show_plan, show_results, invoke\n", + "\n", + "# This page always runs Perception Test. Use All benchmarks to switch.\n", + "BENCHMARK = \"perception-test\"\n" + ] + }, + { + "id": "configure", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 01 / Configure the run\n", + "\n", + "Keep `SCOPE = \"smoke\"` for a quick pipeline check; choose `\"full\"` for the complete Perception Test benchmark. Each attempt gets a fresh timestamped results folder.\n" + ] + }, + { + "id": "settings", + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from benchmarks import validate_selection\n", + "\n", + "SCOPE = os.environ.get(\"EVAL_SCOPE\", \"smoke\") # \"smoke\" or \"full\"\n", + "DOWNLOAD_PUBLIC = True # False only when the selected pinned inputs are staged.\n", + "API_NPROC = 1 # 1–4 simultaneous API workers; do not run concurrent notebooks.\n", + "\n", + "# Advanced: rescore existing Perception Test predictions without a model/video.\n", + "SCORE_ONLY = os.environ.get(\"EVAL_SCORE_ONLY\") == \"1\"\n", + "EXISTING_WORK = None\n", + "PREDICTIONS_FILE = None # Set exactly one existing source when SCORE_ONLY is True.\n", + "\n", + "# Persistent mounts are separate from source. Override these for another runtime.\n", + "DATA_ROOT = Path(os.environ.get(\"BENCHMARK_DATA_DIR\", \"/datasets\"))\n", + "RESULTS_ROOT = Path(os.environ.get(\"EVAL_RESULTS_DIR\", \"/results\"))\n", + "DATA = (Path(os.environ.get(\"PERCEPTION_TEST_DATA_DIR\", \"/perception-test\"))\n", + " if BENCHMARK == \"perception-test\" else DATA_ROOT / BENCHMARK)\n", + "LAUNCHABLE = Path.cwd()\n", + "CLI = [sys.executable, \"-u\", str(LAUNCHABLE / \"benchmarks.py\")]\n", + "STAMP = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%S%fZ\")\n", + "WORK = RESULTS_ROOT / BENCHMARK / f\"{SCOPE}-{'score-' if SCORE_ONLY else ''}{STAMP}\"\n", + "LOG = WORK.with_suffix(\".log\")\n", + "\n", + "validate_selection(BENCHMARK, SCOPE, SCORE_ONLY)\n", + "if not 1 <= API_NPROC <= 4:\n", + " raise ValueError(\"Choose API_NPROC between 1 and 4.\")\n", + "if SCORE_ONLY and (EXISTING_WORK is None) == (PREDICTIONS_FILE is None):\n", + " raise ValueError(\"Set exactly one of EXISTING_WORK or PREDICTIONS_FILE.\")\n", + "if not (LAUNCHABLE / \"benchmarks.py\").is_file():\n", + " raise ValueError(\"Run this notebook from its launchable directory.\")\n", + "show_plan(BENCHMARK, SCOPE, DATA, WORK, API_NPROC, SCORE_ONLY)\n" + ] + }, + { + "id": "download", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 02 / Download the public inputs\n", + "\n", + "Downloads use pinned publisher URLs and checksums, then safe extraction. Existing verified assets are reused. **Check the download budget above before running:** a small smoke can still require a large source archive, especially MMSI-Video-Bench. No Hugging Face token or private NVIDIA dataset is required.\n", + "\n", + "Inputs stay on your instance. Keep publisher attribution and applicable license terms; public access is not permission to redistribute every underlying video. For Perception Test score-only mode, this cell fetches metadata only.\n" + ] + }, + { + "id": "fetch", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "if DOWNLOAD_PUBLIC:\n", + " command = CLI + [\"download\", \"--benchmark\", BENCHMARK,\n", + " \"--scope\", SCOPE, \"--data-root\", str(DATA)]\n", + " if SCORE_ONLY:\n", + " command += [\"--metadata-only\"]\n", + " invoke(command, WORK.with_suffix(\".download.log\"))\n", + "else:\n", + " print(\"Downloads skipped. The runner will validate the selected staged inputs before inference.\")\n" + ] + }, + { + "id": "execute", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 03 / Run inference and score locally\n", + "\n", + "This cell checks selected media/annotation coverage before model requests, streams real progress below, and writes a durable log. It then applies the selected benchmark’s local scorer. **No external scoring submission, external judge, or manual benchmark command.**\n", + "\n", + "**Perception Test protocol:** 16 source-cut-aware sampled images; local top1 on public validation labels.\n", + "\n", + "Run one benchmark at a time. Full runs can take hours. Interrupting leaves diagnostics; rerun the configuration cell before a new attempt to get a fresh output folder.\n" + ] + }, + { + "id": "infer-and-score", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "command = CLI + [\"run\", \"--benchmark\", BENCHMARK, \"--scope\", SCOPE,\n", + " \"--data-root\", str(DATA), \"--work-dir\", str(WORK),\n", + " \"--api-nproc\", str(API_NPROC)]\n", + "if SCORE_ONLY:\n", + " command += [\"--score-only\"]\n", + " command += ([\"--existing-work\", str(EXISTING_WORK)] if EXISTING_WORK is not None\n", + " else [\"--predictions\", str(PREDICTIONS_FILE)])\n", + "invoke(command, LOG)\n" + ] + }, + { + "id": "review", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 04 / Read the result, retain the evidence\n", + "\n", + "The heading below is checked against the result artifact’s benchmark and scope. Accuracy and correlation are not the same metric; do not combine scores across benchmarks. **Smoke results are not full benchmark results.**\n", + "\n", + "Expand the protocol record to inspect the exact setting. JSON reports, prediction evidence, source/dataset revisions, and logs remain in the benchmark’s own result folder. Links below are private runtime artifacts—not files to commit or publish.\n" + ] + }, + { + "id": "results", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "SUMMARY = WORK / (\"notebook-summary.json\" if SCORE_ONLY else \"summary.json\")\n", + "summary = json.loads(SUMMARY.read_text())\n", + "show_results(summary, BENCHMARK, SCOPE, WORK, LOG)\n" + ] + }, + { + "id": "handoff", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reviewer checklist / Before sharing with Dan\n", + "\n", + "- Confirm the selected dataset, split, frame/pixel budget, generation settings, and primary scorer are the intended protocol.\n", + "- Inspect coverage, invalid/truncated answers, source/data/model revisions, and the retained logs. A matching score alone does not establish reproduction of a published result.\n", + "- To switch benchmarks, use [← All benchmarks](Evaluate_Cosmos3.ipynb) and open another notebook. Finish or interrupt this run first; closing its tab does not stop its kernel. For a new attempt here, use **Restart Kernel and Run All Cells**. Verified inputs are cached; new results are isolated.\n", + "- Share code and a reviewed aggregate result, not videos, labels, raw predictions, endpoint credentials, or executed notebook outputs. Clear all outputs before committing.\n", + "\n", + "### Stop safely\n", + "\n", + "In the **host terminal**, `docker compose --profile local stop` preserves mounted data/results. The existing evaluator service is still named `perception-eval` and the Compose project `cosmos3-perception-test`; both serve all three benchmark notebooks. Stopping containers does **not** stop Brev VM billing—stop the instance separately when finished. This notebook targets native Linux x86-64 on Brev, with the pinned Cosmos3-Nano endpoint or a configured compatible endpoint.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "title": "Perception Test" + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index eb31a921e..d92aa0d87 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -1,22 +1,38 @@ -# Cosmos3-Nano · Perception Test validation MCQ on Brev +# Cosmos3-Nano · Public video benchmarks on Brev -Run the public **Perception Test validation split**, from download to local top1, -inside an authenticated notebook. The default smoke is five questions (IDs 0–4) -from `video_8241`, stable indices 13404–13408. Full validation is opt-in: **19,140 -questions / 5,260 MCQ videos**. This is not the held-out test split. Source changes -and an unexecuted notebook are not evidence of hardware execution or accuracy. +The authenticated notebook overview links to **MMSI-Video-Bench** (the default +full reference run), **Perception Test**, **VideoPhy2**, and +**[your labeled video data](Customer_Data.ipynb)**. Each page runs inference +and local scoring with **Run → Run All Cells**; no manual Docker benchmark +command is required. Source notebooks contain no executed outputs. -This launchable is a standalone public-data validation workflow, not a -customer-ingestion evaluation agent. It needs no private annotation bundle, -internal dataset service, dataset credentials, or API judge. +**Customer data needs labels, not just footage:** questions, answer options and +actual correct answers, plus permission to process the videos. Read the +**[standalone customer-data recipe](CUSTOM_DATA_RECIPE.md)** before provisioning +a GPU. Existing coding agents perform the one-off conversion; this is not a +benchmark builder, ingestion agent, packaging workflow or automatic converter. + +The launchable uses no private annotations, internal dataset service or dataset +credentials, external submission, or API judge. Published Cosmos reference +configurations have not been independently reproduced; the included +**[expected pinned measurements](baselines.json)** state that boundary. ## Start on Brev Use a native Linux x86-64 GPU instance with Docker Compose, NVIDIA container -support, and enough GPU capacity for the public Nano checkpoint. Full video -archive is 75,375,143,328 bytes before extraction; provision room for the archive, -extracted videos, model cache, and outputs. Smoke downloads the approximately -225 MB official sample archive; only `video_8241` is selected for inference. +support, and enough GPU capacity for the pinned public Nano checkpoint. Put data, +model cache, and results on a disk with sufficient space **and the persistence +you need**; instance-local NVMe can be lost when a VM stops or is replaced. + +| `BENCHMARK` | Full public scope | Protocol / primary metric | Download budget before extraction | +| --- | --- | --- | --- | +| `perception-test` | Validation MCQ: 19,140 questions / 5,260 videos | 16 source-cut-aware frames; top1 accuracy | Smoke: 225 MB. Full: 75.38 GB | +| `videophy2` | Test: 3,397 video-caption pairs | 32 × 224 × 224 frames; mean SA/PC Pearson ×100, **not accuracy** | 3.93 GB publisher archive plus original URLs for absent entries; smoke may also need the archive | +| `mmsivideo-bench` | Main: 1,106 questions | Uniform-50 canonical frames plus references; official-extractor accuracy | Both scopes: about 16.1 GB frames plus references; no original videos | + +Allow additional extraction space, model cache, and outputs. A small question +smoke does not imply a small download. Perception Test uses public validation, +not the held-out test leaderboard. From this directory on the host: @@ -24,6 +40,8 @@ From this directory on the host: export LAUNCHABLE_STATE_DIR=/mnt/cosmos3-perception-test-state # Optional separate data disk; defaults to $LAUNCHABLE_STATE_DIR/perception-test. export PERCEPTION_TEST_DATA_DIR=/mnt/perception-test-data +# Other benchmark data; defaults to $LAUNCHABLE_STATE_DIR/datasets. +export BENCHMARK_DATA_DIR=/mnt/public-benchmark-data bash setup.sh --brev ``` @@ -36,19 +54,44 @@ URL in the host terminal with: docker compose exec perception-eval jupyter server list ``` -Open **Evaluate_Cosmos3.ipynb** and run all cells. The notebook downloads the -selected public data, validates coverage **before any model request**, streams -native inference progress, scores locally, and displays top1/counts, coverage, -and artifact links. Leave `RUN_FULL = False` for smoke; set it to `True` to run -full validation directly, without first rerunning smoke. There is no manual -Docker benchmark command required. Committed notebook outputs are empty. - -Compose project is `cosmos3-perception-test`; evaluator service is -`perception-eval`. Data is mounted read-write at `/perception-test`, cache at -`/cache`, and fresh run directories at `/results`. Host paths must be absolute, -outside the checkout, with data separate from cache/results. No Docker socket, -cloud credential directory, model weights, videos, annotations, or answer-bearing -results are baked into the evaluator. The model service has **no host port**. +Open **[Evaluate_Cosmos3.ipynb](Evaluate_Cosmos3.ipynb)**, the benchmark overview. +Its links work immediately, without running any cells: + +- **[Perception Test](Perception_Test.ipynb)** +- **[VideoPhy2](VideoPhy2.ipynb)** +- **[MMSI-Video-Bench](MMSI_Video_Bench.ipynb)** +- **[Your labeled video data](Customer_Data.ipynb)** — follow the + [recipe](CUSTOM_DATA_RECIPE.md) to supply canonical JSONL and configuration. + +Each notebook has its own title and an **All benchmarks** back link; it does not +repeat the overview banner or catalog. The benchmark is fixed by the page. +Choose the scope in that notebook's configuration cell: + +```python +SCOPE = "full" # MMSI default; use "smoke" only for a pipeline check +``` + +Perception Test and VideoPhy2 still default to smoke. Only a complete full run +can be compared with a full baseline. The expected metric, model revision, +scope and protocol are displayed before a full run; paper values are labeled +separately from measured launchable values. + +Run all cells in the selected benchmark notebook. It downloads pinned public +inputs, checks coverage **before model requests**, streams real progress, scores +locally, and displays its metric, coverage, warnings, protocol, and private +artifact links. All three pages reuse `benchmarks.py` and `notebook_ui.py`; +there are no separate runner implementations or additional model instances. +Result identity must match both benchmark and scope before display. Every run +gets a fresh results folder under its benchmark's own directory. +Run one notebook at a time; `API_NPROC` supports 1–4 workers. + +The existing Compose project remains `cosmos3-perception-test`; the evaluator +service remains `perception-eval`. Perception data is mounted at +`/perception-test`; other datasets at `/datasets/`; cache at `/cache`; +results at `/results//-`. Host paths must be +absolute and outside the checkout, with data separate from cache/results. +No Docker socket, cloud credential directory, model weights, videos, annotations, +or answer-bearing results are baked into the evaluator. The model has no host port. Only authenticated Jupyter is exposed on the selected private address. ### Public model and serving pins @@ -56,8 +99,17 @@ Only authenticated Jupyter is exposed on the selected private address. - Model: [`nvidia/Cosmos3-Nano`](https://huggingface.co/nvidia/Cosmos3-Nano), revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`. - Serving image: `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55`. -- Existing `CosmosReason2` generation recipe: temperature 0, seed 1, thinking - disabled, maximum 8,192 output tokens; default one API worker. +- Perception Test: existing `CosmosReason2` recipe, temperature 0, seed 1, + thinking disabled, maximum 8,192 output tokens. +- VideoPhy2: temperature 0, seed 42, thinking disabled, maximum 256 output tokens. + Every accepted response must have 16 bounded groups of 49 video tokens + (784 total), consistent with 32 frame slots and a 224×224 spatial grid under + the pinned processor profile—not proof of decoded or unique frames. + Incompatible endpoints fail the audit. +- MMSI-Video-Bench: temperature 0, seed 1, maximum 2,048 output tokens, low image + detail and processor maximum 151,200 pixels. Server thinking defaults are not + overridden. Invalid and truncated answers stay in the denominator, with issues + reported rather than selectively regenerating outputs. Public downloads do not require a Hugging Face token. For an existing compatible endpoint, export complete `COSMOS_API_BASE` (`.../v1/chat/completions`), exact @@ -66,6 +118,69 @@ endpoint, export complete `COSMOS_API_BASE` (`.../v1/chat/completions`), exact GPU serving for BYOM; do not write keys into the notebook or source. An endpoint's model display name does not prove its checkpoint revision. +The serving configuration disables FlashInfer sampling with +`VLLM_USE_FLASHINFER_SAMPLER=0`. This exact pinned vLLM build +(`0.23.1rc1.dev1306+gbfee6a802`) does **not** expose +`--trust-request-mm-kwargs`; its renderer forwards request processor overrides +unconditionally. Adding that unsupported flag would prevent startup. Effective +input evidence, not the presence of a flag, is the verification boundary. + +## Customer-data recipe and model comparison + +Follow [CUSTOM_DATA_RECIPE.md](CUSTOM_DATA_RECIPE.md) and then +[Customer_Data.ipynb](Customer_Data.ipynb). Supply a fixed external JSONL +manifest with stable IDs, videos, questions, 2–26 ordered candidates and one +full-text ground-truth answer per row. Optional category and temporal bounds +are documented. The config names the dataset, manifest/media paths, model +revision, frame/pixel budget and generation settings; no public benchmark +checksum or row-count gate is weakened. + +Validation checks every selected input before inference. Scores use local +strict single-letter MCQ accuracy, with invalid/truncated answers counted +wrong. Transport failures cannot become a completed score. Change only the +model configuration/endpoint for comparison, retain the same dataset identity, +and inspect effective input budgets. Custom results are **not** official +benchmark scores or leaderboard submissions. + +## VideoPhy2 and MMSI-Video-Bench + +`benchmarks.py` dispatches the notebook's selection. `videophy2.py` and +`mmsivideo.py` contain benchmark-specific public download, qualification, native +inference, local scoring, and provenance. Dataset/code revisions and SHA-256 +digests are pinned in those modules; no runtime credentials are committed. + +VideoPhy2 uses the official categorical 1–5 rating questions/parser and segment- +center sampling. The publisher archive covers most rows; missing entries use +only their exact official CSV URLs. All selected videos must be verified before +inference. Its two-row smoke may have undefined Pearson correlation because +ratings do not vary; `null` is intentional, not a fabricated zero or failure. +Attribution: [VideoPhy2 / Hritik Bansal and collaborators](https://github.com/Hritikbansal/videophy/tree/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2); +publisher code/data license MIT. + +MMSI uses maintained `MMSIVideoBench` prompts and Uniform-50 sampling, with +selected IDs, options, labels, prompts, frames, and references checked against the +official public annotations. Its primary scorer is the pinned publisher's +`evaluation.py` extractor. Maintained no-judge exact matching is retained only as +a diagnostic because it can reject valid verbose answers. Full scope also runs +the unmodified official scorer CLI and checks agreement. Attribution: +[MMSI-Video-Bench / Jingli Lin et al.](https://github.com/InternRobotics/MMSI-Video-Bench/tree/412f5a4a9eb6d89613c60b541b17df74f11a6736), +arXiv:2512.10863; publisher CC BY 4.0. Underlying third-party redistribution rights +are not independently cleared; retain media locally, not in the image or PR. + +Both adapters retain raw responses, exact IDs, failure/finish reasons, source and +model revision records, and measured timing in each fresh run folder. Transport +or incomplete coverage cannot produce a completed full score. Neither protocol +is claimed to reproduce the paper's Nano references (45.6 / 27.2); the exact +reference recipes remain unconfirmed. Do not average these distinct metrics. + +Equivalent evaluator-terminal commands, **not required by the notebook**: + +```bash +python benchmarks.py download --benchmark videophy2 --scope smoke --data-root /datasets/videophy2 +python benchmarks.py run --benchmark videophy2 --scope smoke --data-root /datasets/videophy2 \ + --work-dir /results/videophy2/new-smoke +``` + ## Dataset identity, attribution, and source cuts Dataset: [Perception Test](https://github.com/google-deepmind/perception_test), @@ -101,10 +216,10 @@ This is a **launchable prototype protocol**, not reference implementation parity or a published Nano score reproduction. The text-to-option parser is an adapter protocol, not a claim of official letter-parser parity. -## Controller and native APIs +## Perception-specific controller and native APIs -The notebook calls `launchable.py`; the same public controller is available -inside the evaluator terminal: +For Perception Test, `benchmarks.py` delegates to `launchable.py`. Its native +controller is also available inside the evaluator terminal: ```bash python launchable.py download --scope metadata # annotations, source cuts, README @@ -142,8 +257,8 @@ directory. Helper API in `vlmeval.perception_test_local`: ## Score existing predictions without videos or a model Start only authenticated Jupyter with `bash setup.sh --score-only` (add `--brev` -for private mesh access). Set `SCORE_ONLY = True`, choose explicit `SCORE_SCOPE`, -and set exactly one of `EXISTING_WORK` or `PREDICTIONS_FILE` in the notebook. +for private mesh access). Open **Perception_Test.ipynb**, set `SCORE_ONLY = True`, +choose `SCOPE`, and set exactly one of `EXISTING_WORK` or `PREDICTIONS_FILE`. The notebook downloads **metadata only** and prepares with `require_media=False`; video files, model construction, endpoint requests and GPU are not required. Set `DOWNLOAD_PUBLIC = False` to rescore completely offline @@ -168,12 +283,29 @@ dummy answers, random answers, or partial-full-score fallback. ## Results and lifecycle -The notebook shows one concise human result and links to the score summary, -run provenance and log. `run.json` records source `perception-test-local`, scope, -annotation and media fingerprints, source hashes, model/source revisions, -generation parameters, frame protocol, and UTC start/end plus measured elapsed -seconds. Score artifacts include `summary.json` and `controller.json` with local -scoring timing/provenance. No cost or peak-memory estimate is invented. +The notebook shows one identity-checked result card with metric units, selected +coverage, measured time, warnings, protocol, and private artifact links. Every +benchmark has a `summary.json`; Perception's original native report remains at +`scores/summary.json`. Score-only Perception retains its native `summary.json` +and writes the notebook report separately as `notebook-summary.json`. + +Each benchmark retains its own provenance and predictions. Perception's +`run.json` and `scores/controller.json` additionally record native annotation/cut +identities, source/model revisions, frame/generation protocol, and timing. +VideoPhy2 and MMSI retain `provenance.json` plus per-response evidence and local +scorer diagnostics. No cost or peak-memory estimate is invented. + +`execution-summary.json` aggregates per-response **requested settings and +observed input evidence**. Supported explicit processor profiles allow +conditional derivation of image-grid pixel areas and video frame slots from +bounded visual-token groups. Frame slots are not proof of decoded/unique +frames; area is not width/height/crop. Effective generation settings and +checkpoint revision are not attested by ordinary chat responses and remain +unknown. Unknown evidence is never presented as verification. Customer runs +also retain `requested-execution.json` and their immutable data identity. + +Reviewed aggregate expected measurements in `baselines.json` are deliberately +shipped; label-bearing runtime reports, media and predictions are not. ```bash # Host terminal: preserve mounted data/results while stopping containers. @@ -184,20 +316,33 @@ Stopping containers does not stop underlying Brev VM billing; stop the VM separately after retaining needed results. Do not commit credentials, videos, labels, prediction tables, or answer-bearing artifacts. -## Observed validation · 2026-10-30 - -- On an existing Brev instance with one RTX PRO 6000 Blackwell Server Edition - (96 GB), the actual Jupyter **Run All Cells** completed all four code cells. - Nano returned five valid choices for the exact smoke IDs; local top1 was - **4/5 (80%)**, with zero malformed answers. -- First local score was produced about **44 seconds** after Run All, using an - already-running model and staged smoke inputs. Controller inference took - 20.53 seconds and local scoring 0.74 seconds. This is **not a cold launch**. -- A separate CPU container with **network disabled and no videos or GPU** - rescored the actual predictions identically, including diagnostic counts and - annotation/prediction identities. -- All **43 native behavioral checks** passed in the Linux CPU evaluator. - -These observations cover five questions from one video, not full validation, -the held-out test leaderboard, reference-score parity, BYOM, or a fresh-VM -launch. Driver was 595.112.07; peak memory and cost were not measured. +## Kaizen UI presentation and developer skills + +The notebook adapts **Kaizen UI 1.19** typography, semantic color tokens, +density spacing, entity cards, status labels, and read-only key/value patterns +to native notebook HTML in `notebook_ui.py`. Jupyter owns the application shell +and light/dark theme. No global CSS reset, downloaded fonts, React application, +CDN request, or Node runtime is required to run the notebook. + +The benchmark configuration stays visible. Helper, execution, and result-rendering +inputs are collapsed by default and can be expanded. Native keyboard-accessible +disclosures retain the full protocol, attribution, and scoring notes. +After updating an already-open notebook, restart its kernel before Run All so +Python loads the updated presentation helper. + +The overview is static Markdown/HTML, so its navigation does not depend on code +execution or saved outputs. Its conservative markup uses Jupyter's native link +colors because Jupyter sanitizes Markdown CSS. The richer run/result panels +remain scoped to `notebook_ui.py`; no application shell is duplicated. + +Optional design-tool setup, from the **Framework repository root**: + +```bash +npm ci +``` + +The root `package.json` pins `@nvidia/foundations-react-core` as a development-only +dependency. Its `prepare` script runs `kui-sync-skills --dir .agents`, keeping +`.agents/skills/kaizen-ui` in sync; `kui.disablePostInstall` suppresses the hint. +Keep the lockfile in source control; `node_modules` is excluded from Git and Docker. + diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/VideoPhy2.ipynb b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/VideoPhy2.ipynb new file mode 100644 index 000000000..38b12e122 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/VideoPhy2.ipynb @@ -0,0 +1,217 @@ +{ + "cells": [ + { + "id": "welcome", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[← All benchmarks](Evaluate_Cosmos3.ipynb)\n", + "\n", + "# VideoPhy2\n", + "\n", + "Rate caption alignment and physical plausibility, then compare with the published human ratings.\n", + "\n", + "Choose smoke or full scope below, then select **Run → Run All Cells**.\n" + ] + }, + { + "id": "shared-helpers", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "from notebook_ui import show_plan, show_results, invoke\n", + "\n", + "# This page always runs VideoPhy2. Use All benchmarks to switch.\n", + "BENCHMARK = \"videophy2\"\n", + "SCORE_ONLY = False\n", + "EXISTING_WORK = None\n", + "PREDICTIONS_FILE = None\n" + ] + }, + { + "id": "configure", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 01 / Configure the run\n", + "\n", + "Keep `SCOPE = \"smoke\"` for a quick pipeline check; choose `\"full\"` for the complete VideoPhy2 benchmark. Each attempt gets a fresh timestamped results folder.\n" + ] + }, + { + "id": "settings", + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from benchmarks import validate_selection\n", + "\n", + "SCOPE = os.environ.get(\"EVAL_SCOPE\", \"smoke\") # \"smoke\" or \"full\"\n", + "DOWNLOAD_PUBLIC = True # False only when the selected pinned inputs are staged.\n", + "API_NPROC = 1 # 1–4 simultaneous API workers; do not run concurrent notebooks.\n", + "\n", + "\n", + "# Persistent mounts are separate from source. Override these for another runtime.\n", + "DATA_ROOT = Path(os.environ.get(\"BENCHMARK_DATA_DIR\", \"/datasets\"))\n", + "RESULTS_ROOT = Path(os.environ.get(\"EVAL_RESULTS_DIR\", \"/results\"))\n", + "DATA = (Path(os.environ.get(\"PERCEPTION_TEST_DATA_DIR\", \"/perception-test\"))\n", + " if BENCHMARK == \"perception-test\" else DATA_ROOT / BENCHMARK)\n", + "LAUNCHABLE = Path.cwd()\n", + "CLI = [sys.executable, \"-u\", str(LAUNCHABLE / \"benchmarks.py\")]\n", + "STAMP = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%S%fZ\")\n", + "WORK = RESULTS_ROOT / BENCHMARK / f\"{SCOPE}-{'score-' if SCORE_ONLY else ''}{STAMP}\"\n", + "LOG = WORK.with_suffix(\".log\")\n", + "\n", + "validate_selection(BENCHMARK, SCOPE, SCORE_ONLY)\n", + "if not 1 <= API_NPROC <= 4:\n", + " raise ValueError(\"Choose API_NPROC between 1 and 4.\")\n", + "if SCORE_ONLY and (EXISTING_WORK is None) == (PREDICTIONS_FILE is None):\n", + " raise ValueError(\"Set exactly one of EXISTING_WORK or PREDICTIONS_FILE.\")\n", + "if not (LAUNCHABLE / \"benchmarks.py\").is_file():\n", + " raise ValueError(\"Run this notebook from its launchable directory.\")\n", + "show_plan(BENCHMARK, SCOPE, DATA, WORK, API_NPROC, SCORE_ONLY)\n" + ] + }, + { + "id": "download", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 02 / Download the public inputs\n", + "\n", + "Downloads use pinned publisher URLs and checksums, then safe extraction. Existing verified assets are reused. **Check the download budget above before running:** a small smoke can still require a large source archive, especially MMSI-Video-Bench. No Hugging Face token or private NVIDIA dataset is required.\n", + "\n", + "Inputs stay on your instance. Keep publisher attribution and applicable license terms; public access is not permission to redistribute every underlying video.\n" + ] + }, + { + "id": "fetch", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "if DOWNLOAD_PUBLIC:\n", + " command = CLI + [\"download\", \"--benchmark\", BENCHMARK,\n", + " \"--scope\", SCOPE, \"--data-root\", str(DATA)]\n", + " if SCORE_ONLY:\n", + " command += [\"--metadata-only\"]\n", + " invoke(command, WORK.with_suffix(\".download.log\"))\n", + "else:\n", + " print(\"Downloads skipped. The runner will validate the selected staged inputs before inference.\")\n" + ] + }, + { + "id": "execute", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 03 / Run inference and score locally\n", + "\n", + "This cell checks selected media/annotation coverage before model requests, streams real progress below, and writes a durable log. It then applies the selected benchmark’s local scorer. **No external scoring submission, external judge, or manual benchmark command.**\n", + "\n", + "**VideoPhy2 protocol:** 32 × 224 × 224 video frames, with a per-response visual-token audit; categorical semantic/physics ratings and Pearson correlations. A two-pair smoke can legitimately have undefined correlation because ratings do not vary.\n", + "\n", + "Run one benchmark at a time. Full runs can take hours. Interrupting leaves diagnostics; rerun the configuration cell before a new attempt to get a fresh output folder.\n" + ] + }, + { + "id": "infer-and-score", + "cell_type": "code", + "metadata": { + "scrolled": true, + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "command = CLI + [\"run\", \"--benchmark\", BENCHMARK, \"--scope\", SCOPE,\n", + " \"--data-root\", str(DATA), \"--work-dir\", str(WORK),\n", + " \"--api-nproc\", str(API_NPROC)]\n", + "if SCORE_ONLY:\n", + " command += [\"--score-only\"]\n", + " command += ([\"--existing-work\", str(EXISTING_WORK)] if EXISTING_WORK is not None\n", + " else [\"--predictions\", str(PREDICTIONS_FILE)])\n", + "invoke(command, LOG)\n" + ] + }, + { + "id": "review", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 04 / Read the result, retain the evidence\n", + "\n", + "The heading below is checked against the result artifact’s benchmark and scope. Accuracy and correlation are not the same metric; do not combine scores across benchmarks. **Smoke results are not full benchmark results.**\n", + "\n", + "Expand the protocol record to inspect the exact setting. JSON reports, prediction evidence, source/dataset revisions, and logs remain in the benchmark’s own result folder. Links below are private runtime artifacts—not files to commit or publish.\n" + ] + }, + { + "id": "results", + "cell_type": "code", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "execution_count": null, + "outputs": [], + "source": [ + "SUMMARY = WORK / (\"notebook-summary.json\" if SCORE_ONLY else \"summary.json\")\n", + "summary = json.loads(SUMMARY.read_text())\n", + "show_results(summary, BENCHMARK, SCOPE, WORK, LOG)\n" + ] + }, + { + "id": "handoff", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reviewer checklist / Before sharing with Dan\n", + "\n", + "- Confirm the selected dataset, split, frame/pixel budget, generation settings, and primary scorer are the intended protocol.\n", + "- Inspect coverage, invalid/truncated answers, source/data/model revisions, and the retained logs. A matching score alone does not establish reproduction of a published result.\n", + "- To switch benchmarks, use [← All benchmarks](Evaluate_Cosmos3.ipynb) and open another notebook. Finish or interrupt this run first; closing its tab does not stop its kernel. For a new attempt here, use **Restart Kernel and Run All Cells**. Verified inputs are cached; new results are isolated.\n", + "- Share code and a reviewed aggregate result, not videos, labels, raw predictions, endpoint credentials, or executed notebook outputs. Clear all outputs before committing.\n", + "\n", + "### Stop safely\n", + "\n", + "In the **host terminal**, `docker compose --profile local stop` preserves mounted data/results. The existing evaluator service is still named `perception-eval` and the Compose project `cosmos3-perception-test`; both serve all three benchmark notebooks. Stopping containers does **not** stop Brev VM billing—stop the instance separately when finished. This notebook targets native Linux x86-64 on Brev, with the pinned Cosmos3-Nano endpoint or a configured compatible endpoint.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "title": "VideoPhy2" + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/baselines.json b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/baselines.json new file mode 100644 index 000000000..b47e932bb --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/baselines.json @@ -0,0 +1,48 @@ +{ + "format_version": 1, + "primary_benchmark": "mmsivideo-bench", + "model": "nvidia/Cosmos3-Nano", + "model_revision": "e59a53c25979a090fa8706c9acc0c254a6e89b92", + "serving_image": "vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55", + "scope": "full", + "measurement_date": "2026-10-30", + "meaning": "Recorded full-run measurements of the launchable protocols. Smoke subsets have no performance target. These values are not an assertion that the paper's unpublished reference configurations were reproduced.", + "paper_source": "https://arxiv.org/html/2606.02800v4#S6.T10", + "benchmarks": { + "perception-test": { + "metric": "top1", + "unit": "percent", + "value": 74.38871473354232, + "correct": 14238, + "examples": 19140, + "videos": 5260, + "protocol": "Public validation MCQ; 16 source-cut-aware images; aspect-preserving maximum side 768; temperature 0, seed 1, thinking disabled; local top1.", + "paper_value": null, + "paper_note": "No Perception Test reference in the cited Cosmos3 reasoner table; this is a measured launchable baseline, not published-Cosmos-score reproduction.", + "verification": "Complete coverage and independent label/score reduction; identical offline rescore. Historical measurement, not a fresh-VM timing." + }, + "videophy2": { + "metric": "mean_sa_pc_pearson_x100", + "unit": "pearson_x100", + "value": 48.13230090428659, + "examples": 3397, + "requests": 6794, + "protocol": "32 frames at 224x224; temperature 0, seed 42, thinking disabled; categorical 1-5 ratings; mean semantic-alignment/physical-commonsense Pearson correlation times 100; audited visual input.", + "paper_value": 45.6, + "paper_note": "Numeric difference is in correlation points, not accuracy percentage points. Exact paper prompt, preprocessing and scalar-rating conversion have not been confirmed.", + "verification": "Complete coverage; independent centered-covariance reduction. Earlier incorrect four-frame diagnostics were excluded." + }, + "mmsivideo-bench": { + "metric": "official_accuracy", + "unit": "percent", + "value": 27.93851717902351, + "correct": 309, + "examples": 1106, + "measurement_date": "2026-10-30", + "protocol": "Uniform-50 canonical frames plus references; maximum processor pixels 151200; temperature 0, seed 1; 2048 output tokens; official pinned local answer extractor; invalid/truncated outputs included.", + "paper_value": 27.2, + "paper_note": "Close numerically, but the paper's exact frame/pixel/generation/scoring configuration is not public in the cited table; no identical-reference-protocol claim.", + "verification": "Complete coverage; official scorer CLI and independent count reduction agree. Eight invalid answers, including seven truncated responses, remain counted." + } + } +} diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/benchmarks.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/benchmarks.py new file mode 100755 index 000000000..807913b67 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/benchmarks.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Public benchmark selection for the Cosmos3 evaluation notebook.""" +from __future__ import annotations + +import argparse +import importlib +import json +from pathlib import Path +import time + +CATALOG = { + "perception-test": { + "title": "Perception Test", + "question": "What happened in the video?", + "metric": "Multiple-choice accuracy", + "scope": "Public validation MCQ · 19,140 questions / 5,260 videos", + "smoke": "Five fixed questions from video_8241", + "download": "Smoke: 225 MB archive. Full: 75.38 GB archive, plus extraction space.", + "protocol": "16 source-cut-aware frames · aspect-preserving maximum side 768 · local top1", + "source": "https://github.com/google-deepmind/perception_test", + "license": "Google DeepMind / Perception Test authors · code Apache-2.0, other materials CC BY 4.0", + }, + "videophy2": { + "title": "VideoPhy2", + "question": "Does the video match its caption and obey physics?", + "metric": "Mean SA / PC Pearson correlation ×100", + "scope": "Public test · 3,397 video-caption pairs / 6,794 rating requests", + "smoke": "Two fixed video-caption pairs; four rating requests", + "download": "Publisher archive: 3.93 GB, plus original public URLs for absent entries and extraction space. Smoke may also need this archive.", + "protocol": "32 audited video frames · 224 × 224 · official categorical 1–5 ratings", + "source": "https://github.com/Hritikbansal/videophy/tree/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2", + "license": "VideoPhy2 / Hritik Bansal and collaborators · publisher code and data MIT", + }, + "mmsivideo-bench": { + "title": "MMSI-Video-Bench", + "question": "Can the model reason about space across video views?", + "metric": "Official-extractor answer accuracy", + "scope": "Public main benchmark · 1,106 questions", + "smoke": "One fixed question from each of five task families", + "download": "Both scopes: about 16.1 GB compressed canonical frames, plus references and extraction space. Original videos are not needed.", + "protocol": "Uniform-50 canonical frames plus references · 2,048-token cap · local official scoring", + "source": "https://github.com/InternRobotics/MMSI-Video-Bench/tree/412f5a4a9eb6d89613c60b541b17df74f11a6736", + "license": "MMSI-Video-Bench authors · publisher CC BY 4.0; underlying third-party redistribution rights not independently cleared", + }, +} + + +def validate_selection(benchmark, scope, score_only=False): + if benchmark not in CATALOG: + raise ValueError(f"Choose BENCHMARK from {', '.join(CATALOG)}") + if scope not in {"smoke", "full"}: + raise ValueError("Choose SCOPE = 'smoke' or 'full'") + if score_only and benchmark != "perception-test": + raise ValueError("Notebook score-only mode currently supports Perception Test; use infer-and-score for the other benchmarks") + + +def download(benchmark, data_root, scope, metadata_only=False): + validate_selection(benchmark, scope, score_only=metadata_only) + data_root = Path(data_root).resolve() + if benchmark == "perception-test": + from vlmeval.perception_test_local import fetch_dataset + + return fetch_dataset(data_root, scope="metadata" if metadata_only else scope) + module = importlib.import_module("videophy2" if benchmark == "videophy2" else "mmsivideo") + return module.download(data_root, scope) + + +def run(benchmark, data_root, scope, work, api_nproc=1, score_only=False, + existing_work=None, predictions=None): + validate_selection(benchmark, scope, score_only) + if type(api_nproc) is not int or not 1 <= api_nproc <= 4: + raise ValueError("API_NPROC must be an integer between 1 and 4") + data_root, work = Path(data_root).resolve(), Path(work).resolve() + if work.exists(): + raise ValueError(f"Choose a fresh work directory; refusing to overwrite {work}") + if work == data_root or work.is_relative_to(data_root) or data_root.is_relative_to(work): + raise ValueError("Keep benchmark data and run outputs in separate directories") + if score_only and (existing_work is None) == (predictions is None): + raise ValueError("Score-only requires exactly one of EXISTING_WORK or PREDICTIONS_FILE") + if not score_only and (existing_work is not None or predictions is not None): + raise ValueError("Existing prediction inputs require SCORE_ONLY = True") + if benchmark != "perception-test": + import launchable + + launchable.endpoint(wait=1800) + module = importlib.import_module("videophy2" if benchmark == "videophy2" else "mmsivideo") + return module.run(data_root, scope, work, api_nproc=api_nproc) + + import launchable + + clock = time.monotonic() + if score_only: + native = launchable.score(data_root, scope, work=existing_work, predictions=predictions, + score_dir=work) + else: + state = launchable.run(work, "all", scope == "smoke", False, data_root, + api_nproc=api_nproc, nframe=16) + native = json.loads((work / "scores" / "summary.json").read_text()) + metric = native["primary_metric"] + summary = { + "benchmark": benchmark, "scope": scope, "status": "completed", + "primary_metric": {**metric, "value": metric["value"] * 100, "unit": "percent"}, + "coverage": {"expected": native["coverage"]["selected_rows"], "scored": metric["total"]}, + "invalid_count": native["invalid_count"], + "protocol": launchable.PROTOCOL, + "warnings": ["Public validation split, not the held-out test leaderboard.", + "Adapter frame/answer-parsing protocol; no published-score parity claim."], + "reference_parity": False, "elapsed_seconds": time.monotonic() - clock, + "score_only": score_only, + "per_reasoning": native["per_reasoning"], + "native_summary": "summary.json" if score_only else "scores/summary.json", + } + if not score_only: + summary["execution_evidence"] = state.get("execution_evidence") + summary["model"] = state["model"] + summary["model_revision"] = state["model_revision"] + # Preserve the native score-only artifact; the notebook has its own report name. + report = work / ("notebook-summary.json" if score_only else "summary.json") + report.write_text(json.dumps(summary, indent=2, allow_nan=False) + "\n") + print(f"Notebook result: {report}", flush=True) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + fetch = sub.add_parser("download", help="Download checksum-pinned public data") + fetch.add_argument("--metadata-only", action="store_true") + runner = sub.add_parser("run", help="Infer and score the selected public benchmark") + runner.add_argument("--work-dir", type=Path, required=True) + runner.add_argument("--api-nproc", type=int, default=1) + runner.add_argument("--score-only", action="store_true") + sources = runner.add_mutually_exclusive_group() + sources.add_argument("--existing-work", type=Path) + sources.add_argument("--predictions", type=Path) + for command in (fetch, runner): + command.add_argument("--benchmark", choices=CATALOG, required=True) + command.add_argument("--scope", choices=["smoke", "full"], default="smoke") + command.add_argument("--data-root", type=Path, required=True) + args = parser.parse_args() + if args.command == "download": + download(args.benchmark, args.data_root, args.scope, args.metadata_only) + else: + run(args.benchmark, args.data_root, args.scope, args.work_dir, + api_nproc=args.api_nproc, score_only=args.score_only, + existing_work=args.existing_work, predictions=args.predictions) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml index 2f90c9820..619fc6c21 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -17,6 +17,7 @@ services: ipc: host environment: HF_HOME: /cache/huggingface + VLLM_USE_FLASHINFER_SAMPLER: '0' volumes: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache # Deliberately no host port: only the evaluator can reach this API. @@ -36,6 +37,8 @@ services: - --enable-chunked-prefill - --mm-processor-cache-gb - '0' + # This pinned 0.23.1rc1.dev1306+gbfee6a802 build accepts per-request + # mm_processor_kwargs unconditionally; --trust-request-mm-kwargs is absent. - --mm-encoder-tp-mode - data - --media-io-kwargs @@ -66,6 +69,7 @@ services: JUPYTER_TOKEN: ${JUPYTER_TOKEN:?Run bash setup.sh to generate a Jupyter token} LMUData: /cache/lmudata PERCEPTION_TEST_DATA_DIR: /perception-test + BENCHMARK_DATA_DIR: /datasets EVAL_SCORE_ONLY: ${EVAL_SCORE_ONLY:-0} AWS_CONFIG_FILE: /dev/null AWS_SHARED_CREDENTIALS_FILE: /dev/null @@ -74,6 +78,7 @@ services: - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/cache:/cache - ${LAUNCHABLE_STATE_DIR:?Set an external absolute state directory}/results:/results - ${PERCEPTION_TEST_DATA_DIR:?Set an external writable Perception Test data directory}:/perception-test + - ${BENCHMARK_DATA_DIR:?Set an external writable benchmark data directory}:/datasets healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8888/login', timeout=5)"] interval: 15s diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/custom_eval.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/custom_eval.py new file mode 100644 index 000000000..70131025a --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/custom_eval.py @@ -0,0 +1,340 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Local labeled customer-video MCQ evaluation; no dataset conversion or judge.""" + +import argparse +import concurrent.futures +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import tempfile +import time +import urllib.parse + +SYSTEM_PROMPT = "Answer the multiple-choice video question with only its single uppercase option letter." + + +def _json(path, value): + Path(path).write_text(json.dumps(value, indent=2, allow_nan=False) + "\n", encoding="utf-8") + + +def _digest(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _identity(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()).hexdigest() + + +def _external(path): + path = Path(path).expanduser().resolve() + source = next((p for p in Path(__file__).resolve().parents if (p / ".git").exists()), Path(__file__).resolve().parents[1]) + if path.is_relative_to(source) or source.is_relative_to(path): + raise ValueError("Customer inputs and results must be outside the source checkout") + return path + + +def _text(value, name): + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{name} must be a nonempty string without surrounding whitespace") + return value + + +def _number(value, name, minimum=0): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < minimum: + raise ValueError(f"{name} must be finite and >= {minimum}") + return value + + +def _integer(value, name, minimum=1): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + return value + + +def validate_row(row, media_root, seen): + """Validate one canonical annotation without changing its label or option order.""" + if not isinstance(row, dict) or set(row) - {"id", "video", "question", "candidates", "answer", "category", "start", "end"}: + raise ValueError("Row must use only canonical fields") + row = dict(row) + sample_id = _text(row.get("id"), "id") + if sample_id in seen: + raise ValueError(f"Duplicate id: {sample_id}") + seen.add(sample_id) + _text(row.get("question"), "question") + options = row.get("candidates") + if not isinstance(options, list) or not 2 <= len(options) <= 26: + raise ValueError("candidates must contain 2–26 options") + for option in options: + _text(option, "candidate") + if len(set(options)) != len(options): + raise ValueError("Duplicate candidate text") + if not isinstance(row.get("answer"), str) or options.count(row["answer"]) != 1: + raise ValueError("answer must equal exactly one full candidate text") + if "category" in row: + _text(row["category"], "category") + relative = _text(row.get("video"), "video") + parts = PurePosixPath(relative) + if parts.is_absolute() or ".." in parts.parts or "\\" in relative or ":" in relative: + raise ValueError("video must be a safe relative POSIX path within media_root") + path = (media_root / relative).resolve() + if not path.is_relative_to(media_root) or not path.is_file() or path.stat().st_size <= 0: + raise ValueError(f"Missing, empty or escaping media for id {sample_id}") + _external(path) + if ("start" in row) != ("end" in row): + raise ValueError("start and end must be supplied together in seconds") + if "start" in row: + _number(row["start"], "start") + _number(row["end"], "end") + if row["end"] <= row["start"]: + raise ValueError("end must exceed start") + return row, path + + +def probe_video(path): + from decord import VideoReader, cpu + + reader = VideoReader(str(path), ctx=cpu(0), num_threads=1) + fps, count = float(reader.get_avg_fps()), len(reader) + if count < 1 or not math.isfinite(fps) or fps <= 0: + raise ValueError("Video has no usable frames or FPS") + reader.get_batch([0, count - 1]) + return {"source_frames": count, "source_fps": fps, "duration_seconds": count / fps} + + +def _load(config_path): + config_path = _external(config_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + allowed = {"dataset", "manifest", "media_root", "nframes", "max_pixels", "min_pixels", "image_patch_size", "model", "model_revision", "generation", "processor_profile"} + if not isinstance(config, dict) or set(config) - allowed: + raise ValueError("Unknown config fields") + dataset = config.get("dataset") + if not isinstance(dataset, dict) or set(dataset) != {"name", "revision", "source"}: + raise ValueError("dataset requires name, revision, source") + for key, value in dataset.items(): + _text(value, "dataset." + key) + manifest = _external(config_path.parent / _text(config.get("manifest"), "manifest")) + root = _external(config_path.parent / _text(config.get("media_root"), "media_root")) + frames = _integer(config.get("nframes"), "nframes", 4) + if frames % 2: + raise ValueError("nframes must be even for the native video processor") + maximum = _integer(config.get("max_pixels"), "max_pixels") + minimum = _integer(config.get("min_pixels"), "min_pixels") + if minimum > maximum: + raise ValueError("min_pixels must not exceed max_pixels") + _integer(config.get("image_patch_size"), "image_patch_size") + model = _text(config.get("model", os.environ.get("COSMOS_MODEL")), "model or COSMOS_MODEL") + revision = _text(config.get("model_revision", os.environ.get("COSMOS_MODEL_REVISION", "unknown")), "model_revision") + generation = {"temperature": 0, "seed": 42, "max_tokens": 32, "enable_thinking": False} + supplied = config.get("generation", {}) + if not isinstance(supplied, dict) or set(supplied) - set(generation): + raise ValueError("Unknown generation fields") + generation.update(supplied) + _number(generation["temperature"], "temperature") + _integer(generation["seed"], "seed", 0) + _integer(generation["max_tokens"], "max_tokens") + if not isinstance(generation["enable_thinking"], bool): + raise ValueError("enable_thinking must be boolean") + if config.get("processor_profile") is not None and not isinstance(config["processor_profile"], dict): + raise ValueError("processor_profile must be an explicit evidence-profile object") + rows, media, seen = [], [], set() + lines = manifest.read_text(encoding="utf-8").split("\n") + if lines and lines[-1] == "": + lines.pop() + for line in lines: + if not line.strip(): + raise ValueError("Blank JSONL lines are not canonical rows") + row, path = validate_row(json.loads(line), root, seen) + info = probe_video(path) + if "end" in row and row["end"] > info["duration_seconds"]: + raise ValueError(f"Temporal end exceeds video duration for {row['id']}") + rows.append(row) + media.append({"id": row["id"], "video": row["video"], "sha256": _digest(path), **info}) + if not rows: + raise ValueError("Manifest must contain at least one labeled row") + data = {"dataset": dataset, "rows": rows, "media": media} + report = {"status": "validated", "benchmark": "customer-video-mcq", "scope": "custom", "dataset": dataset, + "protocol": {"name": "customer_video_mcq_native_v1", "scorer": "strict_single_uppercase_letter", + "truncation_policy": "wrong_without_retry", "official_benchmark": False}, + "dataset_identity": _identity(data), "manifest_sha256": _digest(manifest), "expected": len(rows), + "media": media, "model": {"name": model, "revision": revision}, + "generation": generation, "sampling": {k: config[k] for k in ("nframes", "min_pixels", "max_pixels", "image_patch_size")}} + return config, root, rows, report + + +def validate(config_path) -> dict: + """Validate all labels, media, IDs and temporal bounds without endpoint requests.""" + return _load(config_path)[3] + + +def build_payload(row, video_url, video_kwargs, settings): + """Only video, question and candidate text cross the inference boundary.""" + question = row["question"] + "\n" + "\n".join(f"{chr(65 + i)}. {option}" for i, option in enumerate(row["candidates"])) + return {"model": settings["model"]["name"], "messages": [{"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": [{"type": "video_url", "video_url": {"url": video_url}}, {"type": "text", "text": question}]}], + **{k: settings["generation"][k] for k in ("temperature", "seed", "max_tokens")}, + "chat_template_kwargs": {"enable_thinking": settings["generation"]["enable_thinking"]}, + "mm_processor_kwargs": video_kwargs, + "media_io_kwargs": {"video": {"num_frames": settings["sampling"]["nframes"], + "min_frames": settings["sampling"]["nframes"], + "max_frames": settings["sampling"]["nframes"]}}, + "return_token_ids": True} + + +def score_answer(row, content, finish_reason): + """Strict single-letter exact match; malformed and truncated outputs count wrong.""" + predicted = content.strip() if isinstance(content, str) else "" + valid = len(predicted) == 1 and "A" <= predicted < chr(65 + len(row["candidates"])) and finish_reason == "stop" + correct = bool(valid and row["candidates"][ord(predicted) - 65] == row["answer"]) + return {"prediction": predicted if len(predicted) == 1 and "A" <= predicted <= "Z" else None, + "valid": valid, "correct": correct, "finish_reason": finish_reason if finish_reason in ("stop", "length") else None} + + +def summarize(rows, results): + """Complete coverage is mandatory; transport failures never become wrong answers.""" + by_id = {item["id"]: item for item in results} + if len(by_id) != len(results) or set(by_id) - {row["id"] for row in rows}: + raise ValueError("Unexpected or duplicate result IDs") + scored = [row for row in rows if row["id"] in by_id and "error" not in by_id[row["id"]]] + correct = sum(bool(by_id[row["id"]]["correct"]) for row in scored) + categories = {} + for row in scored: + category = row.get("category", "uncategorized") + metric = categories.setdefault(category, {"correct": 0, "total": 0}) + metric["total"] += 1 + metric["correct"] += bool(by_id[row["id"]]["correct"]) + for metric in categories.values(): + metric["accuracy_percent"] = 100 * metric["correct"] / metric["total"] + complete = len(scored) == len(rows) + return {"status": "completed" if complete else "failed", "coverage": {"expected": len(rows), "scored": len(scored)}, + "primary_metric": {"name": "accuracy_percent", "value": 100 * correct / len(rows) if complete else None, + "unit": "percent", "correct": correct, "total": len(rows)}, "categories": categories} + + +def run(config_path, work, api_nproc=1) -> dict: + """Preflight every selected video before sending any request; write fresh local evidence.""" + from vlmeval.execution_evidence import aggregate_execution_records + + _integer(api_nproc, "api_nproc") + if api_nproc > 4: + raise ValueError("api_nproc must be at most 4") + config, root, rows, validated = _load(config_path) + api = os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions") + parts = urllib.parse.urlsplit(api) + if parts.scheme not in ("http", "https") or not parts.netloc or not parts.path.endswith("/chat/completions"): + raise ValueError("COSMOS_API_BASE must be the complete /v1/chat/completions URL") + timeout = float(os.environ.get("COSMOS_TIMEOUT", "300")) + _number(timeout, "COSMOS_TIMEOUT", 0.001) + key = os.environ.get("COSMOS_API_KEY", "EMPTY") + work = _external(work) + work.mkdir(parents=True, exist_ok=False) + (work / "responses").mkdir() + started = time.monotonic() + summary = {**validated, **summarize(rows, []), + "requested_execution": "requested-execution.json", "effective_execution": "responses/*.json#execution", + "results": "results.jsonl", "workers": api_nproc} + results = [] + _json(work / "validated.json", validated) + try: + import requests + from vlmeval.api.cosmos_reason import process_video_info_to_video_url + from vlmeval.execution_evidence import build_execution_record, nano_processor_profile + + profile = config.get("processor_profile") or nano_processor_profile(validated["model"]["name"], validated["model"]["revision"]) + _json(work / "requested-execution.json", {"model": validated["model"], "generation": validated["generation"], + "sampling": validated["sampling"], "processor_profile": profile}) + + with tempfile.TemporaryDirectory(prefix="custom-transport-", dir=work) as temporary: + prepared = [] + # Native decode/resize/temporal handling, fully preflighted before first request. + for index, row in enumerate(rows): + kwargs = {k: config[k] for k in ("nframes", "min_pixels", "max_pixels")} + if "start" in row: + kwargs.update(video_start=row["start"], video_end=row["end"]) + url, video_kwargs = process_video_info_to_video_url(str(root / row["video"]), image_patch_size=config["image_patch_size"], use_cache=False, **kwargs) + if _digest(root / row["video"]) != validated["media"][index]["sha256"]: + raise ValueError("Selected media changed during preflight; restore the fixed dataset") + transport = Path(temporary) / f"{index:06d}.txt" + transport.write_text(url, encoding="utf-8") + prepared.append((index, row, transport, video_kwargs)) + + def predict(item): + index, row, transport, video_kwargs = item + evidence_path = f"responses/{index:06d}.json" + result = {"id": row["id"], "category": row.get("category", "uncategorized"), "evidence": evidence_path} + record = {} + try: + payload = build_payload(row, transport.read_text(encoding="utf-8"), video_kwargs, validated) + response = requests.post(api, json=payload, headers={"Authorization": f"Bearer {key}"}, timeout=timeout) + record["http_status"] = response.status_code + response.raise_for_status() + raw = response.json() + if not isinstance(raw, dict): + raise ValueError("Non-object response") + record["execution"] = build_execution_record(payload, raw, + media={"expected_frame_slots": config["nframes"]}, processor_profile=profile, + requested_model_revision=validated["model"]["revision"]) + choices = raw.get("choices") + if "error" in raw or (isinstance(choices, list) and any( + isinstance(choice, dict) and choice.get("finish_reason") in ("error", "abort") + for choice in choices)): + raise ValueError("Endpoint reported an unsuccessful generation") + # Successful HTTP responses with malformed answers remain in denominator. + choice = choices[0] if isinstance(choices, list) and len(choices) == 1 and isinstance(choices[0], dict) else {} + message = choice.get("message") + content = message.get("content") if isinstance(message, dict) else None + result.update(score_answer(row, content, choice.get("finish_reason"))) + # Store only the selected answer, not arbitrary raw endpoint fields. + record["answer"] = {k: result[k] for k in ("prediction", "valid", "correct", "finish_reason")} + except Exception as exc: + # Exception strings/URLs/raw responses may contain secrets or prompt echoes. + result["error"] = type(exc).__name__ + record["error"] = type(exc).__name__ + _json(work / evidence_path, record) + return result + + with concurrent.futures.ThreadPoolExecutor(max_workers=api_nproc) as pool: + for result in pool.map(predict, prepared): + results.append(result) + with (work / "results.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(result, allow_nan=False) + "\n") + summary.update(summarize(rows, results)) + if summary["status"] != "completed": + raise RuntimeError("Customer evaluation incomplete; inspect summary.json and responses") + except Exception as exc: + summary["status"] = "failed" + summary["failure"] = type(exc).__name__ + raise + finally: + records = [json.loads(path.read_text(encoding="utf-8")).get("execution") + for path in sorted((work / "responses").glob("*.json"))] + summary["execution_evidence"] = aggregate_execution_records(records, expected_requests=len(rows)) + summary["execution_summary"] = "execution-summary.json" + _json(work / "execution-summary.json", summary["execution_evidence"]) + summary["elapsed_seconds"] = time.monotonic() - started + _json(work / "summary.json", summary) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("validate") + check.add_argument("--config", type=Path, required=True) + execute = sub.add_parser("run") + execute.add_argument("--config", type=Path, required=True) + execute.add_argument("--work-dir", type=Path, required=True) + execute.add_argument("--api-nproc", type=int, default=1) + args = parser.parse_args() + result = validate(args.config) if args.command == "validate" else run(args.config, args.work_dir, args.api_nproc) + print(json.dumps(result, indent=2, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py index 0ceecb659..b50057b31 100755 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -139,7 +139,7 @@ def verify_predictions(path, selected_indices): def source_provenance(): files = ("run.py", "vlmeval/perception_test_local.py", "vlmeval/dataset/perception_test.py", - "vlmeval/dataset/video_dataset_config.py", "vlmeval/api/cosmos_reason.py", + "vlmeval/dataset/video_dataset_config.py", "vlmeval/api/cosmos_reason.py", "vlmeval/execution_evidence.py", "launchable/launchable.py") return {"framework_revision": os.environ.get("COSMOS_COMMIT", "unknown"), "source_sha256": {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() @@ -150,6 +150,17 @@ def _write_state(work, state): (work / "run.json").write_text(json.dumps(state, indent=2, default=str) + "\n") +def collect_execution_evidence(work, expected): + from vlmeval.execution_evidence import aggregate_execution_records + + records = [json.loads(path.read_text()) for path in sorted((work / "execution-responses").glob("*.json"))] + aggregate = aggregate_execution_records(records, expected_requests=expected) + aggregate.update(per_response="execution-responses", requested_configuration="config.json", + summary="execution-summary.json") + (work / "execution-summary.json").write_text(json.dumps(aggregate, indent=2) + "\n") + return aggregate + + def _protocol(nframe): if nframe != 16: raise ValueError("The launchable prototype protocol requires --nframe 16") @@ -171,6 +182,10 @@ def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): model = {"class": recipe["class"], **recipe["defaults"]} model.update(api_base=os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions"), model=os.environ.get("COSMOS_MODEL", "nvidia/Cosmos3-Nano")) + from vlmeval.execution_evidence import nano_processor_profile, PINNED_SERVING_COMPATIBILITY + model.update(execution_evidence_dir=str(work / "execution-responses"), + execution_processor_profile=nano_processor_profile( + model["model"], os.environ.get("COSMOS_MODEL_REVISION", "unknown"))) config = {"model": {"cosmos": model}, "data": {"PerceptionTest": { "class": "PerceptionTestLocal", "dataset": "PerceptionTest", "data_root": str(data_root), "nframe": nframe, "smoke": smoke}}} @@ -186,13 +201,16 @@ def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): "data_root": str(data_root), "nframe": nframe, "frame_protocol": PROTOCOL, "api_nproc": api_nproc, "config": config, "generation_parameters": {key: value for key, value in model.items() - if key not in {"class", "api_base", "model"}}, + if key not in {"class", "api_base", "model", "execution_evidence_dir", + "execution_processor_profile"}}, "model": model["model"], "model_revision": os.environ.get("COSMOS_MODEL_REVISION", MODEL_REVISION if local and model["model"] == "nvidia/Cosmos3-Nano" else "unknown"), "serving_image": (os.environ.get("VLLM_IMAGE", SERVING_IMAGE) if local else os.environ.get("COSMOS_SERVING_IMAGE", "unknown")), "provenance": source_provenance(), "predictions": None, "scores": None, + "execution_evidence": None, + "pinned_runtime_compatibility": PINNED_SERVING_COMPATIBILITY, "inference_timing": {"started_utc": started}} config_path.write_text(json.dumps(config, indent=2) + "\n") _write_state(work, state) @@ -209,6 +227,7 @@ def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): config_path.write_text(json.dumps(config, indent=2) + "\n") _write_state(work, state) run_command(command, log_path=work / "run.log") + state["execution_evidence"] = collect_execution_evidence(work, len(prepared["selected_indices"])) path = verify_predictions(work / "native", prepared["selected_indices"]) state.update(status="inferred", predictions=str(path.relative_to(work))) state["inference_timing"].update(ended_utc=utc_now(), elapsed_seconds=time.monotonic() - clock) @@ -220,6 +239,7 @@ def run(work, mode, smoke, dry_run, data_root, api_nproc=1, nframe=16): except BaseException as exc: state = json.loads((work / "run.json").read_text()) state.update(status="failed", error={"type": type(exc).__name__, "message": str(exc)}) + state["execution_evidence"] = collect_execution_evidence(work, len(prepared["selected_indices"])) if "ended_utc" not in state["inference_timing"]: state["inference_timing"].update(ended_utc=utc_now(), elapsed_seconds=time.monotonic() - clock) _write_state(work, state) @@ -271,6 +291,9 @@ def score(data_root, scope, work=None, predictions=None, score_dir=None, nframe= smoke=scope == "smoke", nframe=nframe) timing = {"started_utc": started, "ended_utc": utc_now(), "elapsed_seconds": time.monotonic() - clock} + summary["execution_evidence"] = state.get("execution_evidence") if state is not None else { + "verification_counts": {"unknown": 1}, "reason": "existing predictions have no request/response evidence"} + (score_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") # Keep controller/model provenance beside the native trusted-label score artifact. (score_dir / "controller.json").write_text(json.dumps({ "source": SOURCE, "scope": scope, "timing": timing, @@ -279,6 +302,7 @@ def score(data_root, scope, work=None, predictions=None, score_dir=None, nframe= "model_revision": state.get("model_revision", "unknown") if state is not None else "unknown", "serving_image": state.get("serving_image", "unknown") if state is not None else "unknown", "generation_parameters": state.get("generation_parameters") if state is not None else None, + "execution_evidence": summary["execution_evidence"], "inference_run": str(work / "run.json") if work is not None else None, }, indent=2) + "\n") if state is not None: diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/mmsivideo.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/mmsivideo.py new file mode 100755 index 000000000..1e1cabf7c --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/mmsivideo.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned public MMSI-Video-Bench Uniform-50 download and evaluation. + +This is not the distinct multi-image MMSI-Bench. Assets stay outside source; +only the maintained prompt/sampling and CosmosReason2 implementation are reused. +""" +from __future__ import annotations + +import argparse +import ast +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import importlib.util +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import subprocess +import sys +import threading +import time +from urllib.parse import urlsplit +import urllib.request +import zipfile + +DATASET_REVISION = '8eab53d8eb8b5c0863cfbfe1428c936f3e9e243e' +EASI_REVISION = '52bb1dd2bc5cd58a291b3a780dab08d19c0ccf0d' +OFFICIAL_REVISION = '412f5a4a9eb6d89613c60b541b17df74f11a6736' +DATASET_URL = f'https://huggingface.co/datasets/rbler/MMSI-Video-Bench/resolve/{DATASET_REVISION}' +OFFICIAL_URL = f'https://raw.githubusercontent.com/InternRobotics/MMSI-Video-Bench/{OFFICIAL_REVISION}' +FILES = { + 'frames.zip': (f'{DATASET_URL}/frames.zip', '278bafcf4f8dc4abbaac7e0c10370aa4fe223c20e5fda47037478f04a44b1f08'), + 'ref_images.zip': (f'{DATASET_URL}/ref_images.zip', '50347c966703a749bacc87769c1f011deb012e82a9fd0bf0503e6551490ea7f7'), + 'mmsivideo.json': (f'{DATASET_URL}/mmsivideo.json', '58dd55fbbcf52eeafa368c07ac9ace0968a06d4a07da79856afff706db3b2bc9'), + 'MMSIVideoBench.tsv': ( + f'https://huggingface.co/datasets/lmms-lab-si/EASI-Leaderboard-Data/resolve/{EASI_REVISION}/MMSIVideoBench.tsv', + '0fb24d10e25d2387496a641b3807ad570303d442759f3e31fd7f93de565bf3bd'), + 'official_README.md': (f'{OFFICIAL_URL}/README.md', 'c4fb14c27445faeb16c90bc56fa4040ebe628229b50a76093bf75f13ed7a2c1c'), + 'official_dataset.py': (f'{OFFICIAL_URL}/dataset.py', 'cb719ea7eaf03fe98b9b6db31ec2c5fa07f25d76982cdd6ad64a950b48355801'), + 'official_evaluation.py': (f'{OFFICIAL_URL}/evaluation.py', '6424797dc6fe1517839e1fd8c4027d605fd344dc8822418cd892e1062603011d'), +} +WARNINGS = [ + 'Publisher license: CC BY 4.0. Attribution: MMSI-Video-Bench, Jingli Lin et al. (2025), arXiv:2512.10863; InternRobotics/MMSI-Video-Bench.', + 'Underlying third-party media redistribution rights are not independently verified (including ScanNet, Ego4D and Waymo). Keep media local; no raw-media redistribution. Further redistribution/commercial use requires underlying rights review.', + 'Independent public Uniform-50 measurement, not reference parity. Paper reference 27.2%; exact benchmark-specific paper recipe is undisclosed.', + 'Pinned EASI TSV differs from the maintained adapter stale MD5; SHA256 is pinned and selected IDs/labels/options/prompts/media are checked against official JSON.', + 'Maintained no-judge extraction is diagnostic only: it can undercount valid verbose outputs compared with the official extractor.', +] +PROTOCOL = { + 'setting': 'MMSIVideoBench_50frame (Uniform-50)', + 'sampling': 'Maintained global numpy.linspace(0,total-1,min(50,total),dtype=int) across canonical annotation frames, segment order preserved; all references', + 'prompt': 'Maintained MMSIVideoBench.build_prompt; official system + newline + task + user + format text interleaved with canonical frames/references; no hints', + 'temperature': 0, 'top_p': 1.0, 'top_k': -1, 'seed': 1, + 'max_tokens': 2048, 'repetition_penalty': 1.0, 'presence_penalty': 0.0, + 'img_size': -1, 'img_detail': 'low', 'max_pixels': 151200, + 'thinking_mode': 'No override; CosmosReason2 (not Think), server chat-template default; no suffix', + 'chat_template_kwargs': 'omitted', 'response_format': 'omitted', + 'primary_scorer': 'Pinned official evaluation.py extract_answer, exact match; invalid/blank/truncated remain in denominator', + 'diagnostic_scorer': 'Maintained MMSIVideoBench.evaluate(model=exact_matching), no external judge', + 'raw_videos_downloaded': False, 'reference_score_percent': 27.2, +} + + +def _scope(scope): + if scope not in ('smoke', 'full'): + raise ValueError('scope must be smoke or full') + + +def _sha(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def _save(path, value): + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n') + temporary.replace(path) + + +def _external_root(path): + root = Path(path).expanduser().resolve() + source = next((parent for parent in Path(__file__).resolve().parents + if (parent / '.git').exists()), Path(__file__).resolve().parents[1]) + if root == source or root.is_relative_to(source): + raise ValueError('Dataset assets must be outside the source checkout') + return root + + +def _member_path(root, name): + relative = PurePosixPath(name) + if relative.is_absolute() or '..' in relative.parts or '\\' in name: + raise ValueError(f'Unsafe media path: {name!r}') + path = root.joinpath(*relative.parts) + if not path.resolve().is_relative_to(root.resolve()): + raise ValueError(f'Media path escapes owned root: {name!r}') + return path + + +def _fetch(root, name, url, expected): + path = root / name + if path.is_file() and not path.is_symlink() and _sha(path) == expected: + print(f'MMSI verified cache: {name}', flush=True) + else: + partial = root / (name + '.part') + if partial.is_symlink() or path.is_symlink(): + raise ValueError(f'Refusing symlink download target: {name}') + received = 0 + last = time.monotonic() + print(f'MMSI downloading: {name}', flush=True) + with urllib.request.urlopen(url, timeout=300) as response, partial.open('wb') as output: + total = response.headers.get('Content-Length', 'unknown') + while block := response.read(8 << 20): + output.write(block) + received += len(block) + if time.monotonic() - last >= 5: + print(f'MMSI {name}: {received} / {total} bytes', flush=True) + last = time.monotonic() + if _sha(partial) != expected: + raise ValueError(f'SHA256 mismatch: {name}; partial retained for inspection') + partial.replace(path) + return {'file': name, 'url': url, 'sha256': expected, 'bytes': path.stat().st_size} + + +def _extract(root, name): + # CRC-check cached members too: a marker alone cannot establish media integrity. + count = 0 + with zipfile.ZipFile(root / name) as archive: + for info in archive.infolist(): + destination = _member_path(root, info.filename) + mode = info.external_attr >> 16 + if stat.S_ISLNK(mode) or (stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR)): + raise ValueError(f'Unsafe archive member: {info.filename!r}') + if info.is_dir(): + destination.mkdir(parents=True, exist_ok=True) + continue + if destination.is_symlink(): + raise ValueError(f'Refusing symlink extraction target: {info.filename!r}') + destination.parent.mkdir(parents=True, exist_ok=True) + # Reading through zipfile validates each pinned archive member's CRC. + with archive.open(info) as source, destination.open('wb') as output: + shutil.copyfileobj(source, output, 1 << 20) + count += 1 + if count % 2000 == 0: + print(f'MMSI extracting {name}: {count} files', flush=True) + return {'archive': name, 'files': count} + + +def download(data_root: Path, scope: str): + """Download and safely unpack public pinned canonical frames, never videos.zip.""" + _scope(scope) + started = time.monotonic() + root = _external_root(data_root) + root.mkdir(parents=True, exist_ok=True) + downloads = [_fetch(root, name, *spec) for name, spec in FILES.items()] + extracted = [_extract(root, name) for name in ('frames.zip', 'ref_images.zip')] + result = {'benchmark': 'mmsivideo-bench', 'scope': scope, 'status': 'downloaded', + 'data_root': str(root), 'downloads': downloads, 'extracted': extracted, + 'dataset_revision': DATASET_REVISION, 'easi_revision': EASI_REVISION, + 'official_revision': OFFICIAL_REVISION, 'warnings': WARNINGS, + 'raw_videos_downloaded': False, 'elapsed_seconds': time.monotonic() - started} + _save(root / 'download.json', result) + return result + + +def _verify_sources(root): + for name, (_, expected) in FILES.items(): + # The archive pins establish provenance even when only extracted members are consumed. + if not (root / name).is_file() or (root / name).is_symlink() or _sha(root / name) != expected: + raise ValueError(f'Missing or changed pinned asset {name}; run download first') + + +def _dataset(root): + import pandas as pd + from vlmeval.dataset.mmsibench import MMSIVideoBench + + class StagedMMSIVideo(MMSIVideoBench): + def __init__(self): + # Do not call VideoBaseDataset.__init__: it creates shared LMU caches + # and invokes network staging. Only owned paths/data are needed here. + self.dataset_name = 'MMSIVideoBench' + self.dataset_path = str(root) + self.data_root = str(root) + self.pack = False + self.nframe = 50 + self.fps = -1 + self.data = pd.read_csv(root / 'MMSIVideoBench.tsv', sep='\t', keep_default_na=False) + def absolute(value): + if isinstance(value, list): + return [absolute(item) for item in value] + return str(_member_path(root, value.lstrip('/\\'))) + for column in ('image_path', 'ref_images'): + self.data[column] = self.data[column].map(lambda value: absolute(ast.literal_eval(value))) + + return StagedMMSIVideo() + + +def _qualify(dataset, root, scope, work): + annotations = json.loads((root / 'mmsivideo.json').read_text()) + by_id = {row['id']: row for row in annotations} + data = dataset.data + if len(by_id) != 1106 or len(annotations) != 1106 or len(data) != 1106: + raise ValueError('Expected exactly 1106 unique official questions and metadata rows') + if data.raw_id.duplicated().any() or data['index'].duplicated().any() or set(data.raw_id) != set(by_id): + raise ValueError('Metadata identities differ from official annotations') + if len(set(data.task_type)) != 5: + raise ValueError('Expected five task categories') + selected = data if scope == 'full' else data.groupby('task_type', sort=True).head(1) + manifest = [] + prompts = {} + for _, row in selected.iterrows(): + original = by_id[row.raw_id] + if Path(row.raw_id).name != row.raw_id or row.raw_id in ('.', '..'): + raise ValueError('Unsafe official question ID') + expected_type = (row.task_type if row.task_type in ('Planning', 'Prediction') + else f'({row.task_type}) {row.sub_task_type}') + if original['type'] != expected_type: + raise ValueError(f'Official task category mismatch: {row.raw_id}') + options = ast.literal_eval(row.options) + if row.answer != original['ground_truth'] or options != original['options']: + raise ValueError(f'Label/options mismatch: {row.raw_id}') + if row.answer not in list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')[:len(options)]: + raise ValueError(f'Invalid ground truth: {row.raw_id}') + for field in ('system_prompt', 'task_prompt', 'user_prompt', 'format_prompt'): + if row[field] != original[field]: + raise ValueError(f'Official prompt mismatch: {row.raw_id} {field}') + frames = dataset.parse_image_path(row.image_path) + expected_frames = [[str(_member_path(root, 'frames/' + name)) for name in segment] + for segment in original['frames_list']] + refs = row.ref_images + expected_refs = [str(_member_path(root, 'ref_images/' + name)) for name in original['ref_images']] + if frames != expected_frames or refs != expected_refs: + raise ValueError(f'Canonical media mismatch: {row.raw_id}') + for value in [item for segment in frames for item in segment] + refs: + path = Path(value) + if not path.resolve().is_relative_to(root) or not path.is_file() or path.stat().st_size == 0: + raise ValueError(f'Missing/unsafe media for {row.raw_id}: {path.name}') + prompt = dataset.build_prompt(row) + sampled = dataset.save_video_frames(frames, row.video_fps) + prompts[row.raw_id] = prompt + manifest.append({'id': row.raw_id, 'index': int(row['index']), 'task_type': row.task_type, + 'sub_task_type': row.sub_task_type, 'answer': row.answer, + 'selected_paths': [[str(Path(p).relative_to(root)) for p in segment] for segment in sampled], + 'references': [str(Path(p).relative_to(root)) for p in refs], + 'annotation_frames': sum(map(len, frames)), 'selected_frames': sum(map(len, sampled)), + 'prompt': prompt, + 'prompt_text_sha256': hashlib.sha256(''.join(p['value'] for p in prompt if p['type'] == 'text').encode()).hexdigest()}) + _save(work / 'sample_manifest.json', manifest) + print(f'MMSI qualified {len(selected)} questions; all selected labels/options/prompts/media match official annotations', flush=True) + return selected, by_id, prompts + + +def _runtime_provenance(): + root = Path(__file__).resolve().parents[1] + paths = ['launchable/mmsivideo.py', 'vlmeval/dataset/mmsibench.py', + 'vlmeval/api/cosmos_reason.py', 'vlmeval/dataset/utils/multiple_choice.py', + 'vlmeval/dataset/utils/spatial_bench/cal_scores.py', 'vlmeval/execution_evidence.py'] + return {name: _sha(root / name) for name in paths} + + +def _score(dataset, selected, records, by_id, root, work, scope, api_nproc): + from vlmeval.dataset.utils.multiple_choice import build_choices, extract_answer_from_item + spec = importlib.util.spec_from_file_location('mmsivideo_official_evaluation', root / 'official_evaluation.py') + official = importlib.util.module_from_spec(spec) + spec.loader.exec_module(official) + inputs = work / 'official_inputs' + inputs.mkdir() + scored = [] + for record in records: + row = selected.loc[selected.raw_id == record['id']].iloc[0].to_dict() + row.update(dict(zip('ABCDEFGHIJKLMNOPQRSTUVWXYZ', ast.literal_eval(row['options'])))) + row['prediction'] = record['prediction'] + diagnostic = extract_answer_from_item(None, row, dataset_name='MMSIVideoBench')['opt'] + primary = official.extract_answer(record['prediction']) + annotation = by_id[record['id']] + _save(inputs / f"{record['id']}.json", {'id': record['id'], 'response': record['prediction'], + 'ground_truth': annotation['ground_truth'], 'type': annotation['type']}) + scored.append({'id': record['id'], 'answer': row['answer'], 'task_type': row['task_type'], + 'sub_task_type': row['sub_task_type'], 'official_prediction': primary, + 'diagnostic_prediction': diagnostic, 'official_hit': int(primary == row['answer']), + 'diagnostic_hit': int(diagnostic == row['answer']), + 'official_invalid': primary not in build_choices(row), + 'diagnostic_invalid': diagnostic not in build_choices(row), + 'truncated': record['truncated'], 'finish_reason': record['finish_reason']}) + _save(work / 'scored.json', scored) + frame = selected.copy() + predictions = {r['id']: r['prediction'] for r in records} + frame['prediction'] = frame.raw_id.map(predictions) + frame.to_csv(work / 'predictions.tsv', sep='\t', index=False) + dataset.data = selected.copy() + native = dataset.evaluate(str(work / 'predictions.tsv'), model='exact_matching', nproc=api_nproc) + def reduce(items): + count = len(items) + return {'count': count, 'official_correct': sum(r['official_hit'] for r in items), + 'official_accuracy_percent': 100 * sum(r['official_hit'] for r in items) / count, + 'diagnostic_accuracy_percent': 100 * sum(r['diagnostic_hit'] for r in items) / count, + 'official_invalid': sum(r['official_invalid'] for r in items), + 'diagnostic_invalid': sum(r['diagnostic_invalid'] for r in items), + 'truncated': sum(r['truncated'] for r in items)} + result = reduce(scored) + if abs(float(native['overall']) - result['diagnostic_accuracy_percent']) > 1e-9: + raise RuntimeError('Maintained diagnostic scorer disagrees with independent reduction') + _save(work / 'diagnostic_summary.json', native) + if scope == 'full': + completed = subprocess.run([sys.executable, str(root / 'official_evaluation.py'), + '--eval_dir', str(inputs), '--bench', 'main'], + capture_output=True, text=True, cwd=root) + (work / 'official_scorer.log').write_text(completed.stdout + completed.stderr) + completed.check_returncode() + line = next(line for line in completed.stdout.splitlines() if line.startswith('Overall :')) + accuracy = float(line.split(':', 1)[1].split()[0]) * 100 + if abs(accuracy - result['official_accuracy_percent']) > 1e-9: + raise RuntimeError('Official CLI disagrees with independent reduction') + result['official_cli_agrees'] = True + else: + (work / 'official_scorer.log').write_text('Smoke: pinned official extract_answer used locally for every row. CLI requires all 13 subcategories; full scope runs unmodified CLI.\n') + result['per_category'] = {name: reduce([r for r in scored if r['task_type'] == name]) + for name in sorted({r['task_type'] for r in scored})} + result['per_subcategory'] = {name: reduce([r for r in scored if r['sub_task_type'] == name]) + for name in sorted({r['sub_task_type'] for r in scored})} + result['scorer_discrepancies'] = sum(r['official_prediction'] != r['diagnostic_prediction'] for r in scored) + result['diagnostic_scorer_agrees'] = True + return result + + +def _successful_transport(code, http_status, wrapper_error): + """A truncation marker cannot turn an HTTP success into a transport failure.""" + return (200 <= http_status < 300 + and (code == 0 or wrapper_error == 'truncated')) + + +def run(data_root: Path, scope: str, work: Path, api_nproc: int = 4): + """Evaluate a fresh work directory, retaining every response and generation issue.""" + _scope(scope) + if not isinstance(api_nproc, int) or isinstance(api_nproc, bool) or not 1 <= api_nproc <= 4: + raise ValueError('api_nproc must be an integer from 1 through 4') + root = _external_root(data_root) + work = _external_root(work) + # A clean run has no prediction reuse or selective generation retries. + work.mkdir(parents=True, exist_ok=False, mode=0o700) + started = time.monotonic() + records = [] + from vlmeval.execution_evidence import aggregate_execution_records, nano_processor_profile, PINNED_SERVING_COMPATIBILITY + expected = 5 if scope == 'smoke' else 1106 + def progress(message): + print(message, flush=True) + with (work / 'run.log').open('a') as log: + log.write(message + '\n') + try: + _verify_sources(root) + dataset = _dataset(root) + selected, by_id, prompts = _qualify(dataset, root, scope, work) + base = os.environ.get('COSMOS_API_BASE', 'http://cosmos-endpoint:8080/v1/chat/completions') + parsed = urlsplit(base) + if parsed.scheme not in ('http', 'https') or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError('Use a plain HTTP(S) endpoint without URL credentials, query or fragment; supply COSMOS_API_KEY separately') + if not parsed.path.endswith('/chat/completions'): + raise ValueError('COSMOS_API_BASE must be the complete /v1/chat/completions URL') + model_name = os.environ.get('COSMOS_MODEL', 'nvidia/Cosmos3-Nano') + provenance = {'benchmark': 'mmsivideo-bench', 'scope': scope, + 'dataset_revision': DATASET_REVISION, 'easi_revision': EASI_REVISION, + 'official_revision': OFFICIAL_REVISION, + 'assets': {name: {'url': url, 'sha256': digest} for name, (url, digest) in FILES.items()}, + 'model': model_name, 'model_revision': os.environ.get('COSMOS_MODEL_REVISION', 'unknown'), + 'endpoint': base, 'protocol': dict(PROTOCOL, api_nproc=api_nproc), + 'runtime_source_sha256': _runtime_provenance(), 'warnings': WARNINGS, + 'execution_evidence': {'summary': 'execution-summary.json', 'per_response': 'execution-responses'}, + 'pinned_runtime_compatibility': PINNED_SERVING_COMPATIBILITY, + 'reference_parity': False, 'started_unix': time.time()} + _save(work / 'provenance.json', provenance) + # Import/instantiate only after all selected samples have been qualified. + from vlmeval.api.cosmos_reason import CosmosReason2 + local = threading.local() + def infer(row): + sample_started = time.monotonic() + record = {'id': row.raw_id, 'index': int(row['index'])} + try: + if not hasattr(local, 'model'): + local.model = CosmosReason2(model=model_name, api_base=base, + key=os.environ.get('COSMOS_API_KEY', 'EMPTY'), temperature=0, top_p=1.0, + top_k=-1, repetition_penalty=1.0, presence_penalty=0.0, + max_tokens=2048, seed=1, timeout=600, retry=1, wait=0, + img_size=-1, img_detail='low', + execution_evidence_dir=str(work / 'execution-responses'), + execution_processor_profile=nano_processor_profile( + model_name, os.environ.get('COSMOS_MODEL_REVISION', 'unknown')), + extra_body={'mm_processor_kwargs': {'max_pixels': 151200}}) + if local.model.model != model_name: + raise RuntimeError('Endpoint model differs from COSMOS_MODEL; refusing implicit remapping') + code, prediction, response = local.model.generate_inner(prompts[row.raw_id]) + raw = response.json() if response is not None else None + record.update(status_code=code, prediction=prediction, response=raw, + wrapper_error=local.model._last_error_type, served_model=local.model.model) + record['execution_evidence'] = local.model._last_execution_record + # Preserve the actual wire response as well as wrapper-extracted text. + if response is not None: + record['raw_response_text'] = response.text + transport_ok = (response is not None and _successful_transport( + code, response.status_code, local.model._last_error_type)) + if not transport_ok or not isinstance(raw, dict) or not raw.get('choices'): + raise RuntimeError('Transport failure or malformed completion response') + choice = raw['choices'][0] + if not isinstance(prediction, str) or not isinstance(choice.get('message', {}).get('content'), str): + raise RuntimeError('Completion missing string content') + finish = choice.get('finish_reason') + if not isinstance(finish, str): + raise RuntimeError('Completion missing finish reason') + record.update(finish_reason=finish, truncated=finish != 'stop') + if local.model._last_error_type not in (None, 'truncated'): + raise RuntimeError('Endpoint generation failed') + except Exception as error: + # Exception strings can contain authentication URLs/headers. Never persist them. + record['failure'] = type(error).__name__ + record['elapsed_seconds'] = time.monotonic() - sample_started + _save(work / f"{row.raw_id}.json", record) + return record + with ThreadPoolExecutor(max_workers=api_nproc) as pool: + futures = [pool.submit(infer, row) for _, row in selected.iterrows()] + for future in as_completed(futures): + record = future.result() + records.append(record) + progress(f'MMSI {len(records)}/{expected}: {record["id"]} ' + + ('FAILED' if record.get('failure') else str(record['finish_reason']))) + records.sort(key=lambda record: record['index']) + _save(work / 'responses.json', records) + evidence = aggregate_execution_records( + [record.get('execution_evidence') or {} for record in records], expected_requests=expected) + evidence.update(summary='execution-summary.json', per_response='execution-responses') + _save(work / 'execution-summary.json', evidence) + provenance['execution_evidence'] = evidence + _save(work / 'provenance.json', provenance) + if len(records) != expected or any(r.get('failure') for r in records): + raise RuntimeError('Incomplete coverage/transport failure; inspect durable per-sample records; no score published') + metrics = _score(dataset, selected, records, by_id, root, work, scope, api_nproc) + truncated = sum(r['truncated'] for r in records) + warnings = list(WARNINGS) + if truncated: + warnings.append(f'{truncated} non-stop/truncated completions retained and scored without regeneration.') + if metrics['official_invalid']: + warnings.append(f'{metrics["official_invalid"]} invalid official extractions retained as incorrect.') + summary = {'benchmark': 'mmsivideo-bench', 'scope': scope, + 'status': ('completed_with_generation_issues' + if truncated or metrics['official_invalid'] else 'completed'), + 'primary_metric': {'name': 'official_accuracy', 'value': metrics['official_accuracy_percent'], 'unit': 'percent'}, + 'coverage': {'expected': expected, 'scored': len(records)}, + 'protocol': provenance['protocol'], 'warnings': warnings, 'reference_parity': False, + 'elapsed_seconds': time.monotonic() - started, 'metrics': metrics, + 'execution_evidence': evidence, + 'finish_reasons': dict(Counter(r['finish_reason'] for r in records)), + 'model': model_name, 'model_revision': provenance['model_revision']} + _save(work / 'summary.json', summary) + provenance['elapsed_seconds'] = summary['elapsed_seconds'] + provenance['finished_unix'] = time.time() + _save(work / 'provenance.json', provenance) + progress(f'MMSI {summary["status"]}: official accuracy {metrics["official_accuracy_percent"]:.4f}%') + return summary + except Exception as error: + _save(work / 'failure.json', {'benchmark': 'mmsivideo-bench', 'scope': scope, 'status': 'failed', + 'error_type': type(error).__name__, 'expected': expected, 'responses_retained': len(records), + 'elapsed_seconds': time.monotonic() - started, 'score_published': False}) + progress('MMSI failed; inspect failure.json and retained evidence. No completed score published.') + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('download', 'run')) + parser.add_argument('--data-root', type=Path, required=True) + parser.add_argument('--scope', choices=('smoke', 'full'), default='smoke') + parser.add_argument('--work', type=Path) + parser.add_argument('--api-nproc', type=int, default=4) + args = parser.parse_args() + if args.action == 'download': + result = download(args.data_root, args.scope) + else: + if args.work is None: + parser.error('--work is required for run') + result = run(args.data_root, args.scope, args.work, args.api_nproc) + print(json.dumps(result, indent=2), flush=True) + + +if __name__ == '__main__': + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/notebook_ui.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/notebook_ui.py new file mode 100644 index 000000000..6d8c6ae66 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/notebook_ui.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Scoped, theme-aware presentation for the public benchmark notebook.""" +from __future__ import annotations + +from html import escape +import json +import math +from pathlib import Path +import subprocess + +from IPython.display import FileLink, HTML, display + +from benchmarks import CATALOG, validate_selection + +# Kaizen UI 1.19 semantic tokens, adapted to native notebook HTML. Jupyter owns +# the shell and theme; do not load KUI's global reset, fonts, or React runtime. +STYLE = """""" + + +def _html(body): + # Each output is self-contained when displayed or rerun independently. + display(HTML(STYLE + '
' + body + '
')) + + +def show_plan(benchmark, scope, data, work, workers, score_only=False): + validate_selection(benchmark, scope, score_only) + item = CATALOG[benchmark] + _html('

Run configuration

' + f'{escape(scope.title())}
' + f'

{escape(item["scope"] if scope == "full" else item["smoke"])}

' + '
Execution
' + f'
{"Score existing predictions" if score_only else "Inference and local scoring"}
' + f'
API workers
{workers}
' + '' + '
Protocol and files' + f'

{escape(item["protocol"])}

' + f'
Dataset directory
{escape(str(data))}
' + f'
New results directory
{escape(str(work))}
' + '
Source and attribution' + f'

{escape(item["license"])}

Publisher source' + '

Public hosting does not clear all underlying redistribution rights. ' + 'Downloads stay on your instance; no raw data is bundled or uploaded by this notebook.

') + if scope == "full": + show_baseline(benchmark) + + +def show_baseline(benchmark): + """Publish the pinned full-run target without presenting it as paper parity.""" + record = json.loads(Path(__file__).with_name("baselines.json").read_text()) + baseline = record["benchmarks"][benchmark] + suffix = "%" if baseline["unit"] == "percent" else " correlation points" + paper = baseline["paper_value"] + comparison = ("No reference in the cited Cosmos3 table." if paper is None + else f'Paper reference: {paper:.2f}{suffix}.') + _html('') + + +def invoke(command, log_path=None): + """Stream real subprocess progress; preserve failures and a private log.""" + log = None + if log_path is not None: + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + log = log_path.open("x") + try: + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, bufsize=1) as process: + try: + for line in process.stdout: + print(line, end="", flush=True) + if log is not None: + log.write(line) + log.flush() + if process.wait(): + raise subprocess.CalledProcessError(process.returncode, command) + except KeyboardInterrupt: + process.terminate() + process.wait() + raise + finally: + if log is not None: + log.close() + + +def show_results(summary, benchmark, scope, work, log_path=None): + """Bind the displayed public benchmark to its artifact.""" + validate_selection(benchmark, scope) + _show_results(summary, benchmark, scope, work, CATALOG[benchmark], log_path) + + +def show_customer_results(summary, work, log_path=None): + """Keep customer results separate from official benchmark measurements.""" + item = {"title": "Customer-defined video evaluation", + "metric": "Multiple-choice accuracy"} + _show_results(summary, "customer-video-mcq", "custom", work, item, log_path) + + +def _execution_facts(summary): + evidence = summary.get("execution_evidence") + if not isinstance(evidence, dict): + return "" + requests = [entry["settings"] for entry in evidence.get("requested_settings", []) + if isinstance(entry.get("settings"), dict)] + sampling = summary.get("sampling", {}) + + def formatted(values, missing="Not observed"): + values = sorted({value for value in values if isinstance(value, (int, float)) + and not isinstance(value, bool) and math.isfinite(value)}) + if not values: + return missing + return f"{values[0]:,}" if len(values) == 1 else f"{values[0]:,}–{values[-1]:,}" + + requested_frames = [sampling.get("nframes")] + [ + item.get("media", {}).get("expected_frame_slots", item.get("media", {}).get("sampled_frames")) + or item.get("image_inputs") or None + for item in requests] + minima = {sampling.get("min_pixels")} | {item.get("mm_processor_kwargs", {}).get("min_pixels") for item in requests} + maxima = {sampling.get("max_pixels")} | {item.get("mm_processor_kwargs", {}).get("max_pixels") for item in requests} + minima.discard(None) + maxima.discard(None) + budget = (formatted(minima | maxima, "Not recorded") if minima and maxima else + "≤" + formatted(maxima) if maxima else "≥" + formatted(minima) if minima else "Not recorded") + shapes = evidence.get("observed_visual_shapes", []) + areas = [area for shape in shapes for key in ("image_grid_pixel_areas", "video_temporal_grid_pixel_areas") + for area in shape.get(key, [])] + facts = { + "Requested input frames/images": formatted(requested_frames, "Not recorded"), + "Observed image groups / video slots": formatted( + shape.get("video_frame_slots") or len(shape.get("image_group_tokens", [])) or None for shape in shapes), + "Requested spatial budget (pixels/frame)": budget, + "Observed spatial grid area (pixels)": formatted(areas), + "Returned model": ", ".join(evidence.get("observed_models", [])) or "Not observed", + "Response evidence": f'{evidence.get("requests_recorded", 0)} / {evidence.get("expected_requests", "?")} requests', + } + return ('

Input execution evidence

' + + "".join(f"
{escape(key)}
{escape(str(value))}
" for key, value in facts.items()) + + '

Grid areas and frame slots are conditional on the explicit processor profile. ' + 'They do not attest decoded/unique frames, width/height, or generation settings. ' + 'See the detailed record for unknowns and geometry differences.

') + + +def _show_results(summary, benchmark, scope, work, item, log_path=None): + if summary.get("benchmark") != benchmark or summary.get("scope") != scope: + raise ValueError("Result identity does not match this notebook's selected benchmark/scope") + if summary.get("status") not in {"completed", "completed_with_generation_issues"}: + raise ValueError("This run is not complete; inspect retained diagnostics before reporting a score") + metric = summary["primary_metric"] + value, unit = metric["value"], metric["unit"] + if value is not None and not math.isfinite(value): + raise ValueError("Metric must be finite or explicitly undefined") + headline = "Not defined" if value is None else f"{value:.2f}{'%' if unit == 'percent' else ''}" + coverage = summary["coverage"] + if coverage["scored"] != coverage["expected"]: + raise ValueError("Refusing to present incomplete coverage as a completed benchmark") + warnings = list(summary.get("warnings", [])) + issues = summary["status"] == "completed_with_generation_issues" + notes = "".join(f'
  • {escape(str(note))}
    • ' for note in warnings) + detail = "Correlation ×100 — not accuracy %" if unit == "pearson_x100" else "Accuracy across all selected questions" + if value is None: + detail = "Insufficient rating variance for correlation; inspect the retained predictions." + elapsed = summary.get("elapsed_seconds") + elapsed_text = "Not recorded" if elapsed is None else f"{elapsed / 60:.1f} min" + status = "Completed with generation issues" if issues else "Run completed" + banner = ("Generation issues were retained in the score; inspect the notes below. " if issues else "") + if scope == "smoke": + banner += "Smoke run only: this checks the pipeline, not full-benchmark performance." + if scope == "custom": + banner += "Customer-defined labeled data: not an official benchmark or leaderboard result." + execution = { + "requested": summary.get("requested_execution"), + "effective": summary.get("effective_execution"), + "evidence": summary.get("execution_evidence"), + } + notice = (f'' if banner else "") + _html(f'
    ' + '

    Benchmark result

    ' + f'

    {escape(item["title"])}

    ' + f'{status}
    ' + f'

    {escape(scope.title())} scope · {escape(benchmark)}

    ' + f'{notice}
    {escape(item["metric"])}' + f'
    {escape(headline)}
    ' + f'

    {escape(detail)}

    ' + '
    Selected coverage
    ' + f'
    {coverage["scored"]:,} of {coverage["expected"]:,} examples
    ' + f'
    Measured run time
    {escape(elapsed_text)}
    ' + f'{_execution_facts(summary)}' + '

    Published-score equivalence has not been established.

    ' + f'Data and scoring notes
      {notes}
    ' + '
    Exact protocol and metric record' + f'
    {escape(json.dumps({"primary_metric": metric, "protocol": summary["protocol"]}, indent=2))}
    ' + '
    Requested and effective execution settings' + f'
    {escape(json.dumps(execution, indent=2))}
    ' + '

    Null means not recorded in this artifact, not verified. ' + 'Effective input geometry is derived from observed processor tokens when supported; ' + 'requested generation parameters are not a server attestation.

    ') + # These links are private runtime artifacts, never source files to publish. + artifacts = [Path(work) / "notebook-summary.json", Path(work) / "summary.json", + Path(work) / "provenance.json", Path(work) / "run.json", + Path(work) / "scores" / "summary.json", Path(work) / "run.log", + Path(work) / "execution-summary.json", + Path(work) / "requested-execution.json", Path(work) / "effective-execution.json"] + if log_path is not None: + artifacts.append(Path(log_path)) + for artifact in artifacts: + if artifact.is_file(): + display(FileLink(str(artifact), url_prefix="/files")) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh index 3c9adc2b6..30a44bcd2 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -12,21 +12,25 @@ export COSMOS_COMMIT COSMOS_COMMIT=$(git rev-parse HEAD) : "${LAUNCHABLE_STATE_DIR:?Set an absolute state directory outside the Framework checkout}" export PERCEPTION_TEST_DATA_DIR="${PERCEPTION_TEST_DATA_DIR:-$LAUNCHABLE_STATE_DIR/perception-test}" +export BENCHMARK_DATA_DIR="${BENCHMARK_DATA_DIR:-$LAUNCHABLE_STATE_DIR/datasets}" export LAUNCHABLE_STATE_DIR -python3 - "$LAUNCHABLE_STATE_DIR" "$PERCEPTION_TEST_DATA_DIR" "$(git rev-parse --show-toplevel)" <<'PY' +python3 - "$LAUNCHABLE_STATE_DIR" "$PERCEPTION_TEST_DATA_DIR" "$BENCHMARK_DATA_DIR" "$(git rev-parse --show-toplevel)" <<'PY' import os from pathlib import Path import sys -checkout = Path(sys.argv[3]).resolve() -state, data = (Path(value) for value in sys.argv[1:3]) -for path in (state, data): +checkout = Path(sys.argv[4]).resolve() +state, perception, benchmarks = (Path(value) for value in sys.argv[1:4]) +for path in (state, perception, benchmarks): if not path.is_absolute() or path.resolve().is_relative_to(checkout): - raise SystemExit("State and Perception Test data must use absolute paths outside the Framework checkout") -if any(data.resolve().is_relative_to((state / name).resolve()) - or (state / name).resolve().is_relative_to(data.resolve()) for name in ("cache", "results")): - raise SystemExit("Perception Test data must be separate from writable cache/results mounts") -for path in (state / "cache", state / "results", data): + raise SystemExit("State and data must use absolute paths outside the Framework checkout") +for data in (perception, benchmarks): + if any(data.resolve().is_relative_to((state / name).resolve()) + or (state / name).resolve().is_relative_to(data.resolve()) for name in ("cache", "results")): + raise SystemExit("Benchmark data must be separate from writable cache/results mounts") +if perception.resolve().is_relative_to(benchmarks.resolve()) or benchmarks.resolve().is_relative_to(perception.resolve()): + raise SystemExit("Perception Test and other benchmark data must use separate directories") +for path in (state / "cache", state / "results", perception, benchmarks): path.mkdir(parents=True, exist_ok=True) if not os.access(path, os.W_OK): raise SystemExit(f"Runtime directory must be writable: {path}") @@ -91,5 +95,5 @@ fi printf 'Jupyter is starting on %s:8888. Open its authenticated secure link or SSH tunnel.\n' "${JUPYTER_BIND_ADDRESS:-127.0.0.1}" printf '%s\n' \ 'Retrieve the notebook login token with: docker compose exec perception-eval jupyter server list' \ - 'Open Evaluate_Cosmos3.ipynb to download public Perception Test validation data, prepare, infer, and score.' \ - 'Default smoke: five video_8241 questions; full 19,140-question validation is explicit opt-in.' + 'Open Evaluate_Cosmos3.ipynb and follow a link to the benchmark notebook you want to run.' \ + 'Run All downloads public inputs, validates coverage, infers, and scores locally. SCOPE=full is explicit opt-in.' diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_custom_eval.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_custom_eval.py new file mode 100644 index 000000000..517928a57 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_custom_eval.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Deterministic annotation, inference-boundary, scoring and identity contracts.""" + +import copy +import json +from pathlib import Path +import shutil +import subprocess + +import pytest + +import custom_eval as custom + + +@pytest.fixture +def row(): + return {"id": "invented-001", "video": "clip.mp4", "question": "What happens next?", + "candidates": ["A box moves left", "A box moves right", "A box stays still", "A box vanishes"], + "answer": "A box moves right", "category": "motion", "start": 0, "end": 1} + + +@pytest.fixture +def root(tmp_path): + media = tmp_path / "media" + media.mkdir() + # Annotation/path unit tests need only a nonempty file, never decode it. + (media / "clip.mp4").write_bytes(b"invented path-only fixture; not media") + return media + + +@pytest.mark.parametrize("change", [ + {"answer": "B"}, {"answer": "A box moves right "}, {"answer": None}, + {"candidates": ["same", "same"], "answer": "same"}, + {"candidates": ["only one"], "answer": "only one"}, + {"candidates": [str(i) for i in range(27)], "answer": "0"}, + {"start": 1, "end": 1}, {"start": -1}, {"end": float("nan")}, + {"start": True}, {"video": "../clip.mp4"}, {"video": "/tmp/clip.mp4"}, + {"video": "https://example.invalid/video.mp4"}, {"video": "missing.mp4"}, +]) +def test_reject_bad_canonical_rows(row, root, change): + row.update(change) + with pytest.raises(ValueError): + custom.validate_row(row, root.resolve(), set()) + + +def test_labels_ids_ranges_and_symlinks(row, root, tmp_path): + seen = set() + validated, path = custom.validate_row(row, root.resolve(), seen) + assert validated == row and path == root / "clip.mp4" + with pytest.raises(ValueError, match="Duplicate id"): + custom.validate_row(row, root.resolve(), seen) + del row["end"] + with pytest.raises(ValueError, match="supplied together"): + custom.validate_row(row, root.resolve(), set()) + row.pop("start") + del row["answer"] + with pytest.raises(ValueError, match="answer"): + custom.validate_row(row, root.resolve(), set()) + outside = tmp_path / "outside.mp4" + outside.write_bytes(b"invented") + (root / "link.mp4").symlink_to(outside) + row.update(answer=row["candidates"][0], video="link.mp4") + with pytest.raises(ValueError, match="escaping"): + custom.validate_row(row, root.resolve(), set()) + + +def test_two_and_twenty_six_options_are_supported(row, root): + for count in (2, 26): + row.update(candidates=[f"Option {i}" for i in range(count)], answer=f"Option {count - 1}") + custom.validate_row(row, root.resolve(), set()) + assert custom.score_answer(row, chr(64 + count), "stop")["correct"] + + +def test_inference_payload_is_label_independent(row): + settings = {"model": {"name": "invented-model"}, "sampling": {"nframes": 8}, + "generation": {"temperature": 0, "seed": 42, "max_tokens": 32, "enable_thinking": False}} + payload = custom.build_payload(row, "data:video/mp4;base64,INVENTED", {"fps": 2}, settings) + changed = {**row, "answer": row["candidates"][0], "category": "SECRET_CATEGORY", "id": "SECRET_ID", "start": 50, "end": 60} + assert custom.build_payload(changed, "data:video/mp4;base64,INVENTED", {"fps": 2}, settings) == payload + text = json.dumps(payload) + assert "SECRET" not in text and '"answer"' not in text and '"category"' not in text + assert all(candidate in text for candidate in row["candidates"]) + + +def test_invalid_and_truncated_answers_stay_in_denominator(row): + rows = [{**row, "id": str(i), "category": "motion" if i < 2 else "other"} for i in range(5)] + answers = [("B", "stop"), ("B", "length"), ("B. explanation", "stop"), (None, "stop"), ("A", "stop")] + results = [{"id": r["id"], **custom.score_answer(r, answer, finish)} for r, (answer, finish) in zip(rows, answers)] + summary = custom.summarize(rows, results) + assert summary["status"] == "completed" + assert summary["coverage"] == {"expected": 5, "scored": 5} + assert summary["primary_metric"] == {"name": "accuracy_percent", "value": 20, "unit": "percent", "correct": 1, "total": 5} + assert summary["categories"]["motion"]["accuracy_percent"] == 50 + assert summary["categories"]["other"]["total"] == 3 + assert results[2]["prediction"] is None + + +def test_transport_failure_or_missing_rows_cannot_complete(row): + rows = [row, {**row, "id": "second"}] + good = {"id": row["id"], "correct": True} + for results in ([good], [good, {"id": "second", "error": "ConnectionError"}]): + summary = custom.summarize(rows, results) + assert summary["status"] == "failed" + assert summary["coverage"] == {"expected": 2, "scored": 1} + assert summary["primary_metric"]["value"] is None + assert summary["primary_metric"]["total"] == 2 + with pytest.raises(ValueError, match="duplicate"): + custom.summarize(rows, [good, good]) + + +@pytest.fixture +def real_config(tmp_path, row): + pytest.importorskip("decord") + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + pytest.skip("ffmpeg is required to generate invented media") + media = tmp_path / "media" + media.mkdir() + subprocess.run([ffmpeg, "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", + "color=c=blue:s=64x64:r=8:d=2", "-c:v", "libx264", "-pix_fmt", "yuv420p", + str(media / "clip.mp4")], check=True) + manifest = tmp_path / "rows.jsonl" + manifest.write_text(json.dumps(row) + "\n") + config = {"dataset": {"name": "invented", "revision": "test-v1", "source": "synthetic test metadata"}, + "manifest": "rows.jsonl", "media_root": "media", "nframes": 8, + "min_pixels": 1024, "max_pixels": 4096, "image_patch_size": 16, + "model": "invented-model-one", "model_revision": "invented-revision"} + path = tmp_path / "config.json" + path.write_text(json.dumps(config)) + return path, manifest, config + + +def test_real_validation_identity_is_model_and_cwd_independent(real_config, tmp_path, monkeypatch): + path, manifest, config = real_config + first = custom.validate(path) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + changed = copy.deepcopy(config) + changed["model"] = "invented-model-two" + changed["generation"] = {"seed": 123} + path.write_text(json.dumps(changed)) + second = custom.validate(path) + assert first["dataset_identity"] == second["dataset_identity"] + # Independently serialized equivalent canonical JSONL keeps semantic data identity. + annotation = json.loads(manifest.read_text()) + manifest.write_text(json.dumps(annotation, sort_keys=True) + "\n") + assert custom.validate(path)["dataset_identity"] == first["dataset_identity"] + annotation["answer"] = annotation["candidates"][0] + manifest.write_text(json.dumps(annotation) + "\n") + assert custom.validate(path)["dataset_identity"] != first["dataset_identity"] + annotation["end"] = 10 + manifest.write_text(json.dumps(annotation) + "\n") + with pytest.raises(ValueError, match="exceeds video duration"): + custom.validate(path) + + +def test_real_media_bytes_participate_in_identity(real_config): + path, _, _ = real_config + first = custom.validate(path) + with (path.parent / "media" / "clip.mp4").open("ab") as stream: + stream.write(b"invented trailing metadata") + assert custom.validate(path)["dataset_identity"] != first["dataset_identity"] + + +def test_unicode_line_separators_preserve_a_single_jsonl_question(real_config): + path, manifest, _ = real_config + annotation = json.loads(manifest.read_text()) + annotation["question"] = "What moves\u2028after the box\u2029stops?" + manifest.write_text(json.dumps(annotation, ensure_ascii=False) + "\n") + assert custom._load(path)[2][0]["question"] == annotation["question"] + + +def test_ancestor_media_root_cannot_include_source_media(tmp_path, monkeypatch, row): + source = tmp_path / "checkout" + (source / ".git").mkdir(parents=True) + (source / "launchable").mkdir() + (source / "clip.mp4").write_bytes(b"invented nonempty path fixture") + monkeypatch.setattr(custom, "__file__", str(source / "launchable" / "custom_eval.py")) + row["video"] = "checkout/clip.mp4" + with pytest.raises(ValueError, match="outside the source"): + custom.validate_row(row, tmp_path, set()) diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_execution_evidence.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_execution_evidence.py new file mode 100644 index 000000000..5d82a6186 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_execution_evidence.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Consumer-boundary evidence tests: no endpoint, model imports or media downloads.""" +import importlib.util +import json +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location( + 'execution_evidence', Path(__file__).resolve().parents[1] / 'vlmeval/execution_evidence.py') +evidence = importlib.util.module_from_spec(spec) +spec.loader.exec_module(evidence) + + +def video_response(sizes=None): + sizes = [49] * 16 if sizes is None else sizes + return {'model': 'nvidia/Cosmos3-Nano', 'usage': {'prompt_tokens': 850, 'completion_tokens': 1, 'total_tokens': 851}, + 'prompt_token_ids': [token for size in sizes for token in [151652, *([151656] * size), 151653]], + 'choices': [{'finish_reason': 'stop', 'message': {'content': '4'}}]} + + +def request(): + return {'model': 'nvidia/Cosmos3-Nano', 'temperature': 0, 'max_tokens': 256, + 'messages': [{'role': 'user', 'content': [{'type': 'video_url', 'video_url': {'url': 'data:video/mp4;base64,PRIVATE'}}, + {'type': 'text', 'text': 'PRIVATE QUESTION; answer label B'}]}], + 'mm_processor_kwargs': {'do_sample_frames': False}, + 'return_token_ids': True} + + +def record(raw=None, profile=True, media=None): + return evidence.build_execution_record(request(), video_response() if raw is None else raw, + processor_profile=evidence.nano_processor_profile('nvidia/Cosmos3-Nano') if profile else None, + media={'sampled_frames': 32, 'expected_group_tokens': 49} if media is None else media) + + +class ExecutionEvidenceTests(unittest.TestCase): + def test_per_group_area_and_frame_slots_are_not_decoded_frame_claims(self): + result = record() + self.assertEqual(result['verification']['status'], 'verified_under_explicit_profile') + observed = result['observed'] + self.assertEqual(observed['video_frame_slots'], 32) + self.assertEqual(observed['processor_grid_pixel_area_per_video_temporal_group'], [224 * 224] * 16) + self.assertIsNone(observed['decoded_video_frames']) + self.assertIsNone(observed['processor_width_height']) + self.assertIsNone(observed['effective_decoder_fps']) + self.assertIsNone(observed['effective_generation_settings']) + self.assertIsNone(observed['effective_model_revision']) + self.assertEqual(observed['model'], 'nvidia/Cosmos3-Nano') + self.assertEqual(observed['usage']['total_tokens'], 851) + self.assertEqual(observed['finish_reasons'], ['stop']) + self.assertEqual(result['requested']['generation']['temperature'], 0) + + def test_dropped_frame_slots_and_equal_total_pixel_mismatch_are_detected(self): + dropped = record(video_response([49] * 2)) + self.assertEqual(dropped['observed']['video_frame_slots'], 4) + self.assertEqual(dropped['verification']['checks']['video_frame_slots'], 'mismatch') + wrong_pixels = record(video_response([48, 50] + [49] * 14)) + self.assertEqual(wrong_pixels['verification']['checks']['video_frame_slots'], 'match') + self.assertEqual(wrong_pixels['verification']['checks']['pixel_geometry'], 'mismatch') + self.assertEqual(wrong_pixels['verification']['status'], 'mismatch') + + def test_unobservable_profile_tokens_or_requested_geometry_never_verify(self): + for result in (record(profile=False), record({'choices': video_response()['choices']}), + record(media={})): + with self.subTest(result=result['verification']): + self.assertEqual(result['verification']['status'], 'unknown') + self.assertIsNone(record(profile=False)['observed']['video_frame_slots']) + self.assertIsNone(evidence.nano_processor_profile('unrecognized-model')) + + def test_raw_unbounded_token_total_does_not_prove_temporal_geometry(self): + raw = video_response() + raw['prompt_token_ids'] = [151656] * 784 + result = record(raw) + self.assertIsNone(result['observed']['video_frame_slots']) + self.assertEqual(result['verification']['checks']['pixel_geometry'], 'mismatch') + + def test_failed_or_malformed_response_never_looks_verified(self): + for raw in ({}, {'error': {'message': 'PRIVATE API KEY'}}, + dict(video_response(), error={'message': 'failure'}), + dict(video_response(), choices=[]), + dict(video_response(), choices=[{'message': {'content': '4'}, 'finish_reason': 'error'}])): + result = record(raw) + self.assertEqual(result['verification']['status'], 'failed_response') + self.assertIsNone(result['observed']['video_frame_slots']) + + def test_redaction_is_allowlisted_not_recursive_copying(self): + payload = request() + payload.update(api_key='SECRET', authorization='Bearer SECRET', endpoint='https://u:p@host/?token=SECRET', + labels={'answer': 'PRIVATE LABEL'}, mm_processor_kwargs={'max_pixels': 50176, 'api_key': 'SECRET'}) + raw = video_response() + raw['choices'][0]['message']['content'] = 'PRIVATE OUTPUT' + raw.update(debug='SECRET', prompt='PRIVATE QUESTION') + result = evidence.build_execution_record(payload, raw, + media={'sampled_frames': 32, 'source': 'PRIVATE PATH', 'labels': 'PRIVATE LABEL'}, + processor_profile=dict(evidence.nano_processor_profile('nvidia/Cosmos3-Nano'), api_key='SECRET')) + encoded = json.dumps(result, allow_nan=False) + for secret in ('SECRET', 'PRIVATE', 'Bearer', 'data:video', 'https://u:p'): + self.assertNotIn(secret, encoded) + self.assertNotIn('prompt_token_ids', result['observed']) + self.assertEqual(result['requested']['mm_processor_kwargs']['max_pixels'], 50176) + + def test_image_pixel_mismatch_and_resize_ambiguity_remain_distinct(self): + payload = {'model': 'nvidia/Cosmos3-Nano', 'messages': [{'content': [{'type': 'image_url'}]}]} + raw = video_response([]) + raw['prompt_token_ids'] = [151652] + [151655] * 64 + [151653] + profile = evidence.nano_processor_profile('nvidia/Cosmos3-Nano') + aligned = evidence.build_execution_record(payload, raw, processor_profile=profile, + media={'image_dimensions': [{'width': 256, 'height': 256}]}) + self.assertEqual(aligned['verification']['status'], 'verified_under_explicit_profile') + mismatch = evidence.build_execution_record(payload, raw, processor_profile=profile, + media={'image_dimensions': [{'width': 256, 'height': 288}]}) + self.assertEqual(mismatch['verification']['checks']['pixel_geometry'], 'mismatch') + resized = evidence.build_execution_record(payload, raw, processor_profile=profile, + media={'image_dimensions': [{'width': 224, 'height': 224}]}) + self.assertEqual(resized['verification']['checks']['pixel_geometry'], 'unknown') + self.assertEqual(resized['observed']['processor_grid_pixel_area_per_image'], [65536]) + + def test_aggregate_retains_actual_shapes_and_never_counts_missing_evidence_as_coverage(self): + summary = evidence.aggregate_execution_records([record(), record(video_response([49] * 2)), {}], expected_requests=3) + self.assertEqual(summary['requests_recorded'], 2) + self.assertFalse(summary['coverage_complete']) + self.assertFalse(summary['all_requested_geometry_checks_matched_under_explicit_profile']) + self.assertEqual(summary['verification_counts']['mismatch'], 1) + self.assertEqual({shape['video_frame_slots'] for shape in summary['observed_visual_shapes']}, {4, 32}) + self.assertEqual(summary['usage_totals']['total_tokens'], 1702) + self.assertIsNone(summary['effective_model_revision']) + json.dumps(summary, allow_nan=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mmsivideo.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mmsivideo.py new file mode 100644 index 000000000..62e303e25 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_mmsivideo.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU-only boundary tests; no model imports, network, or public media downloads.""" +import hashlib +import importlib.util +from pathlib import Path +import stat +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +spec = importlib.util.spec_from_file_location('launchable_mmsivideo', Path(__file__).with_name('mmsivideo.py')) +mmsivideo = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mmsivideo) + + +class PublicStagingTests(unittest.TestCase): + def test_rejects_traversal_and_absolute_archive_members(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in ('../escape', '/absolute', 'frames/../../escape', 'frames\\escape'): + with self.subTest(name=name): + with zipfile.ZipFile(root / 'frames.zip', 'w') as archive: + archive.writestr(name, b'image') + with self.assertRaises(ValueError): + mmsivideo._extract(root, 'frames.zip') + + def test_rejects_archive_and_existing_directory_symlinks(self): + with tempfile.TemporaryDirectory() as directory, tempfile.TemporaryDirectory() as external: + root = Path(directory) + info = zipfile.ZipInfo('frames/link') + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(root / 'frames.zip', 'w') as archive: + archive.writestr(info, '../../escape') + with self.assertRaises(ValueError): + mmsivideo._extract(root, 'frames.zip') + (root / 'frames').symlink_to(external, target_is_directory=True) + with zipfile.ZipFile(root / 'frames.zip', 'w') as archive: + archive.writestr('frames/image.jpg', b'image') + with self.assertRaises(ValueError): + mmsivideo._extract(root, 'frames.zip') + self.assertFalse((Path(external) / 'image.jpg').exists()) + + def test_extracts_nested_canonical_layout(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with zipfile.ZipFile(root / 'frames.zip', 'w') as archive: + archive.writestr('frames/clip/001.jpg', b'canonical image') + result = mmsivideo._extract(root, 'frames.zip') + self.assertEqual(result['files'], 1) + self.assertEqual((root / 'frames/clip/001.jpg').read_bytes(), b'canonical image') + + def test_download_checksum_failure_never_promotes_partial(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'public.bin' + source.write_bytes(b'changed public data') + with self.assertRaises(ValueError): + mmsivideo._fetch(root, 'download.bin', source.as_uri(), hashlib.sha256(b'pinned').hexdigest()) + self.assertFalse((root / 'download.bin').exists()) + + def test_missing_assets_fail_before_dataset_or_endpoint(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with patch.object(mmsivideo, '_dataset') as dataset: + with self.assertRaises(ValueError): + mmsivideo.run(root / 'data', 'full', root / 'work') + dataset.assert_not_called() + failure = root / 'work' / 'failure.json' + self.assertTrue(failure.is_file()) + self.assertFalse((root / 'work' / 'summary.json').exists()) + + def test_refuses_reusing_work_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / 'work' + work.mkdir() + with patch.object(mmsivideo, '_verify_sources') as verify: + with self.assertRaises(FileExistsError): + mmsivideo.run(root / 'data', 'smoke', work) + verify.assert_not_called() + + def test_caps_endpoint_concurrency_before_creating_work(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for workers in (0, 5, -1, 1.5, True): + with self.subTest(workers=workers), self.assertRaises(ValueError): + mmsivideo.run(root / 'data', 'smoke', root / 'work', workers) + self.assertFalse((root / 'work').exists()) + + def test_refuses_output_inside_source_checkout(self): + with tempfile.TemporaryDirectory() as directory: + work = Path(mmsivideo.__file__).parent / 'forbidden-mmsi-test-output' + with patch.object(mmsivideo, '_verify_sources') as verify: + with self.assertRaises(ValueError): + mmsivideo.run(Path(directory), 'smoke', work) + verify.assert_not_called() + self.assertFalse(work.exists()) + + def test_truncation_preserves_http_success_but_not_http_failure(self): + self.assertTrue(mmsivideo._successful_transport(0, 200, None)) + self.assertTrue(mmsivideo._successful_transport(1, 200, 'truncated')) + self.assertFalse(mmsivideo._successful_transport(1, 200, 'http_error')) + self.assertFalse(mmsivideo._successful_transport(500, 500, 'truncated')) + self.assertFalse(mmsivideo._successful_transport(0, 500, None)) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_notebook_ui.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_notebook_ui.py new file mode 100644 index 000000000..8601d112a --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_notebook_ui.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Prevent mislabeled or incomplete cross-benchmark result cards.""" +from pathlib import Path +import unittest + +from notebook_ui import show_results + + +class ResultIdentityTests(unittest.TestCase): + def test_rejects_another_benchmarks_artifact_before_display(self): + with self.assertRaises(ValueError): + show_results({'benchmark': 'mmsivideo-bench', 'scope': 'full'}, + 'videophy2', 'full', Path('/unused')) + + def test_rejects_smoke_artifact_as_a_full_benchmark(self): + with self.assertRaises(ValueError): + show_results({'benchmark': 'perception-test', 'scope': 'smoke'}, + 'perception-test', 'full', Path('/unused')) + + def test_rejects_failed_run_even_with_a_numeric_score(self): + with self.assertRaises(ValueError): + show_results({'benchmark': 'videophy2', 'scope': 'full', 'status': 'failed', + 'primary_metric': {'value': 48.13, 'unit': 'pearson_x100'}}, + 'videophy2', 'full', Path('/unused')) + + def test_incomplete_coverage_cannot_be_presented_as_complete(self): + with self.assertRaises(ValueError): + show_results({'benchmark': 'mmsivideo-bench', 'scope': 'full', + 'status': 'completed', 'coverage': {'expected': 1106, 'scored': 1105}, + 'primary_metric': {'value': 27.94, 'unit': 'percent'}}, + 'mmsivideo-bench', 'full', Path('/unused')) + + +if __name__ == '__main__': + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_videophy2.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_videophy2.py new file mode 100644 index 000000000..937655c89 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_videophy2.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU-only scoring, media integrity, and audited-response boundaries.""" + +import io +import json +from pathlib import Path +import stat +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +import videophy2 + + +def response(output="4", finish="stop", count=784): + groups, remainder = divmod(count, 49) + sizes = [49] * groups + ([remainder] if remainder else []) + return {"prompt_token_ids": [token for size in sizes for token in [151652, *([151656] * size), 151653]], + "choices": [{"message": {"content": output}, "finish_reason": finish}]} + + +def rating_row(sa_truth, pc_truth, sa_output, pc_output): + return {"truth": {"sa": sa_truth, "pc": pc_truth}, + "responses": {task: {"score": videophy2.official_parse(output)[0]} + for task, output in (("sa", sa_output), ("pc", pc_output))}} + + +class ScoringTests(unittest.TestCase): + def test_official_parser_retains_order_and_invalid_zero(self): + cases = (("", (0, True)), ("invalid", (0, True)), ("5", (5, False)), + ("five", (5, False)), ("one or five", (1, False)), + ("0", (0, False)), ("9", (0, True))) + for text, expected in cases: + with self.subTest(text=text): + self.assertEqual(videophy2.official_parse(text), expected) + + def test_sa_and_pc_are_independent_and_mean_is_correlation_not_accuracy(self): + rows = [rating_row(1, 1, "1", "5"), rating_row(3, 3, "3", "3"), rating_row(5, 5, "5", "1")] + scores = videophy2.correlations(rows) + self.assertAlmostEqual(scores["sa_pearson_x100"], 100) + self.assertAlmostEqual(scores["pc_pearson_x100"], -100) + self.assertAlmostEqual(scores["mean_sa_pc_pearson_x100"], 0) + + def test_blank_predictions_stay_in_denominator(self): + scores = videophy2.correlations([rating_row(1, 1, "1", "1"), rating_row(3, 3, "3", "3"), rating_row(5, 5, "", "")]) + self.assertEqual(scores["rows"], 3) + self.assertLess(scores["mean_sa_pc_pearson_x100"], 0) + + def test_undefined_correlations_are_json_null(self): + scores = videophy2.correlations([rating_row(4, 2, "5", "4"), rating_row(2, 2, "1", "4")]) + self.assertAlmostEqual(scores["sa_pearson_x100"], 100) + self.assertIsNone(scores["pc_pearson_x100"]) + self.assertIsNone(scores["mean_sa_pc_pearson_x100"]) + json.dumps(scores, allow_nan=False) + + def test_truncated_or_blank_completed_transport_is_not_dropped(self): + self.assertEqual(videophy2.audit_response(response("", "length")), ("", "length", 784)) + self.assertEqual(videophy2.audit_response(response(None)), ("", "stop", 784)) + + def test_four_frame_and_incomplete_transport_are_rejected(self): + for raw in (response(count=98), response(finish=None), response(finish="error"), + {"choices": []}, {"prompt_token_ids": [151656] * 784, "choices": [{"finish_reason": "stop"}]}): + with self.subTest(raw_shape=list(raw)): + with self.assertRaises(ValueError): + videophy2.audit_response(raw) + + def test_equal_total_tokens_cannot_hide_per_group_pixel_mismatch(self): + raw = response() + raw["prompt_token_ids"] = [token for size in [48, 50] + [49] * 14 + for token in [151652, *([151656] * size), 151653]] + self.assertEqual(raw["prompt_token_ids"].count(151656), 784) + with self.assertRaises(ValueError): + videophy2.audit_response(raw) + + def test_unbounded_single_run_does_not_prove_32_frame_slots(self): + raw = response() + raw["prompt_token_ids"] = [151656] * 784 + with self.assertRaises(ValueError): + videophy2.audit_response(raw) + + +class MediaTests(unittest.TestCase): + def test_archive_mapping_uses_exact_path_then_model_not_caption(self): + row = {"video_url": "https://public.example/hunyuan_xdit/video%3Bname.mp4", "model_name": "hunyuan"} + self.assertEqual(videophy2.archive_match(row, ["hunyuan_xdit/video;name.mp4", "wan/video;name.mp4"]), "hunyuan_xdit/video;name.mp4") + row["video_url"] = "https://public.example/unavailable/video.mp4" + self.assertEqual(videophy2.archive_match(row, ["hunyuan_xdit/video.mp4", "wan/video.mp4"]), "hunyuan_xdit/video.mp4") + self.assertIsNone(videophy2.archive_match(row, ["hunyuan_xdit/different.mp4"])) + with self.assertRaises(ValueError): + videophy2.archive_match(row, ["hunyuan_xdit/video.mp4", "hunyuan_other/video.mp4"]) + + def test_safe_extraction_rejects_traversal_and_symlink(self): + for name, mode in (("../escape.mp4", 0), ("/absolute.mp4", 0), ("safe\\escape.mp4", 0), ("link.mp4", stat.S_IFLNK | 0o777)): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + info = zipfile.ZipInfo(name) + info.external_attr = mode << 16 + archive.writestr(info, b"outside") + buffer.seek(0) + with zipfile.ZipFile(buffer) as archive, self.assertRaises(ValueError): + videophy2._archive_files(archive) + + def test_all_selected_media_must_have_matching_bytes_and_hash(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / "media").mkdir() + (root / "media" / "00000.mp4").write_bytes(b"video") + row = {"video_url": "https://public.example/video.mp4"} + entry = {"index": 0, "sample_id": "videophy2_test:00000", "video_url": row["video_url"], + "bytes": 5, "sha256": videophy2._sha256(root / "media" / "00000.mp4"), + "dataset_sha256": videophy2.DATASET_SHA256, "archive_entry": None} + (root / "media-manifest.jsonl").write_text(json.dumps(entry) + "\n") + self.assertEqual(len(videophy2._preflight(root, [(0, row)])), 1) + with self.assertRaises(ValueError): + videophy2._preflight(root, [(0, row), (1, row)]) + (root / "media" / "00000.mp4").write_bytes(b"other") + with self.assertRaises(ValueError): + videophy2._preflight(root, [(0, row)]) + + def test_worker_bounds_and_model_revision_do_not_infer_endpoint_identity(self): + with patch.dict("os.environ", {}, clear=True): + for workers in (0, 5, 20, 1.5, True): + with self.subTest(workers=workers), self.assertRaises(ValueError): + videophy2._settings(workers) + _, _, settings = videophy2._settings(4) + self.assertEqual(settings["workers"], 4) + self.assertEqual(settings["model_revision"], "unknown") + + +if __name__ == "__main__": + unittest.main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/videophy2.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/videophy2.py new file mode 100644 index 000000000..a776a9279 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/launchable/videophy2.py @@ -0,0 +1,522 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned public VideoPhy2 categorical SA/PC protocol (not published Nano parity).""" + +import argparse +import concurrent.futures +import csv +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import time +import urllib.parse +import urllib.request +import zipfile + +DATASET_REVISION = "90b81ffa54f565d9e40e83b7a2c247ba1dccfa2b" +DATASET_SHA256 = "85a6690b9508b7e69c592f3cbcbc4113efd3a573eb5ec69d6ae030a8ffb8a4e7" +DATASET_URL = f"https://huggingface.co/datasets/videophysics/videophy2_test/resolve/{DATASET_REVISION}/videophy2_test.csv" +ARCHIVE_REVISION = "1b920c23525d460ec47c42a135cd2eeb805a64be" +ARCHIVE_SHA256 = "69578681c7831f987e37743ad6adc979b5d72d03279632b10d103d468571a453" +ARCHIVE_URL = f"https://huggingface.co/datasets/videophysics/videophy2_test_videos/resolve/{ARCHIVE_REVISION}/videophy2_test%20.zip" +SOURCE_REVISION = "01d366c979e4a82875852adbf427ccc036e7373a" +SOURCE_URL = f"https://github.com/Hritikbansal/videophy/tree/{SOURCE_REVISION}/VIDEOPHY2" +SYSTEM_PROMPT = "You are a helpful assistant. Answer the rating question with only a single integer from 1 to 5." +PC_QUESTION = "Does this video adhere to the physical laws? Rate the video on a scale from 1 to 5, where 5 means full compliance and 1 means significant violations." +NUM_MAP = dict(zip(("zero", "one", "two", "three", "four", "five", "0", "1", "2", "3", "4", "5"), (0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5))) +MODEL_DIRS = {"hunyuan": "hunyuan", "wan": "wan", "cosmos": "cosmos", "cogvideo": "cogvideo", "ray2": "luma", "sora": "sora", "videocrafter": "videocrafter"} +SMOKE_INDICES = (0, 1) + + +def _json(path, value): + path.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + + +def _sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _outside_source(path): + path = Path(path).expanduser().resolve() + source = next((parent for parent in Path(__file__).resolve().parents + if (parent / ".git").exists()), Path(__file__).resolve().parents[1]) + if path.is_relative_to(source): + raise ValueError("Benchmark data and outputs must be outside the source directory") + return path + + +def _selected(rows, scope): + if scope not in ("smoke", "full"): + raise ValueError("scope must be smoke or full") + return [(index, rows[index]) for index in (SMOKE_INDICES if scope == "smoke" else range(len(rows)))] + + +def _rows(root): + path = root / "videophy2_test.csv" + if _sha256(path) != DATASET_SHA256: + raise ValueError("VideoPhy2 CSV SHA256 mismatch") + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + if len(rows) != 3397: + raise ValueError("VideoPhy2 test split must contain exactly 3397 rows") + return rows + + +def _fetch(url, target, expected_sha256=None): + started = time.monotonic() + if target.is_file(): + digest = _sha256(target) + if expected_sha256 is not None and digest != expected_sha256: + raise ValueError(f"SHA256 mismatch: {target.name}") + return {"sha256": digest, "bytes": target.stat().st_size, "cached": True, + "elapsed_seconds": time.monotonic() - started} + parts = urllib.parse.urlsplit(url) + encoded = urllib.parse.urlunsplit(parts._replace(path=urllib.parse.quote(parts.path, safe="/%:;,@!$&'()*+=-._~"))) + temporary = target.with_suffix(target.suffix + ".partial") + digest = hashlib.sha256() + try: + with urllib.request.urlopen(encoded, timeout=300) as response, temporary.open("wb") as out: + for chunk in iter(lambda: response.read(1024 * 1024), b""): + digest.update(chunk) + out.write(chunk) + if expected_sha256 is not None and digest.hexdigest() != expected_sha256: + raise ValueError(f"SHA256 mismatch: {target.name}") + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + return {"sha256": digest.hexdigest(), "bytes": target.stat().st_size, "cached": False, + "elapsed_seconds": time.monotonic() - started} + + +def _archive_files(archive): + files = [] + for info in archive.infolist(): + path = PurePosixPath(info.filename) + if path.is_absolute() or ".." in path.parts or "\\" in info.filename or stat.S_ISLNK(info.external_attr >> 16): + raise ValueError("Unsafe entry in publisher archive") + if info.filename.lower().endswith(".mp4") and not info.filename.startswith("__MACOSX/"): + files.append(info.filename) + if len(files) != len(set(files)): + raise ValueError("Duplicate archive member names") + return files + + +def archive_match(row, files): + """Exact URL path, then basename and original model directory; never caption-match.""" + path = urllib.parse.unquote(urllib.parse.urlsplit(row["video_url"]).path).lstrip("/") + candidates = [name for name in files if name == path or name.endswith("/" + path)] + if not candidates: + candidates = [name for name in files if PurePosixPath(name).name == PurePosixPath(path).name] + if len(candidates) > 1: + candidates = [name for name in candidates if MODEL_DIRS[row["model_name"]] in name.lower()] + if len(candidates) > 1: + raise ValueError("Ambiguous publisher archive mapping") + return candidates[0] if candidates else None + + +def download(data_root: Path, scope: str): + """Download the pinned archive; absent entries use only their exact official CSV URL.""" + if scope not in ("smoke", "full"): + raise ValueError("scope must be smoke or full") + root = _outside_source(data_root) + root.mkdir(parents=True, exist_ok=True) + (root / "media").mkdir(exist_ok=True) + started = time.monotonic() + report = {"benchmark": "videophy2", "scope": scope, "dataset_revision": DATASET_REVISION, + "dataset_sha256": DATASET_SHA256, "archive_revision": ARCHIVE_REVISION, + "archive_sha256": ARCHIVE_SHA256, "sources": {}, "errors": []} + try: + report["sources"]["csv"] = _fetch(DATASET_URL, root / "videophy2_test.csv", DATASET_SHA256) + rows = _rows(root) + chosen = _selected(rows, scope) + report["coverage"] = {"expected": len(chosen), "downloaded": 0} + report["sources"]["archive"] = _fetch(ARCHIVE_URL, root / "official-test-videos.zip", ARCHIVE_SHA256) + manifest_path = root / "media-manifest.jsonl" + entries = {entry["index"]: entry for entry in (json.loads(line) for line in manifest_path.read_text().splitlines())} if manifest_path.exists() else {} + with zipfile.ZipFile(root / "official-test-videos.zip") as archive: + files = _archive_files(archive) + mappings = [(index, row, archive_match(row, files)) for index, row in chosen] + report["archive_media_files"] = len(files) + report["selected_archive_rows"] = sum(name is not None for _, _, name in mappings) + report["selected_original_url_rows"] = sum(name is None for _, _, name in mappings) + for index, row, name in mappings: + before = time.monotonic() + target = root / "media" / f"{index:05d}.mp4" + entry = {"index": index, "sample_id": f"videophy2_test:{index:05d}", + "video_url": row["video_url"], "model_name": row["model_name"], + "path": f"media/{index:05d}.mp4", "archive_entry": name, + "source": "publisher_archive" if name else "official_csv_url", + "dataset_sha256": DATASET_SHA256, + "archive_revision": ARCHIVE_REVISION if name else None, + "archive_sha256": ARCHIVE_SHA256 if name else None} + old = entries.get(index, {}) + try: + cached = (target.is_file() and old.get("archive_entry") == name + and old.get("video_url") == row["video_url"] + and old.get("sha256") == _sha256(target) and target.stat().st_size > 0) + if cached: + entry.update(sha256=old["sha256"], bytes=target.stat().st_size, cached=True) + elif name is not None: + temporary = target.with_suffix(".partial") + try: + with archive.open(name) as source, temporary.open("wb") as out: + shutil.copyfileobj(source, out, 1024 * 1024) + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + entry.update(sha256=_sha256(target), bytes=target.stat().st_size, cached=False) + else: + if target.exists(): + raise ValueError("Unverified cached original-URL media; remove it before downloading") + entry.update(_fetch(row["video_url"], target)) + if entry["bytes"] <= 0: + raise ValueError("Empty media") + except Exception as exc: + entry["error"] = type(exc).__name__ + report["errors"].append({"index": index, "error": type(exc).__name__}) + entry["elapsed_seconds"] = time.monotonic() - before + entries[index] = entry + # Append durable checkpoints; latest entry wins when resuming. + with manifest_path.open("a") as stream: + stream.write(json.dumps(entry, allow_nan=False) + "\n") + stream.flush() + manifest_path.write_text("".join(json.dumps(entries[key], allow_nan=False) + "\n" for key in sorted(entries))) + report["coverage"]["downloaded"] = sum("error" not in entries[index] for index, _ in chosen) + if report["errors"]: + raise RuntimeError("Selected VideoPhy2 media coverage is incomplete; see download-summary.json") + report["status"] = "completed" + except Exception as exc: + report["status"] = "failed" + report["failure"] = type(exc).__name__ + raise + finally: + report["elapsed_seconds"] = time.monotonic() - started + _json(root / "download-summary.json", report) + return report + + +def official_parse(output): + """Official inference.py num_map insertion-order substring/digit fallback semantics.""" + text = output.lower().strip() + for key, value in NUM_MAP.items(): + if key in text: + return value, False + digits = "".join(character for character in text if character.isdigit()) + # The official fallback emits zero when no valid rating is found. + try: + value = int(digits) if digits else 0 + except ValueError: + value = 0 + return (value if value in NUM_MAP.values() else 0), True + + +def correlations(results): + def pearson(task): + actual = [row["truth"][task] for row in results] + predicted = [row["responses"][task]["score"] for row in results] + if len(actual) < 2: + return None + mean_a, mean_p = sum(actual) / len(actual), sum(predicted) / len(predicted) + da, dp = [value - mean_a for value in actual], [value - mean_p for value in predicted] + variance_a, variance_p = sum(value * value for value in da), sum(value * value for value in dp) + if variance_a == 0 or variance_p == 0: + return None + return max(-100.0, min(100.0, 100 * sum(a * p for a, p in zip(da, dp)) / math.sqrt(variance_a * variance_p))) + sa, pc = pearson("sa"), pearson("pc") + return {"sa_pearson_x100": sa, "pc_pearson_x100": pc, + "mean_sa_pc_pearson_x100": (sa + pc) / 2 if sa is not None and pc is not None else None, + "rows": len(results)} + + +def _settings(api_nproc): + base = os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions") + parts = urllib.parse.urlsplit(base) + if parts.scheme not in ("http", "https") or not parts.netloc or not parts.path.endswith("/chat/completions"): + raise ValueError("COSMOS_API_BASE must be the complete /v1/chat/completions URL") + settings = {"model": os.environ.get("COSMOS_MODEL", "nvidia/Cosmos3-Nano"), + "model_revision": os.environ.get("COSMOS_MODEL_REVISION", "unknown"), + "temperature": 0, "seed": 42, "max_tokens": 256, + "timeout_seconds": float(os.environ.get("COSMOS_TIMEOUT", "300")), + "enable_thinking": False, "workers": api_nproc} + if isinstance(api_nproc, bool) or not isinstance(api_nproc, int) or not 1 <= api_nproc <= 4: + raise ValueError("api_nproc must be an integer from 1 to 4") + if settings["timeout_seconds"] <= 0 or not math.isfinite(settings["timeout_seconds"]): + raise ValueError("COSMOS_TIMEOUT must be finite and positive") + # Deliberately do not persist API URL or key; userinfo/query can contain credentials. + return base, os.environ.get("COSMOS_API_KEY", "EMPTY"), settings + + +def sample_video(path): + # Lazy imports keep public downloads, scoring and qualification CPU-only and lightweight. + import numpy as np + import torch + from decord import VideoReader, cpu + from vlmeval.api.cosmos_reason import tensor_video_to_base64 + + reader = VideoReader(str(path), ctx=cpu(0), height=224, width=224, num_threads=1) + total, source_fps = len(reader), float(reader.get_avg_fps()) + if total < 1 or source_fps <= 0 or not math.isfinite(source_fps): + raise ValueError("Video has no usable frames or FPS") + segment = float(total - 1) / 32 + start = int(segment / 2) + indices = np.array([start + int(np.round(segment * index)) for index in range(32)]) + frames = reader.get_batch(indices).asnumpy() + fps = 32 / (total / source_fps) + encoded = tensor_video_to_base64(torch.from_numpy(frames).permute(0, 3, 1, 2), fps=fps, crf=0) + return "data:video/mp4;base64," + encoded, {"source_frames": total, "source_fps": source_fps, + "indices": indices.tolist(), "sampled_frames": 32, "width": 224, "height": 224, + "transport_fps": fps, "crf": 0, + "transport_base64_sha256": hashlib.sha256(encoded.encode()).hexdigest()} + + +def audit_response(raw, processor_profile=None): + from vlmeval.execution_evidence import build_execution_record, nano_processor_profile + + profile = processor_profile or nano_processor_profile('nvidia/Cosmos3-Nano') + evidence = build_execution_record( + {'messages': [{'content': [{'type': 'video_url'}]}]}, raw, + media={'expected_frame_slots': 32, 'expected_group_tokens': 49}, processor_profile=profile) + if evidence['verification']['status'] != 'verified_under_explicit_profile': + raise ValueError('32-frame geometry audit failed: require 16 bounded groups of 49 video tokens (224x224 grid)') + count = sum(group['tokens'] for group in evidence['observed']['video_token_groups']) + choices = raw.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise ValueError("Incomplete or ambiguous response choices") + choice = choices[0] + if choice.get("finish_reason") not in ("stop", "length"): + raise ValueError("Incomplete response finish reason") + message = choice.get("message") + if not isinstance(message, dict) or "content" not in message: + raise ValueError("Incomplete response message") + content = message["content"] + if content is not None and not isinstance(content, str): + raise ValueError("Unexpected response content type") + return content or "", choice["finish_reason"], count + + +def _redact(value, secrets): + if isinstance(value, dict): + return {key: _redact(item, secrets) for key, item in value.items()} + if isinstance(value, list): + return [_redact(item, secrets) for item in value] + if isinstance(value, str): + for secret in secrets: + if secret: + value = value.replace(secret, "[REDACTED]") + return value + + +def _predict(pair, root, work, api, key, settings): + import requests + from vlmeval.execution_evidence import build_execution_record, nano_processor_profile + + profile = nano_processor_profile(settings['model'], settings['model_revision']) + + index, row = pair + started = time.monotonic() + result = {"index": index, "sample_id": f"videophy2_test:{index:05d}", "video_url": row["video_url"], + "model_name": row["model_name"], "truth": {task: int(row[task]) for task in ("sa", "pc", "joint")}, + "is_hard": int(row["is_hard"]), "responses": {}} + parts = urllib.parse.urlsplit(api) + secrets = [key if key != "EMPTY" else "", api, parts.username, parts.password, parts.query] + try: + video, sampling = sample_video(root / "media" / f"{index:05d}.mp4") + result["sampling"] = sampling + for task in ("sa", "pc"): + question = (f'Does this video match the description: "{row["caption"]}"? Please rate the video on a scale from 1 to 5, where 5 indicates a perfect match and 1 indicates no relevance.' if task == "sa" else PC_QUESTION) + payload = {"model": settings["model"], "messages": [{"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": [{"type": "video_url", "video_url": {"url": video}}, {"type": "text", "text": question}]}], + "temperature": settings["temperature"], "seed": settings["seed"], "max_tokens": settings["max_tokens"], + "chat_template_kwargs": {"enable_thinking": settings["enable_thinking"]}, + "mm_processor_kwargs": {"do_sample_frames": False, "fps": sampling["transport_fps"]}, + "media_io_kwargs": {"video": {"num_frames": 32, "fps": sampling["transport_fps"], "min_frames": 32, "max_frames": 32}}, + "return_token_ids": True} + evidence = work / "responses" / f"{index:05d}-{task}.json" + request_metadata = {name: value for name, value in payload.items() if name != "messages"} + request_metadata.update(system_prompt=SYSTEM_PROMPT, question=question, sampling=sampling) + record = {"request": request_metadata} + record['execution_evidence'] = build_execution_record( + payload, {}, media=dict(sampling, expected_frame_slots=32, expected_group_tokens=49), + processor_profile=profile, + requested_model_revision=os.environ.get("COSMOS_MODEL_REVISION", "unknown")) + before = time.monotonic() + try: + response = requests.post(api, json=payload, headers={"Authorization": f"Bearer {key}"}, timeout=settings["timeout_seconds"]) + record["http_status"] = response.status_code + try: + raw = response.json() + record["raw"] = raw + record['execution_evidence'] = build_execution_record( + payload, raw if 200 <= response.status_code < 300 else {}, + media=dict(sampling, expected_frame_slots=32, expected_group_tokens=49), + processor_profile=profile, + requested_model_revision=os.environ.get("COSMOS_MODEL_REVISION", "unknown")) + except ValueError: + record["response_text"] = response.text + raise ValueError("Non-JSON endpoint response") from None + response.raise_for_status() + if not isinstance(raw, dict): + raise ValueError("Non-object endpoint response") + if profile is None: + raise ValueError('VideoPhy2 geometry audit requires an explicit supported processor profile') + output, finish, count = audit_response(raw, processor_profile=profile) + rating, fallback = official_parse(output) + issues = [] + if finish == "length": + issues.append("truncated") + if output.strip() not in ("1", "2", "3", "4", "5"): + issues.append("invalid_categorical_answer") + result["responses"][task] = {"score": rating, "parser_fallback": fallback, "output": output, + "finish_reason": finish, "visual_token_count": count, "issues": issues, + "execution_evidence": record['execution_evidence'], + "evidence": str(evidence.relative_to(work)), "seconds": time.monotonic() - before} + except Exception as exc: + record["error"] = {"type": type(exc).__name__, "message": str(exc) if isinstance(exc, ValueError) else "Endpoint request failed; see HTTP status and raw response"} + raise + finally: + record["elapsed_seconds"] = time.monotonic() - before + _json(evidence, _redact(record, secrets)) + except Exception as exc: + result["error"] = {"type": type(exc).__name__, "message": str(exc) if isinstance(exc, ValueError) else "Media decode or endpoint request failed; inspect response evidence"} + result["seconds"] = time.monotonic() - started + return _redact(result, secrets) + + +def _preflight(root, chosen): + path = root / "media-manifest.jsonl" + entries = {entry["index"]: entry for entry in (json.loads(line) for line in path.read_text().splitlines())} + selected = [] + for index, row in chosen: + entry = entries.get(index) + target = root / "media" / f"{index:05d}.mp4" + if (entry is None or "error" in entry or entry.get("video_url") != row["video_url"] + or entry.get("sample_id") != f"videophy2_test:{index:05d}" or not target.is_file() + or entry.get("dataset_sha256") != DATASET_SHA256 + or (entry.get("archive_entry") is not None and + (entry.get("archive_revision") != ARCHIVE_REVISION or entry.get("archive_sha256") != ARCHIVE_SHA256)) + or target.stat().st_size <= 0 or entry.get("bytes") != target.stat().st_size + or entry.get("sha256") != _sha256(target)): + raise ValueError(f"Missing or unverified VideoPhy2 media for row {index}; run download first") + selected.append(entry) + return selected + + +def run(data_root: Path, scope: str, work: Path, api_nproc: int = 4): + """Require complete selected media, issue audited requests, and retain every rating.""" + root, work = _outside_source(data_root), _outside_source(work) + chosen = _selected(_rows(root), scope) + api, key, settings = _settings(api_nproc) + work.mkdir(parents=True, exist_ok=False, mode=0o700) + (work / "responses").mkdir(mode=0o700) + started, started_unix = time.monotonic(), time.time() + from vlmeval.execution_evidence import aggregate_execution_records, PINNED_SERVING_COMPATIBILITY + protocol = {"name": "official_public_categorical_32frame_protocol_adapted", "sampling": "official get_index segment-center, decord resize", + "frames": 32, "width": 224, "height": 224, "transport": "lossless CRF0 MP4 via native tensor_video_to_base64", + "system_prompt": SYSTEM_PROMPT, "parser": "official num_map substring then digit fallback; invalid blank returns zero", + "media_io_override": "num_frames=min_frames=max_frames=32; positive transport fps; do_sample_frames=false", + "response_audit": {"required_prompt_token_id": 151656, "required_count": 784, + "required_bounded_groups": 16, "required_tokens_per_group": 49, + "frame_slots": 32, "decoded_frames": None, "spatial_grid_area_pixels": 50176}, + "generation": settings, "truncation_policy": "retain and score with official parser; flag generation issue; never regenerate"} + provenance = {"benchmark": "videophy2", "dataset": {"url": DATASET_URL, "revision": DATASET_REVISION, "sha256": DATASET_SHA256, "rows": 3397}, + "archive": {"url": ARCHIVE_URL, "revision": ARCHIVE_REVISION, "sha256": ARCHIVE_SHA256}, + "official_source": {"url": SOURCE_URL, "revision": SOURCE_REVISION, "license": "MIT"}, + "runner_sha256": _sha256(Path(__file__)), "framework_revision": os.environ.get("COSMOS_COMMIT", "unknown"), + "model": settings["model"], "model_revision": settings["model_revision"], + "execution_evidence": {"summary": "execution-summary.json", "per_response": "responses/*-*.json"}, + "pinned_runtime_compatibility": PINNED_SERVING_COMPATIBILITY, + "reference_parity": False, "protocol": protocol, "sample_indices": [index for index, _ in chosen], "started_unix": started_unix} + encoder_source = Path(__file__).resolve().parent.parent / "vlmeval" / "api" / "cosmos_reason.py" + provenance["transport_encoder_sha256"] = _sha256(encoder_source) + provenance["execution_evidence_source_sha256"] = _sha256(encoder_source.parent.parent / "execution_evidence.py") + _json(work / "provenance.json", provenance) + warnings = ["Protocol-adapted public categorical evaluation; not a reproduction of the published Cosmos3-Nano 45.6 reference."] + results = [] + execution = aggregate_execution_records([], expected_requests=2 * len(chosen)) + try: + media = _preflight(root, chosen) + (work / "media-manifest.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in media)) + with (work / "predictions.jsonl").open("w") as predictions, (work / "run.log").open("w") as log: + with concurrent.futures.ThreadPoolExecutor(max_workers=settings["workers"]) as executor: + futures = [executor.submit(_predict, pair, root, work, api, key, settings) for pair in chosen] + for future in concurrent.futures.as_completed(futures): + result = future.result() + predictions.write(json.dumps(result, allow_nan=False) + "\n") + predictions.flush() + results.append(result) + progress = json.dumps({"index": result["index"], "completed": len(results), + "expected": len(chosen), "error": result.get("error"), + "seconds": result["seconds"]}) + log.write(progress + "\n") + log.flush() + print(progress, flush=True) + from vlmeval.execution_evidence import aggregate_execution_records + execution = aggregate_execution_records( + [json.loads(path.read_text()).get("execution_evidence", {}) for path in sorted((work / "responses").glob("*.json"))], + expected_requests=2 * len(chosen)) + execution.update(summary="execution-summary.json", per_response="responses/*-*.json") + _json(work / "execution-summary.json", execution) + provenance["execution_evidence"] = execution + _json(work / "provenance.json", provenance) + complete = [row for row in results if "error" not in row and all(task in row["responses"] for task in ("sa", "pc"))] + if len(complete) != len(chosen): + raise RuntimeError("VideoPhy2 transport/coverage failure; evidence retained, no partial benchmark score") + scores = correlations(complete) + groups = {"all": scores} + for name in sorted({row["model_name"] for row in complete}): + groups[name] = correlations([row for row in complete if row["model_name"] == name]) + groups["unseen_prompts"] = correlations([row for row in complete if row["model_name"] in ("hunyuan", "cosmos", "cogvideo")]) + groups["unseen_video_models"] = correlations([row for row in complete if row["model_name"] not in ("hunyuan", "cosmos", "cogvideo")]) + issue_count = sum(bool(answer["issues"]) for row in complete for answer in row["responses"].values()) + if issue_count: + warnings.append(f"{issue_count} answers had truncation or invalid categorical generation; all remain scored with the official parser.") + if scores["mean_sa_pc_pearson_x100"] is None: + warnings.append("One or both Pearson correlations are undefined (fewer than two rows or zero variance); null is intentional.") + summary = {"benchmark": "videophy2", "scope": scope, "status": "completed_with_generation_issues" if issue_count else "completed", + "primary_metric": {"name": "mean_sa_pc_pearson_x100", "value": scores["mean_sa_pc_pearson_x100"], "unit": "pearson_x100"}, + "coverage": {"expected": len(chosen), "scored": len(complete)}, "metrics": scores, "groups": groups, + "generation_issue_answers": issue_count, "protocol": protocol, "warnings": warnings, + "execution_evidence": execution, + "reference_parity": False, "model": settings["model"], "model_revision": settings["model_revision"], + "sample_indices": [index for index, _ in chosen], "started_unix": started_unix, + "ended_unix": time.time(), "elapsed_seconds": time.monotonic() - started} + _json(work / "summary.json", summary) + return summary + except Exception as exc: + scored = sum("error" not in row and all(task in row["responses"] for task in ("sa", "pc")) for row in results) + _json(work / "execution-summary.json", execution) + provenance["execution_evidence"] = dict(execution, summary="execution-summary.json", per_response="responses/*-*.json") + _json(work / "provenance.json", provenance) + _json(work / "summary.json", {"benchmark": "videophy2", "scope": scope, "status": "failed", + "primary_metric": {"name": "mean_sa_pc_pearson_x100", "value": None, "unit": "pearson_x100"}, + "coverage": {"expected": len(chosen), "scored": scored}, "protocol": protocol, + "execution_evidence": provenance["execution_evidence"], + "warnings": warnings + ["Incomplete transport or media coverage; no benchmark score is reported."], + "error": type(exc).__name__, "reference_parity": False, "elapsed_seconds": time.monotonic() - started}) + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("download", "run")) + parser.add_argument("--data-root", type=Path, required=True) + parser.add_argument("--scope", choices=("smoke", "full"), default="smoke") + parser.add_argument("--work", type=Path) + parser.add_argument("--api-nproc", type=int, default=4) + args = parser.parse_args() + if args.action == "run" and args.work is None: + parser.error("run requires --work (a new output directory)") + result = download(args.data_root, args.scope) if args.action == "download" else run(args.data_root, args.scope, args.work, args.api_nproc) + print(json.dumps(result, indent=2, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/api/cosmos_reason.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/api/cosmos_reason.py index d7d8e7fa9..c76dc8ca5 100644 --- a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/api/cosmos_reason.py +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/api/cosmos_reason.py @@ -297,6 +297,10 @@ def __init__(self, self.response_format = kwargs.pop('response_format', None) # Generic payload passthrough for vLLM-specific fields (e.g. structured_outputs). self.extra_body = kwargs.pop('extra_body', None) + # Opt-in; never forward evidence configuration to BaseAPI or the endpoint. + self.execution_evidence_dir = kwargs.pop('execution_evidence_dir', None) + self.execution_processor_profile = kwargs.pop('execution_processor_profile', None) + self._last_execution_record = None # Default Qwen3 / Reason2 setup. self.image_patch_size = kwargs.pop('image_patch_size', None) self.key = key @@ -533,13 +537,35 @@ def prepare_inputs(self, inputs): input_msgs.append(dict(role='user', content=output[0])) return input_msgs, video_kwargs + def _record_execution(self, payload, raw, media=None): + if self.execution_evidence_dir is None: + return + from pathlib import Path + import tempfile + from vlmeval.execution_evidence import build_execution_record + + record = build_execution_record(payload, raw, media=media, + processor_profile=self.execution_processor_profile, + requested_model_revision=os.environ.get("COSMOS_MODEL_REVISION", "unknown")) + directory = Path(self.execution_evidence_dir) + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + # O_EXCL + mode 0600 from mkstemp, unique across processes/threads/retries. + fd, path = tempfile.mkstemp(prefix='response-', suffix='.json', dir=directory) + with os.fdopen(fd, 'w') as output: + json.dump(record, output, allow_nan=False) + output.write('\n') + self._last_execution_record = record + def generate_inner(self, inputs, **kwargs) -> str: self._last_error_type = None # reset per-sample; prevents leaking across samples + self._last_execution_record = None + execution_media = {} try: input_msgs, video_kwargs = self.prepare_inputs(inputs) except Exception as e: # Non-fail_msg sentinel skips BaseAPI retry (it retries only when fail_msg in answer). logger.warning(f'Video preparation failed (skipping): {type(e).__name__}: {e}') + self._record_execution({'model': self.model}, {}) return 0, 'VIDEO_LOAD_FAILED', None kwargs.pop('dataset', None) temperature = kwargs.pop('temperature', self.temperature) @@ -584,6 +610,22 @@ def generate_inner(self, inputs, **kwargs) -> str: 'mm_processor_kwargs': video_kwargs }) + if self.execution_evidence_dir is not None: + payload['return_token_ids'] = True + # Input geometry is local request evidence, not effective server geometry. + messages = inputs if 'role' not in inputs[0] else [ + item for message in inputs for item in message['content']] + images = [item for item in messages if item.get('type') == 'image'] + if images and self.img_size == -1: + from PIL import Image + dimensions = [] + for image in images: + with Image.open(image['value']) as source: + dimensions.append({'width': source.width, 'height': source.height}) + execution_media['image_dimensions'] = dimensions + if self.nframes is not None: + execution_media['nframes'] = self.nframes + THINKING_RETRY = 1 for i in range(THINKING_RETRY): payload['seed'] += i @@ -603,21 +645,32 @@ def generate_inner(self, inputs, **kwargs) -> str: logger.error(f'⚠️ ENDPOINT TIMEOUT: {self.api_base}') logger.error(f'⚠️ The endpoint took longer than {self.timeout}s to respond. It may be down or overloaded.') self._last_error_type = "timeout" + self._record_execution(payload, {}, execution_media) return 408, self.fail_msg, None except requests.exceptions.ConnectionError as e: logger.error(f'⚠️ ENDPOINT DOWN: {self.api_base}') logger.error(f'⚠️ Cannot connect to the endpoint. Please check if the service is running.') logger.error(f'⚠️ Error: {str(e)}') self._last_error_type = "http_error" + self._record_execution(payload, {}, execution_media) return 503, self.fail_msg, None except Exception as e: logger.error(f'⚠️ REQUEST FAILED: {type(e).__name__}: {str(e)}') self._last_error_type = "http_error" + self._record_execution(payload, {}, execution_media) return 500, self.fail_msg, None ret_code = response.status_code ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code + if self.execution_evidence_dir is not None: + try: + evidence_raw = response.json() + except ValueError: + evidence_raw = {} + if ret_code != 0: + evidence_raw = {} + self._record_execution(payload, evidence_raw, execution_media) # Check for common error status codes if ret_code == 401: diff --git a/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/execution_evidence.py b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/execution_evidence.py new file mode 100644 index 000000000..86acb5158 --- /dev/null +++ b/.agents/skills/evaluation/cosmos3/reasoner/vlmevalkit/vlmeval/execution_evidence.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Redacted, observational evidence; requested settings are never server attestations.""" +from __future__ import annotations + +from collections import Counter +import json +import math +import re + +NANO_REVISION = 'e59a53c25979a090fa8706c9acc0c254a6e89b92' +PINNED_SERVING_COMPATIBILITY = { + 'image': 'vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55', + 'inspected_version': '0.23.1rc1.dev1306+gbfee6a802', + 'inspection_scope': 'existing review container source and serve help; not attestation of this response endpoint', + 'sampler_environment': {'VLLM_USE_FLASHINFER_SAMPLER': '0'}, + 'trust_request_mm_kwargs_flag_supported': False, + 'supported_equivalent': 'unconditional request mm_processor_kwargs via renderers/online_renderer.py RenderParams and renderers/hf.py', +} + + +def _identity(value): + return value if isinstance(value, str) and re.fullmatch(r'[A-Za-z0-9_./+:-]{1,240}', value) and '://' not in value else None + + +def _number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _knobs(value, names): + if not isinstance(value, dict): + return {} + return {key: value[key] for key in names if key in value and + (value[key] is None or isinstance(value[key], bool) or _number(value[key]))} + + +def nano_processor_profile(model, revision='unknown'): + """Explicit pinned architecture assumption, NOT proof of the endpoint revision.""" + if model != 'nvidia/Cosmos3-Nano': + return None + return {'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, + 'image_token_id': 151655, 'video_token_id': 151656, + 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, + 'min_pixels': 65536, 'max_pixels': 16777216, + 'profile_model': model, 'profile_revision': NANO_REVISION, + 'requested_model_revision': revision, + 'provenance': 'pinned-public-nano-config'} + + +def _profile(value): + if not isinstance(value, dict): + return None + names = ('patch_size', 'spatial_merge_size', 'temporal_patch_size', 'image_token_id', + 'video_token_id', 'vision_start_token_id', 'vision_end_token_id') + result = {key: value[key] for key in names if isinstance(value.get(key), int) + and not isinstance(value[key], bool) and value[key] > 0} + for key in ('min_pixels', 'max_pixels'): + if isinstance(value.get(key), int) and not isinstance(value[key], bool) and value[key] > 0: + result[key] = value[key] + if not all(key in result for key in names[:5]): + return None + result['basis'] = 'explicit caller-supplied processor profile; not server-attested' + if value.get('provenance') == 'pinned-public-nano-config': + result.update(profile_model='nvidia/Cosmos3-Nano', profile_revision=NANO_REVISION, + sources=[f'https://huggingface.co/nvidia/Cosmos3-Nano/resolve/{NANO_REVISION}/{name}' + for name in ('config.json', 'preprocessor_config.json')]) + return result + + +def _groups(ids, token, profile): + groups = [] + index = 0 + while index < len(ids): + if ids[index] != token: + index += 1 + continue + start = index + while index < len(ids) and ids[index] == token: + index += 1 + groups.append({'tokens': index - start, + 'bounded_by_vision_markers': bool(start > 0 and index < len(ids) and + ids[start - 1] == profile.get('vision_start_token_id') and + ids[index] == profile.get('vision_end_token_id'))}) + return groups + + +def build_execution_record(payload: dict, response_json: dict, *, media=None, processor_profile=None, + requested_model_revision=None) -> dict: + """Keep only allowlisted settings and visual-token geometry, never prompts/URLs/labels. + + A video group describes one temporal patch (frame slots), not decoded unique + frames. Repeated/padded frames, pruning, width/height and decoder FPS cannot be + established from token counts. Pixel area is conditional on the explicit + processor profile and unpruned standard Qwen3-VL placeholder expansion. + """ + raw = response_json if isinstance(response_json, dict) else {} + media = media if isinstance(media, dict) else {} + profile = _profile(processor_profile) + generation = _knobs(payload, ('temperature', 'top_p', 'top_k', 'seed', 'max_tokens', + 'max_completion_tokens', 'repetition_penalty', 'presence_penalty', + 'frequency_penalty', 'n')) + processor = _knobs(payload.get('mm_processor_kwargs'), + ('fps', 'do_sample_frames', 'min_pixels', 'max_pixels', 'total_pixels', + 'nframes', 'min_frames', 'max_frames', 'do_resize')) + io = payload.get('media_io_kwargs', {}) + io = {kind: _knobs(io.get(kind), ('num_frames', 'fps', 'min_frames', 'max_frames')) + for kind in ('image', 'video') if isinstance(io, dict) and kind in io} + parts = [part for msg in payload.get('messages', []) if isinstance(msg, dict) + for part in (msg.get('content') if isinstance(msg.get('content'), list) else []) + if isinstance(part, dict)] + images = sum(part.get('type') == 'image_url' for part in parts) + videos = sum(part.get('type') == 'video_url' for part in parts) + requested_media = _knobs(media, ('sampled_frames', 'width', 'height', 'expected_frame_slots', + 'expected_group_tokens', 'nframes', 'min_pixels', 'max_pixels', + 'fps', 'total_pixels', 'max_frames')) + dimensions = media.get('image_dimensions', []) + if isinstance(dimensions, list) and all(isinstance(item, dict) and + all(_number(item.get(key)) and item[key] > 0 for key in ('width', 'height')) for item in dimensions): + requested_media['image_dimensions'] = [_knobs(item, ('width', 'height')) for item in dimensions] + choices = raw.get('choices') + valid = ('error' not in raw and isinstance(choices, list) and bool(choices) and + all(isinstance(choice, dict) and choice.get('finish_reason') in ('stop', 'length') and + isinstance(choice.get('message'), dict) and 'content' in choice['message'] and + (choice['message']['content'] is None or isinstance(choice['message']['content'], str)) + for choice in choices)) + observed = {'response_valid': bool(valid), 'model': _identity(raw.get('model')), + 'usage': _knobs(raw.get('usage'), ('prompt_tokens', 'completion_tokens', 'total_tokens')), + 'finish_reasons': [choice.get('finish_reason') if isinstance(choice, dict) and + choice.get('finish_reason') in ('stop', 'length', 'error', 'content_filter') else None + for choice in choices] if isinstance(choices, list) else [], + 'image_token_groups': None, 'video_token_groups': None, + 'processor_grid_pixel_area_per_image': None, 'processor_grid_pixel_area_per_video_temporal_group': None, + 'video_frame_slots': None, 'decoded_video_frames': None, 'processor_width_height': None, + 'effective_generation_settings': None, 'effective_model_revision': None, + 'effective_decoder_fps': None} + checks = {'image_count': 'unknown', 'video_frame_slots': 'unknown', 'pixel_geometry': 'unknown'} + ids = raw.get('prompt_token_ids') + if valid and profile and isinstance(ids, list) and all(isinstance(token, int) for token in ids): + ig = _groups(ids, profile['image_token_id'], profile) + vg = _groups(ids, profile['video_token_id'], profile) + area = (profile['patch_size'] * profile['spatial_merge_size']) ** 2 + observed.update(image_token_groups=ig, video_token_groups=vg, + processor_grid_pixel_area_per_image=[group['tokens'] * area for group in ig], + processor_grid_pixel_area_per_video_temporal_group=[group['tokens'] * area for group in vg]) + bounded = all(group['bounded_by_vision_markers'] for group in ig + vg) + if images and not videos: + checks['image_count'] = 'match' if bounded and len(ig) == images and not vg else 'mismatch' + if videos and vg and bounded and not ig: + observed['video_frame_slots'] = len(vg) * profile['temporal_patch_size'] + expected = media.get('expected_frame_slots', media.get('sampled_frames', media.get('nframes'))) + if _number(expected): + checks['video_frame_slots'] = 'match' if observed['video_frame_slots'] == expected else 'mismatch' + expected_tokens = media.get('expected_group_tokens') + if _number(expected_tokens): + groups = vg if videos else ig + checks['pixel_geometry'] = 'match' if groups and bounded and all( + group['tokens'] == expected_tokens for group in groups) else 'mismatch' + elif images and len(requested_media.get('image_dimensions', [])) == images and bounded and len(ig) == images: + # Alignment alone does not prevent smart_resize from enlarging or shrinking. + dims = requested_media['image_dimensions'] + factor = profile['patch_size'] * profile['spatial_merge_size'] + minimum = processor.get('min_pixels', profile.get('min_pixels')) + maximum = processor.get('max_pixels', profile.get('max_pixels')) + if (_number(minimum) and _number(maximum) and all( + item['width'] % factor == 0 and item['height'] % factor == 0 and + minimum <= item['width'] * item['height'] <= maximum for item in dims)): + checks['pixel_geometry'] = 'match' if all(group['tokens'] * area == item['width'] * item['height'] + for group, item in zip(ig, dims)) else 'mismatch' + if (ig or vg) and bounded: + cap = processor.get('max_pixels') + if _number(cap) and any(group['tokens'] * area > cap for group in ig + vg): + checks['pixel_geometry'] = 'mismatch' + required = ['image_count', 'pixel_geometry'] if images and not videos else ['video_frame_slots', 'pixel_geometry'] + status = ('failed_response' if not valid else 'mismatch' if 'mismatch' in checks.values() else + 'verified_under_explicit_profile' if all(checks[key] == 'match' for key in required) else 'unknown') + return {'format_version': 1, + 'requested': {'model': _identity(payload.get('model')), 'generation': generation, + 'model_revision': _identity(requested_model_revision), + 'mm_processor_kwargs': processor, 'media_io_kwargs': io, + 'chat_template_kwargs': _knobs(payload.get('chat_template_kwargs'), ('enable_thinking',)), + 'image_inputs': images, 'video_inputs': videos, 'media': requested_media}, + 'observed': observed, 'processor_profile': profile, + 'verification': {'status': status, 'checks': checks}, + 'derivation': {'pixel_area_units': 'pixels per image / per temporal group spatial grid (not summed over time)', + 'pixel_area_formula': 'consecutive visual tokens * (patch_size * spatial_merge_size)^2', + 'video_frame_slots_formula': 'bounded video token groups * temporal_patch_size', + 'limits': ['conditional on explicit processor profile and no visual-token pruning', + 'token groups do not prove decoded/unique frames or source sampling indices', + 'area does not establish width, height, crop, decoder FPS or source resolution', + 'generation settings and model revision are requested/configured, not server-attested']}} + + +def aggregate_execution_records(records, expected_requests=None): + records = [record for record in records if isinstance(record, dict) and + isinstance(record.get('requested'), dict) and isinstance(record.get('observed'), dict)] + counts = Counter(record.get('verification', {}).get('status', 'unknown') for record in records) + return {'requests_recorded': len(records), 'expected_requests': expected_requests, + 'coverage_complete': expected_requests is not None and len(records) == expected_requests, + 'verification_counts': dict(counts), + 'all_requested_geometry_checks_matched_under_explicit_profile': bool(records) and + expected_requests == len(records) and counts.get('verified_under_explicit_profile', 0) == len(records), + 'observed_models': sorted({record['observed']['model'] for record in records if record.get('observed', {}).get('model')}), + 'finish_reasons': dict(Counter(reason for record in records for reason in + record.get('observed', {}).get('finish_reasons', []) if reason)), + 'observed_visual_shapes': [ + {'image_group_tokens': list(shape[0]), 'video_group_tokens': list(shape[1]), + 'image_grid_pixel_areas': list(shape[2]), 'video_temporal_grid_pixel_areas': list(shape[3]), + 'video_frame_slots': shape[4], 'requests': count} + for shape, count in Counter( + (tuple(group['tokens'] for group in (record.get('observed', {}).get('image_token_groups') or [])), + tuple(group['tokens'] for group in (record.get('observed', {}).get('video_token_groups') or [])), + tuple(record.get('observed', {}).get('processor_grid_pixel_area_per_image') or []), + tuple(record.get('observed', {}).get('processor_grid_pixel_area_per_video_temporal_group') or []), + record.get('observed', {}).get('video_frame_slots')) for record in records).items()], + 'requested_settings': [ + {'settings': json.loads(value), 'requests': count} + for value, count in Counter(json.dumps(record.get('requested', {}), sort_keys=True) + for record in records).items()], + 'usage_totals': {key: sum(record.get('observed', {}).get('usage', {}).get(key, 0) or 0 for record in records) + for key in ('prompt_tokens', 'completion_tokens', 'total_tokens')}, + 'unknown_fields': ['decoded_video_frames', 'processor_width_height', 'effective_decoder_fps', + 'effective_generation_settings', 'effective_model_revision'], + 'effective_generation_settings': None, 'effective_model_revision': None, + 'decoded_video_frames': None} diff --git a/.agents/skills/kaizen-ui/SKILL.md b/.agents/skills/kaizen-ui/SKILL.md new file mode 100644 index 000000000..4741a47fa --- /dev/null +++ b/.agents/skills/kaizen-ui/SKILL.md @@ -0,0 +1,220 @@ +--- +name: kaizen-ui +description: Advises on Kaizen UI (KUI) React components, design patterns, and page layouts for NVIDIA applications. Applies to all UI work in a KUI React application—building, modifying, or improving an interface, screen, page, or component—even when the user does not mention KUI, design systems, or best practices. Covers forms, filters, search, sort, cards, settings, navigation, detail and key-value displays, dashboards, charts and data visualization, stat blocks, ranked lists, activity feeds, loading, empty and error states, feedback, iconography, visibility, and page layout decisions. +metadata: + library: "kaizen-ui" + library_version: "1.19.0" + type: "core" + version: "5" +--- + +# Kaizen UI Foundations React docs + +Official documentation for Kaizen UI Foundations (KUI), NVIDIA's design system. + +KUI is built from theme-responsive design tokens, shared component CSS, and framework components. The React implementation composes KUI component primitives and shared CSS through typed React APIs. An optional Tailwind plugin exposes the same design tokens as utilities. + +Read [CONFIGURATION.md](./references/guides/CONFIGURATION.md) when installing or configuring KUI React. + +## Gotchas + +- Use existing KUI components over custom implementations. +- Prefer semantic CSS tokens over primitives, for example `text-primary` over `text-gray-100`, so themes work automatically. +- Use sentence case except for acronyms. Do not use all-caps labels or emojis. + +## Skill workflow + +For UI work in a KUI application: + +1. **Identify the page type.** Load the matching reference from [Page type routing](#page-type-routing) before placing content. +2. **Identify the patterns in play.** Load every applicable reference from [Pattern routing](#pattern-routing). +3. **Build outside-in.** Apply [page shell](./references/layouts/page-shell.md), then [layout principles](./references/layouts/layout-principles.md), then patterns and components. +4. **Apply the framework contract.** Use the documented React component and prop APIs. Read [COMPOSITION.md](./references/guides/COMPOSITION.md) when composing through `asChild` or component primitives. +5. **Load component references.** Read each selected component's generated reference under [Components](#components) before implementation. Each reference is generated from the component's source, so its props and exports are complete and current — treat them as authoritative and do not re-derive them from `node_modules` or package source. If something you need is genuinely absent from the reference, it is a documentation gap worth reporting. +6. **Verify the result.** Check semantic markup, accessible names, keyboard behavior, responsive layout, and loading, empty, error, and disabled states. +7. **Re-evaluate existing structure before extending it.** A new region, fourth column, repeated slot, or deeply nested wrapper can signal that the page needs restructuring. Read [page-modification.md](./references/layouts/page-modification.md). + +## Page type routing + +Determine the page type first; its layout guide drives downstream pattern and component decisions. + +- [Browse: find, filter, compare, or select from a collection](./references/layouts/browse-page.md) +- [List view: filterable table or list of items](./references/layouts/list-view.md) +- [Detail: inspect, monitor, or manage one entity](./references/layouts/detail-page.md) +- [Dashboard: aggregated metrics, trends, progress, or health](./references/layouts/dashboard.md) +- [Form: create or edit an entity; input that gets submitted](./references/layouts/form-page.md) +- [Workspace: editor, AI, or command-center surface with a persistent task area](./references/layouts/workspace-page.md) +- [Settings or preferences](./references/patterns/settings.md) with [page shell](./references/layouts/page-shell.md) +- [Full-page error: terminal route failure such as 401, 403, 404, or 500](./references/layouts/error-page.md) + +For a single entity, choose between a contextual panel, standalone page, or both with [detail-placement.md](./references/layouts/detail-placement.md). + +## Pattern routing + +Load every pattern guide whose feature appears on the page. + +- [Navigation — application, section, breadcrumb, and content navigation](./references/patterns/navigation.md) +- [Forms — create, edit, configure, and submit](./references/patterns/forms.md) +- [Filters and sort controls](./references/patterns/filters.md) +- [Search input and query behavior](./references/patterns/search.md) +- [Tables, card grids, or lists of structured data](./references/patterns/data-display.md) +- [Key-value display — read-only labeled facts](./references/patterns/key-value-display.md) +- [Charts, graphs, gauges, and other data visualization](./references/patterns/data-visualization.md) +- [Stat blocks / KPI tiles](./references/patterns/stat-blocks.md) +- [Ranked lists — "top N by X" contributor panels](./references/patterns/ranked-lists.md) +- [Activity feed — chronological events, history, audit log, or timeline](./references/patterns/activity-feed.md) +- [Settings and persistent preferences](./references/patterns/settings.md) +- [Compare flows — select one alternative](./references/patterns/data-display.md#single-selection-among-alternatives-compare-flows) +- [Entity cards — content and interaction inside cards](./references/patterns/entity-cards.md) +- [Toasts, notifications, banners, and durable feedback](./references/patterns/feedback.md) +- [Loading and progress states](./references/patterns/loading.md) +- [Empty states](./references/patterns/empty-states.md) +- [Error states](./references/patterns/error-states.md) +- [Confirmations for destructive or irreversible actions](./references/patterns/confirmations.md) +- [Element visibility — hide, read-only, disable, or enable](./references/patterns/element-visibility.md) +- [Icons — when to use them and which icon fits an action](./references/ICONS.md) + +## Custom work with KUI + +Reuse KUI components and tokens before introducing a parallel visual language. + +- Prefer extending existing components or composing their documented subcomponents before creating a new primitive. +- Read [COMPOSITION.md](./references/guides/COMPOSITION.md) before using `asChild` or replacing a component's rendered element. +- Token references name tokens in their Tailwind spelling. Use those utilities when the application configures `@nvidia/foundations-tailwind-plugin`; otherwise write CSS against the matching variable. Check the application's stylesheet for the plugin import to see which applies. +- Prefixes map one to one: `bg-*` → `--background-color-*`, `text-*` → `--text-color-*`, `border-*` → `--border-color-*`, `rounded-*` → `--radius-*`, `shadow-*` → `--shadow-*`, `gap-*` and `p-*` → `--spacing-*`. +- Reserve inline `style` for runtime values and documented custom properties. Never inline a static value that a utility or a stylesheet rule can carry. +- Load the relevant token references: + - [Colors, backgrounds, borders, and accents](./references/design-tokens/COLORS.md) + - [Chart and data-visualization colors](./references/design-tokens/CHART_DATA_COLORS.md) + - [Raw color primitives — last resort and not theme-aware](./references/design-tokens/COLOR_PRIMITIVES.md) + - [Typography](./references/design-tokens/TYPOGRAPHY.md) + - [Spacing, padding, and gap](./references/design-tokens/SPACING_SCALE.md) + - [Shadows, breakpoints, container widths, transitions, and animations](./references/design-tokens/EFFECTS_AND_LAYOUT.md) + +## Components + +Load every component reference you use for exact imports, props, composition, examples, and related components. + +### Actions + +- [Anchor](references/components/Anchor.md) - Interactive text that navigates the user to another page, section, or resource. + - sometimes known as Link +- [Button](references/components/Button.md) - A clickable element that triggers an action. Use specific verb + noun labels instead of vague text. +- [ButtonGroup](references/components/ButtonGroup.md) - A row of related buttons presented as a single set of actions with consistent styling. +- [Command](references/components/Command.md) - A modal menu that lets users search for and run actions or jump to pages from the keyboard. Opens over the page, filters its items as the user types, and closes when an item is chosen. +- [Dropdown](references/components/Dropdown.md) - A button that opens a menu of actions, links, or selectable options. Use for contextual actions, filters, and overflow menus. +- [Menu](references/components/Menu.md) - A list of actions, toggles, or selections. Use when the list should always be visible (sidebars, settings panels). +- [SegmentedControl](references/components/SegmentedControl.md) - A horizontal group of mutually exclusive options where selection takes effect immediately. Use for switching between alternate views or modes of the same content. +- [Switch](references/components/Switch.md) - A two-state toggle for binary settings whose change takes effect immediately. Commonly used for preferences and feature toggles. + - sometimes known as Toggle, ToggleSwitch +- [ToggleButton](references/components/ToggleButton.md) - A button that switches an option on or off and keeps showing which state it is in. Use for a state that persists after the click, such as pinning, starring, or bold text. + - sometimes known as Toggle, Pressed Button, Stateful Button +- [ToggleButtonGroup](references/components/ToggleButtonGroup.md) - A set of related toggle buttons that each switch an option on or off, with one tab stop for the whole set. Use for options that apply to the same content, such as text formatting or which panels are visible. + +### Chat + +- [ChatBubble](references/components/ChatBubble.md) - The surface of one chat message: a padded, rounded block for a person's words, or flat text for an AI reply. + +### Data + +- [Avatar](references/components/Avatar.md) - A small visual identifier for a user, account, or entity. Prioritizes an image when available and falls back to initials or an icon otherwise. +- [Badge](references/components/Badge.md) - A small color-coded label that displays status, category, or metadata at a glance. Badges are non-interactive and convey information rather than trigger actions. +- [CodeSnippet](references/components/CodeSnippet.md) - Displays source code with syntax highlighting and a copy button. Pick `kind="inline"` for code samples within prose and `kind="block"` for standalone, multi-line samples. +- [Kbd](references/components/Kbd.md) - A visual representation of a keyboard key or shortcut. Place it beside the control or instruction it describes. +- [KeyValueGroup](references/components/KeyValueGroup.md) - A description list of labeled values, such as the properties of one resource. Renders a native `
    `. +- [Legend](references/components/Legend.md) - A key mapping colors to the data series they represent. +- [List](references/components/List.md) - Groups related items into a sequence, supporting numbered, bulleted, and icon-marked formats. +- [Meter](references/components/Meter.md) - A bar showing how much of a bounded resource is in use. +- [SegmentedMeter](references/components/SegmentedMeter.md) - A bar showing how a bounded quantity divides across categories. +- [Severity](references/components/Severity.md) - A ranked severity level shown as a fixed icon and label pairing. Use it to communicate urgency, risk, or priority on issues, alerts, and incidents. +- [StatusIndicator](references/components/StatusIndicator.md) - A small colored dot that signals a new change or short-lived status. +- [Table](references/components/Table.md) - Organizes sets of structured data into columns and rows so users can scan, compare, and analyze information. +- [Tag](references/components/Tag.md) - A small label that categorizes or groups an item with text and optional icons. Use for classification and filtering; reach for Badge to communicate status. +- [Text](references/components/Text.md) - Renders text with the design system's typography tokens applied. + +### Disclosure + +- [Accordion](references/components/Accordion.md) - A vertically stacked group of sections the user can expand and collapse to show or hide their contents. +- [AnimatedChevron](references/components/AnimatedChevron.md) - A chevron icon that animates between two orientations to indicate the open or closed state of a disclosure. Use inside expand and collapse triggers when users need a visual cue for the current state. +- [Collapsible](references/components/Collapsible.md) - An unstyled disclosure that toggles a single block of content between expanded and collapsed states. Use to hide secondary detail behind a clickable summary so users can reveal more on demand. + +### Feedback + +- [Banner](references/components/Banner.md) - A prominent, full-width inline message embedded in the page layout for system-wide announcements, page-level alerts, or important context. +- [Notification](references/components/Notification.md) - A non-blocking overlay message that delivers detailed feedback in the user's peripheral vision, optionally with actions. Reach for it when the message needs more detail than a Toast or an action the user should take. +- [StatusMessage](references/components/StatusMessage.md) - A full-area centered message displayed when there is no content to show — empty states, error pages, permission denied, not found. Replaces the content area rather than overlaying it. +- [Toast](references/components/Toast.md) - A small, temporary overlay that gives the user non-critical feedback after an action. Reach for it when the message is informational and can be missed without consequence. +- [Toaster](references/components/Toaster.md) - Renders and manages the app's toast and notification stacks. Render one near the app root, then call `toast()` or `notification()` from anywhere. + +### Forms + +- [Checkbox](references/components/Checkbox.md) - A form control that lets users toggle a single option on or off. Supports a mixed state for representing partially-selected groups of related options. +- [Combobox](references/components/Combobox.md) - A text input paired with a filterable listbox for selecting one or more values from a list. Use when users need to search through a large set of options. +- [DatePicker](references/components/DatePicker.md) - An input field paired with a calendar popup for selecting a single date or a date range. Use when users need to enter or browse to a specific date. +- [FormField](references/components/FormField.md) - Groups a label, input, and helper text to keep form elements consistent and accessible. + - sometimes known as Form Group, Fieldset +- [InputShell](references/components/InputShell.md) - A low-level wrapper that styles any element as a KUI input. Reach for the higher-level inputs first; use this only when composing a custom control. +- [Label](references/components/Label.md) - A text label for a form control. Prefer FormField for end-user forms; this primitive is intended for composing custom field layouts. +- [NumberInput](references/components/NumberInput.md) - A TextInput preset to `type="number"` for numeric-only entry. +- [OtpInput](references/components/OtpInput.md) - A segmented field for entering a one-time passcode or PIN. One real input backs every cell, so browser autofill, paste, and native validation all keep working. +- [RadioGroup](references/components/RadioGroup.md) - A set of mutually exclusive options where the user picks exactly one. Use when the available options can comfortably fit on screen at once. +- [RangeSlider](references/components/RangeSlider.md) - A two-thumb slider for selecting a minimum and maximum value within a range. Use for filtering by price, time window, or any min/max selection. + - sometimes known as Range Input, Input Range +- [Rating](references/components/Rating.md) - A row of icons that shows a value within a range, or lets the user pick one. +- [Select](references/components/Select.md) - An input that lets the user pick one or more values from a predefined list. +- [Slider](references/components/Slider.md) - A single-value range input rendered along a bar. Use for continuous settings like volume, brightness, or threshold values where the relative position carries meaning. + - sometimes known as Range Input, Input Range +- [TextArea](references/components/TextArea.md) - A multi-line text input for free-form responses such as descriptions, comments, or notes. +- [TextInput](references/components/TextInput.md) - A single-line input for text, numbers, or special symbols. Commonly used in forms; switch to TextArea when the response is likely to wrap. +- [TimeInput](references/components/TimeInput.md) - A TextInput preset to `type="time"` for time-of-day entry, including the built-in native time-picker button. +- [Upload](references/components/Upload.md) - A file picker with drag-and-drop support that renders previews and basic management controls for the selected files. + +### Layout + +- [Block](references/components/Block.md) - A general-purpose block-level container for arbitrary content. Use it as a low-level layout primitive when consistent spacing and overflow behavior are needed without writing custom CSS. +- [Divider](references/components/Divider.md) - A horizontal or vertical line that visually separates groups of content. Use to create breaks between sections, list items, or toolbar regions where a heading or whitespace alone would be too subtle. +- [Flex](references/components/Flex.md) - A layout primitive that arranges its children with CSS flexbox using design-token spacing. +- [Grid](references/components/Grid.md) - A two-dimensional layout primitive that arranges children into rows and columns. Use for dashboards, card listings, and any layout that needs both horizontal and vertical alignment. +- [Group](references/components/Group.md) - A container that joins related items into a single visual unit. Reach for it when adjacent elements should read as one cohesive cluster, such as chips, badges, or button rows. +- [Inline](references/components/Inline.md) - An inline layout primitive that renders a `` with the system's spacing tokens exposed as props. +- [Resizable](references/components/Resizable.md) - A layout of panels separated by draggable edges, such as a sidebar beside main content or an editor above a terminal. +- [Stack](references/components/Stack.md) - A flex container that arranges its children in a column by default, with an optional divider between items. + +### Loading + +- [ProgressBar](references/components/ProgressBar.md) - Visual indicator for long-running operations like uploads, downloads, and other loading states. +- [Skeleton](references/components/Skeleton.md) - A placeholder block that mimics the shape of content while it loads. Match each skeleton's size and shape to the real content as closely as possible to keep layout shift minimal. +- [Spinner](references/components/Spinner.md) - An indicator for short waits with unknown progress. + +### Navigation + +- [AppBar](references/components/AppBar.md) - The top-level masthead for an application, anchoring branding, primary navigation, and global actions. Every application has exactly one. + - sometimes known as Masthead, Navbar, Navigation Bar +- [Breadcrumbs](references/components/Breadcrumbs.md) - A navigation trail that shows the user's location within the app hierarchy and links back to parent pages. Use concise labels for each level. +- [HorizontalNav](references/components/HorizontalNav.md) - A horizontal navigation bar that organizes links to top-level sections of an application or site. +- [Pagination](references/components/Pagination.md) - Navigation control that splits a long list of items across multiple pages. Pair with tables and grids that exceed a single page of content. +- [Stepper](references/components/Stepper.md) - A progress tracker for moving through a fixed, ordered set of steps. +- [Tabs](references/components/Tabs.md) - Organizes related content into sections that share the same context and switches between them in place. Use for managing settings, comparing data, or any group of mutually exclusive views the user moves between without leaving the page. +- [TreeNav](references/components/TreeNav.md) - A hierarchical navigation list that lets users move through links at different levels, similar to a directory tree +- [VerticalNav](references/components/VerticalNav.md) - A vertical navigation menu of single links and nested link groups, typically rendered in a sidebar. Use as section-level navigation paired with a HorizontalNav for app-wide chrome, or as the sole navigation for apps with deep hierarchies and few global destinations. + +### Overlays + +- [Modal](references/components/Modal.md) - A `` that interrupts the page to capture the user's attention until they act on or dismiss its content. Use for confirmations, focused tasks, and information the user must acknowledge before continuing. +- [Popover](references/components/Popover.md) - A floating panel that displays rich content, options, or actions anchored to a trigger. +- [SidePanel](references/components/SidePanel.md) - A `` panel that slides in from the left or right edge to show supplementary content without leaving the current view. +- [Tooltip](references/components/Tooltip.md) - Reveals a short, supplemental label when the user hovers or focuses an element. Use to clarify icon-only controls or surface non-essential context. + +### Surfaces + +- [Card](references/components/Card.md) - A container that groups content, actions, and optional media about a single subject. Cards are fluid and grow to fill their container. +- [Carousel](references/components/Carousel.md) - A horizontal container that cycles through a series of content items. +- [Halo](references/components/Halo.md) - A decorative rotating gradient ring that wraps a single element to draw attention to it. Halo is presentational only — it adds no semantics, so the element inside keeps its own role, label, and behavior. +- [Hero](references/components/Hero.md) - A large page banner that combines headline messaging and calls to action with imagery. +- [Panel](references/components/Panel.md) - Container that groups related content and actions with a background, border, and consistent padding. Use for general-purpose surfaces on dashboards and detail pages. +- [PageHeader](references/components/PageHeader.md) - Top-of-page banner that establishes context with a title, breadcrumbs, and page-level actions. + +### Utilities + +- [PortalProvider](references/components/PortalProvider.md) - Sets default portal behavior for descendant components that support portaling. Wrap the app (or a region) to make overlays portal to a shared target without passing `portal`/`portalContainer` to each one; descendants still override with their own props. +- [ThemeProvider](references/components/ThemeProvider.md) - Establishes a theme context for descendant components, controlling color theme and density. Nest providers to scope different settings to specific parts of the app. diff --git a/.agents/skills/kaizen-ui/assets/color-primitives-hex.md b/.agents/skills/kaizen-ui/assets/color-primitives-hex.md new file mode 100644 index 000000000..baac1af08 --- /dev/null +++ b/.agents/skills/kaizen-ui/assets/color-primitives-hex.md @@ -0,0 +1,141 @@ +# Color primitive hex values + +Exact hex values for every NVIDIA brand color primitive and translucent overlay. This reference is shared data — for **when** to use primitives vs. semantic tokens, see [references/design-tokens/COLOR_PRIMITIVES.md](../references/design-tokens/COLOR_PRIMITIVES.md). + +Reach for these values only when you need a specific hex (brand illustrations, glass/scrim effects on uncontrolled backgrounds). For ordinary UI work, use semantic tokens from [COLORS.md](../references/design-tokens/COLORS.md) instead — they theme automatically. + +## Brand color scales + +Each family ships an `050 → 950` scale (gray adds `000`, `025`, `950`, `975`, `1000`). Lower numbers are lighter; higher numbers are darker. + +```css +--color-black: #000000; +--color-white: #ffffff; +--color-brand: #76b900; +--color-red-050: #ffe9e9; +--color-red-100: #ffd7d7; +--color-red-200: #ffbbbb; +--color-red-300: #ff8181; +--color-red-400: #fe3f3f; +--color-red-500: #e52020; +--color-red-600: #c21e1e; +--color-red-700: #961515; +--color-red-800: #650b0b; +--color-red-900: #4b0404; +--color-red-950: #2d0100; +--color-yellow-050: #feeeb2; +--color-yellow-100: #fcde7b; +--color-yellow-200: #f9c500; +--color-yellow-300: #ef9100; +--color-yellow-400: #df6500; +--color-yellow-500: #d73d00; +--color-yellow-600: #b93100; +--color-yellow-700: #8d2600; +--color-yellow-800: #601600; +--color-yellow-900: #441000; +--color-yellow-950: #2d0b00; +--color-green-025: #dafb7d; +--color-green-050: #cfff40; +--color-green-100: #bff230; +--color-green-200: #a5de15; +--color-green-300: #76b900; +--color-green-400: #549a00; +--color-green-500: #3f8500; +--color-green-600: #327100; +--color-green-700: #265600; +--color-green-800: #193800; +--color-green-900: #142700; +--color-green-950: #0d1a00; +--color-teal-050: #adfcf8; +--color-teal-100: #9aefe5; +--color-teal-200: #3ae3c9; +--color-teal-300: #1dbba4; +--color-teal-400: #139a86; +--color-teal-500: #0d8473; +--color-teal-600: #097061; +--color-teal-700: #04554b; +--color-teal-800: #033831; +--color-teal-900: #022723; +--color-teal-950: #011a19; +--color-blue-050: #cbf5ff; +--color-blue-100: #aaccee; +--color-blue-200: #7cd7fe; +--color-blue-300: #10b1fb; +--color-blue-400: #008af9; +--color-blue-500: #0074df; +--color-blue-600: #0060c7; +--color-blue-700: #0046a4; +--color-blue-800: #002781; +--color-blue-900: #002050; +--color-blue-950: #00112c; +--color-purple-050: #fae9ff; +--color-purple-100: #f9d4ff; +--color-purple-200: #f0b9fd; +--color-purple-300: #cd8ef0; +--color-purple-400: #c359ef; +--color-purple-500: #a846db; +--color-purple-600: #952fc6; +--color-purple-700: #741d9d; +--color-purple-800: #4d1368; +--color-purple-900: #331344; +--color-purple-950: #1f0f27; +--color-fuchsia-050: #ffe8f9; +--color-fuchsia-100: #ffd3f2; +--color-fuchsia-200: #feb5ee; +--color-fuchsia-300: #fc79ca; +--color-fuchsia-400: #e050b7; +--color-fuchsia-500: #d2308e; +--color-fuchsia-600: #b62475; +--color-fuchsia-700: #8c1c55; +--color-fuchsia-800: #5d1337; +--color-fuchsia-900: #420d25; +--color-fuchsia-950: #2d0919; +--color-gray-000: #ffffff; +--color-gray-025: #f7f7f7; +--color-gray-050: #eeeeee; +--color-gray-100: #e0e0e0; +--color-gray-1000: #000000; +--color-gray-200: #cccccc; +--color-gray-300: #a7a7a7; +--color-gray-400: #898989; +--color-gray-500: #757575; +--color-gray-600: #636363; +--color-gray-700: #4b4b4b; +--color-gray-800: #313131; +--color-gray-900: #222222; +--color-gray-950: #161616; +--color-gray-975: #0c0c0c; +``` + +## Translucent overlays + +Black-on-X and white-on-X overlays for hover/pressed effects on imagery, glass-morphism, and scrim layers where transparency must survive uncontrolled backgrounds. + +```css +--color-translucent-black-000: #00000000; +--color-translucent-black-050: #0000000d; +--color-translucent-black-100: #0000001a; +--color-translucent-black-120: #0000001f; +--color-translucent-black-150: #00000026; +--color-translucent-black-200: #00000033; +--color-translucent-black-300: #0000004d; +--color-translucent-black-400: #00000066; +--color-translucent-black-500: #00000080; +--color-translucent-black-600: #00000099; +--color-translucent-black-700: #000000b2; +--color-translucent-black-800: #000000cc; +--color-translucent-black-900: #000000e5; +--color-translucent-white-000: #ffffff00; +--color-translucent-white-050: #ffffff0d; +--color-translucent-white-100: #ffffff1a; +--color-translucent-white-120: #ffffff1f; +--color-translucent-white-200: #ffffff33; +--color-translucent-white-250: #ffffff40; +--color-translucent-white-300: #ffffff4d; +--color-translucent-white-400: #ffffff66; +--color-translucent-white-500: #ffffff80; +--color-translucent-white-600: #ffffff99; +--color-translucent-white-700: #ffffffb2; +--color-translucent-white-800: #ffffffcc; +--color-translucent-white-900: #ffffffe5; +``` diff --git a/.agents/skills/kaizen-ui/assets/icon-catalog.md b/.agents/skills/kaizen-ui/assets/icon-catalog.md new file mode 100644 index 000000000..df3ecb546 --- /dev/null +++ b/.agents/skills/kaizen-ui/assets/icon-catalog.md @@ -0,0 +1,884 @@ +# Icon catalog + +Action→icon mapping and the full list of available icon names. This file is data — for whether/how to use icons (decision rules, variant selection, anti-patterns, icon-only safe set), see [references/ICONS.md](../references/ICONS.md). + +Use the same icon for the same conceptual action across the app. + +## Imports + +Icons are named PascalCase exports from `@nvidia/react-gui-icons/icons` and accept a `variant="fill"` prop for the solid version. Names that would start with a digit or shadow a JS global are prefixed `Icon` (`IconError`, `IconImage`, `Icon3DAxis`). + +```tsx +import { Add, Pencil, Trash, MagnifyingGlass } from "@nvidia/react-gui-icons/icons"; +``` + +The same icons are also available from `@nvidia/react-gui-icons` as `` — see [references/ICONS.md](../references/ICONS.md) Setup section. + +--- + +## CRUD and document actions + +| Action | Icon | Notes | +|---|---|---| +| Rename | `Rename` | | +| Archive | `Archive` | | +| Import | `Import` | | +| Export | `Export` | | +| Upload | `Upload` | | +| Download | `Download` | | + +## Navigation and wayfinding + +| Action | Icon | Notes | +|---|---|---| +| Back | `ArrowLeft` | | +| Forward | `ArrowRight` | | +| Navigate left/right | `ChevronLeft` / `ChevronRight` | Pagination, carousel, breadcrumb navigation | +| Home | `Home` | | +| Menu (hamburger) | `Menu` | Mobile/responsive menu trigger | + +## View and display + +| Action | Icon | Notes | +|---|---|---| +| Expand / Fullscreen | `Expand` | | +| Collapse / Exit fullscreen | `Collapse` | | +| Grid view | `LayoutGrid` | For SegmentedControl view toggles | +| List view | `LayoutList` | For SegmentedControl view toggles | +| Column view | `LayoutColumns` | | +| Show / Visible | `Eye` | | +| Hide / Hidden | `EyeOff` | | + +## Status and feedback + +| Action | Icon | Notes | +|---|---|---| +| Information | `InfoCircle` | Tooltips, info popovers, help | +| Help | `HelpCircle` | | +| Warning | `Warning` | Triangle alert | +| Error | `IconError` | | +| Success / Check | `Check` | Avoid `CheckCircle` in dense contexts — the enclosing circle adds visual weight at small sizes | +| Settings / Configure | `Cog` | | +| Notifications / Alerts | `Bell` | | + +## User and account + +| Action | Icon | Notes | +|---|---|---| +| User / Profile | `Profile` | Single user | +| User group | `ProfileGroup` | Multiple users | +| Add user | `AccountAdd` | | +| Remove user | `AccountRemove` | | +| Lock / Secure | `LockClosed` | | +| Unlock | `LockOpen` | | + +## Content and media + +| Action | Icon | Notes | +|---|---|---| +| Image | `IconImage` | | +| Document | `IconDocument` | | +| Folder (closed) | `FolderClosed` | | +| Folder (open) | `FolderOpen` | | +| Link | `Link` | | +| Unlink | `LinkBreak` | | +| Bookmark | `Bookmark` | Good candidate for solid/outline active state toggle | +| Star / Favorite | `Star` | Good candidate for solid/outline active state toggle | +| Pin | `Pin` | | + +--- + +## All available icons + +Reference list of every icon name the design system uses. These are design-system names, not the export list of any one package: with a different icon library, map each name to its closest equivalent. +For `NvidiaGuiIcon`, pass the kebab-case form of the name. + +### 3d + +- Cloth +- Cube +- Dolly +- Emissive +- Explode +- Icon3DAxis +- Instance +- LightRect +- LightRotate +- Material +- MoveGlobal +- Reference +- Rotate3D +- RotateGlobal +- Segment +- Shader +- ShaderBall +- Skeleton +- Teapot +- Teleport +- Texture + +### Av + +- AutoRecord +- AutoRecordOff +- Broadcast +- Camera +- CameraAdd +- CameraOff +- ClapBoard +- ClosedCaption +- DisplaySurround +- Eject +- Equalizer +- Faders +- FastForward +- FastReverse +- FileReel +- Film +- FilmAdd +- FilmCollection +- Forward10 +- Fps +- HighQuality +- Hud +- Loop +- LoopOff +- Microphone +- MicrophoneOff +- Next +- Notes +- Pause +- PictureInPicture +- Play +- Previous +- Record +- Replay +- ReplayOff +- Reverse10 +- Shuffle +- Soundwaves +- Speaker +- SpeakerHigh +- SpeakerLow +- SpeakerMedium +- SpeakerMute +- SpeakerOff +- StepForward +- StepReverse +- Stop +- Videocam +- VideocamOff +- WatchParty + +### Common + +- Add +- AddCircle +- Alarm +- Bell +- BellOff +- Calendar +- CalendarEvent +- Cancel +- Check +- CheckCircle +- CheckMultiCircle +- Clock +- Close +- CloseCircle +- Cog +- CogOff +- CopyDoc +- CopyGeneric +- Enter +- Exit +- Eye +- EyeOff +- Filter +- HelpCircle +- Home +- HomeOff +- IconClipboard +- IconError +- IconHistory +- Inbox +- InfoCircle +- Link +- LinkBreak +- LockClosed +- LockOpen +- MagnifyingGlass +- MagnifyingGlassMinus +- MagnifyingGlassPlus +- Menu +- MoreHoriz +- MoreVert +- Outbox +- Paperplane +- Redo +- Refresh +- Reset +- Retry +- Share +- Sort +- Star +- Subtract +- SubtractCircle +- Sync +- SyncOff +- Timer +- Trash +- Undo +- Warning + +### Communication + +- Cloud +- CloudOff +- Data +- Download +- Envelope +- EnvelopeOff +- Firewall +- FirewallOff +- Forward +- Isp +- IspOff +- NetworkSignal +- NetworkSignalOff +- Reply +- SyncWarning +- TransferHorizontal +- TransferVertical +- Upload +- Wifi +- WifiOff + +### Cursor + +- Arrow +- Crosshair +- HandClosed +- HandOpen +- HandPinch +- HandPoint +- Hourglass +- HourglassEnd +- HourglassMid +- HourglassStart + +### Editor + +- Anchor +- AnchorCenter +- AnchorE +- AnchorN +- AnchorNe +- AnchorNw +- AnchorS +- AnchorSe +- AnchorSw +- AnchorW +- Approximately +- ArrangeColumns +- ArrangeRows +- AtSymbol +- Backspace +- Bold +- BoundingBox +- Chart +- ChartBar +- ChartBubble +- ChartFlow +- ChartHierarchy +- ChartPerformance +- ChartPie +- ChartScatterplot +- ChartTree +- Collapse +- Commit +- Connection +- CraftKnife +- Divide +- Equal +- Expand +- ExportFromClipboard +- ExportToClipboard +- Fit +- Freehand +- Generate +- Grid +- GridOff +- HAlignCenter +- HAlignLeft +- HAlignRight +- Header +- Highlighter +- IconImage +- IconText +- Italic +- Iterate +- Keyframe +- Lasso +- Layers +- LayersOff +- LineSegment +- ListBullet +- ListCheckmark +- ListNumber +- Markdown +- Move +- NotEqual +- Numbers +- Paperclip +- Paragraph +- Pen +- Pencil +- Rename +- Rotate +- Rotate90Clockwise +- Rotate90Counter +- Ruler +- ScaleDown +- ScaleReset +- ScaleUp +- Scissors +- ScmBranch +- ScmCompare +- ScmFork +- ScmMerge +- ScmPull +- ScmPullClose +- ScmPullDraft +- Section +- SectionBottom +- SectionTop +- SelectBrush +- Signature +- Skip +- Sliders +- SortAscending +- SortDescending +- Sticker +- StickerImage +- StickerShape +- StickerText +- StickerZoom +- Strikethrough +- StrokeWidth +- Style +- Swatches +- TextAlignCenter +- TextAlignFull +- TextAlignLeft +- TextAlignRight +- TextFramed +- TrashDelete +- Typography +- Underline +- VAlignBottom +- VAlignCenter +- VAlignTop +- Wand +- Wrench + +### Files + +- Archive +- Db +- DocumentCheckmark +- DocumentNew +- DocumentPreview +- Export +- Floppy +- FolderClosed +- FolderOpen +- IconDocument +- Import +- SplitFile + +### Hardware + +- Battery0 +- Battery0Off +- Battery100 +- Battery25 +- Battery50 +- Battery75 +- BatteryBoost +- BatteryCharging +- Bluetooth +- BluetoothOff +- Calculator +- Coaxial +- Component +- Cpu +- Display +- DisplayCollection +- DisplayColumn +- DisplayOff +- DisplayRow +- Displayport +- DriveCloud +- DriveCloudOff +- DriveNetwork +- DriveNetworkOff +- DriveRemovable +- DriveRemovableOff +- DriveUsb +- DriveUsbOff +- Drone +- Dvi +- Ethernet +- EthernetPort +- Fan +- FanLoud +- FanQuiet +- FlightStick +- Gamepad +- GamepadOff +- GamepadStart +- Gpu +- GpuCard +- GpuCardMulti +- GpuCardOff +- GpuOff +- GpuSync +- Hdmi +- Headset +- Icon2BLeftClick +- IconKeyboard +- JoystickClassic +- Laptop +- Laser +- Lightning +- Mac +- MacOff +- MediaOptical +- MiniDisplayport +- Mosaic +- Mouse2B +- Mouse2BRightClick +- Mouse3B +- MouseScrollwheel +- MouseScrollwheelActive +- Multidrive +- Network +- NetworkConnection +- NetworkPcs +- NvidiaShield +- NvidiaShieldStand +- OutletA +- OutletB +- OutletC +- OutletE +- OutletG +- OutletH +- OutletI +- OutletJ +- OutletK +- OutletL +- PerformanceHigh +- PerformanceLow +- PerformanceMedium +- PhoneLaptop +- PlugReceptacle +- PlugUsb +- Plugin +- Power +- PowerSupplyUnit +- Ram +- Remote +- Robot +- RobotArm +- RobotDelivery +- RobotTorso +- RobotWheels +- SdCard +- Smartphone +- SmartphoneOff +- Tablet +- TabletOff +- Telephone +- TelephoneOff +- Television +- TelevisionRetro +- Usb +- Vga +- Vr +- Watch +- Webcam +- WebcamOff +- WebcamSettings +- Whispermode +- WirelessModem +- WirelessModemOff +- Workstation +- WorkstationOff +- WorkstationSystem +- WorkstationSystemOff + +### Image + +- Brush +- Bucket +- Camera360 +- CameraLinked +- CameraSuper +- Collection +- Crop +- Denoise +- Easel +- Eraser +- Eyedropper +- Group +- ImportImage +- MaskInside +- MaskOutside +- MirrorHorizontal +- MirrorVertical +- Outpainting +- Palette +- Scale +- ScaleRelative +- ScaleRelativeOff +- SelectEllipse +- SelectPolygon +- SelectRectangle +- SendImage +- Shapes +- SunHigh +- SunLow +- SunOffHigh +- Transparency + +### Maps + +- Airplane +- Airport +- Bank +- Bed +- Bicycle +- Briefcase +- Bush +- Business +- Castle +- City +- Datacenter +- Dirt +- Elevator +- Emergency +- Escalator +- Factory +- Farm +- Flag +- Flower +- Fog +- FuelCan +- FuelPump +- Government +- Grass +- Gravel +- Helicopter +- Hill +- IconLocation +- IconMap +- LocationOff +- MapOff +- Mountain +- Mud +- Park +- PhysicalExit +- River +- Rock +- Route +- Sailboat +- Sand +- Sea +- Ship +- Signpost +- Sky +- Snow +- Stair +- Stone +- Straw +- Suitcase +- Toolbox +- Tree +- Trees +- Water +- World +- WorldOff + +### Misc + +- Accessibility +- Atom +- Bandaid +- Barcode +- Beaker +- Blackboard +- Bone +- Book +- Bottle +- Braille +- Bug +- BugOff +- Bus +- Car +- Cards +- CheckmarkBadge +- Circle1Q +- Circle2Q +- Circle3Q +- CircleTick +- Code +- Compass +- Concierge +- Crown +- CubeStack +- DeliveryTruck +- Die1 +- Die2 +- Die3 +- Die4 +- Die5 +- Die6 +- Ekg +- Engine +- FacialRecognition +- Fingerprint +- Fire +- Firecracker +- Firework +- Food +- Footprint +- Fork +- Gift +- Graduate +- GraphNode +- GraphNodeConnect +- GraphNodeDelete +- Head +- Health +- HighKick +- Hit +- HotAirBalloon +- Hydrant +- IconFunction +- IconInfinity +- IconMath +- IconSensor +- Ime +- Key +- Knife +- Lamp +- Library +- LibraryGames +- Lifebuoy +- Lightbulb +- Lighthouse +- Magnet +- Man +- ManWoman +- Mapping +- Martini +- Meal +- Medicine +- Megaphone +- Microscope +- ModeComponent +- Money +- MoneyCoin +- MoneyPaper +- Moon +- MoonOff +- Mug +- MugBeer +- NeuralNetwork +- News +- Os +- Package +- Pawn +- Pawprint +- PaymentCard +- Pizza +- Placesetting +- Platformer +- Podium +- PrayerRoom +- Projector +- Pulse +- Puzzle +- Qrcode +- Radar +- Radioactive +- Rocket +- Running +- SawWave +- ScaleBalance +- School +- Secure +- Seed +- Shield +- ShieldOff +- ShipWheel +- ShoppingBag +- ShoppingBasket +- ShoppingCart +- ShoppingCartOpen +- SinWave +- Skull +- Soccer +- Sparkle +- Split +- Spoon +- SquareWave +- SteeringWheel +- SuitClub +- SuitDiamond +- SuitSpade +- Sword +- Tag +- TagLabel +- Telescope +- Thermometer +- Ticket +- TrafficCone +- Transform +- TriangleWave +- Trident +- Umbrella +- WeatherCloud +- WeatherFog +- WeatherLightning +- WeatherPartialCloudDay +- WeatherPartialCloudNight +- WeatherRain +- WeatherSnow +- WeatherWind +- Weight +- WeightBar +- Wheel +- Wheelchair +- WineGlass +- Woman +- Yoke + +### Shapes + +- ArrowDown +- ArrowDownLeft +- ArrowDownMax +- ArrowDownRight +- ArrowLeft +- ArrowLeftMax +- ArrowLeftRight +- ArrowRight +- ArrowRightMax +- ArrowUp +- ArrowUpDown +- ArrowUpLeft +- ArrowUpMax +- ArrowUpRight +- ChevronDoubleDown +- ChevronDoubleLeft +- ChevronDoubleRight +- ChevronDoubleUp +- ChevronDown +- ChevronDownLeftUpRight +- ChevronLeft +- ChevronLeftRight +- ChevronRight +- ChevronUp +- ChevronUpDown +- ChevronUpLeftDownRight +- Diamond +- DiamondOff +- Return +- ShapeCircle +- ShapeCircleOff +- ShapeHexagon +- ShapeOctagon +- ShapeSquare +- ShapeSquareOff +- ShapeTriangle + +### Social + +- AccountAdd +- AccountCheck +- AccountMore +- AccountRemove +- AddressBook +- AwardRibbon +- AwardTrophy +- Baby +- Bookmark +- BookmarkOff +- ChatMessage +- ChatMulti +- ChatNew +- ChatSingle +- Chatbot +- Family +- FeelingHappy +- FeelingNeutral +- FeelingSad +- FeelingVeryHappy +- FeelingVerySad +- Founders +- FoundersTier1 +- FoundersTier2 +- FoundersTier3 +- Heart +- HeartOff +- Identification +- License +- Pin +- PinOff +- Profile +- ProfileGroup +- ProfileOff +- RankingXp +- ServiceAccount +- ThumbDown +- ThumbUp + +### View + +- Apps +- CarouselHorizontal +- CarouselVertical +- DisplayShare +- Dual +- Feedback +- FitToPage +- Fullscreen +- FullscreenExit +- GalleryStrip +- GripArea +- GripCorner +- GripEdgeHorizontal +- GripEdgeVertical +- GripMini +- Icon3SquarePlus +- Icon4Square +- IconWindow +- LayoutColumns +- LayoutDetail +- LayoutGrid +- LayoutList +- LayoutRows +- OpenExternal +- PillBarStack +- RectangleGroup +- Render +- Segmap +- StackHorizontal +- WindowCode +- WindowGrid +- WindowStack +- WindowTerminal +- Workspace +- WorkspaceGroup diff --git a/.agents/skills/kaizen-ui/references/ICONS.md b/.agents/skills/kaizen-ui/references/ICONS.md new file mode 100644 index 000000000..dafda2828 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/ICONS.md @@ -0,0 +1,98 @@ +# Icons + +Decision rules for KUI iconography: whether to use an icon, which variant, and which icons are safe icon-only. + +## Setup + +We use [NVIDIA Icons](https://github.com/NVIDIA/icons#documentation) - specifically the React package. Refer to that packages README via the link or node_modules for setup documentation if you're configuring it for the first time or debugging. + +Icons come from `@nvidia/react-gui-icons/icons` as named PascalCase exports. + +Names that would start with a digit or shadow a JS global are prefixed `Icon` — `IconError`, `IconImage`, `IconDocument`, etc. + +```tsx +import { ChevronDown } from "@nvidia/react-gui-icons/icons"; + + // line/outline (default) + // solid; prefer at smaller sizes +``` + +Icons inherit `currentColor`, so set the color on an ancestor rather than on the icon. `size` is a CSS length string (`size="24px"`), defaulting to `1em`. + +## Decision 1: Use an icon at all? + +Icons are not free — they add visual density, scanning cost, and inconsistency risk if applied unevenly. + +| Pattern | Use when | Examples | +|---|---|---| +| **Label only** | The action is a verb-clear contextual choice. The label already communicates intent; an icon would be decorative or ambiguous. Default for review actions, form submits, and most secondary buttons. | Approve, Decline, Save, Cancel, Edit, Submit, Delete (use color, not icon, to signal danger) | +| **Icon + label** | The action belongs to a recognizable system class (navigation, search, refresh, add, export, sort, filter, upload). The icon clarifies the action's _category_, not its _target_. | Refresh, + Add cluster, Search, Export, Filter, Sort | +| **Icon only** | Tight UI surfaces where the icon is universally understood. | Close, kebab, expand/collapse, table-row delete in dense rows | + +**Default to "label only" when uncertain.** Ask whether the icon is doing real work, not decorating. Adding icons indiscriminately makes UI feel busy without improving comprehension; removing icons that genuinely aid recognition (refresh, search, add) makes scanning slower. + +--- + +## Decision 2: Which icon? + +Cheatsheet of most common scenarios: + +| Action | Icon | +|---|---| +| Create / Add | `Add` (plus sign) | +| Edit | `Pencil` | +| Delete / Remove | `Trash`/`TrashDelete` | +| Save / Download | `Download` | +| Copy / Duplicate | `CopyGeneric`/`CopyDoc`(for document specifically) | +| Close / Dismiss | `Close` | +| Search | `MagnifyingGlass` | +| External link | `OpenExternal` | +| Filter | `Filter` | +| Sort | `Sort`(no arrow), `SortAscending`, `SortDescending`| +| Refresh / Reload | `Refresh` | +| Settings | `Cog` | +| More actions (kebab) | `MoreVert` | +| More actions (horizontal) | `MoreHoriz` | + +If none of those fit your use case see [`../assets/icon-catalog.md`](../assets/icon-catalog.md) for the full catalog grouped. + +**Toolbars and PageHeader actions:** when a button needs an icon render it leading + +--- + +## Decision 3: Solid vs. outline variant + +Icons accept a `variant` prop defaulting to `variant="line"` (outline). `variant="fill"` is solid. + +Use solid icons for navigation, small icon sizes for improved readability, or primary/prominent actions - this provides stronger visual weight and easier to recognize at smaller sizes. + +--- + +## Icon-only safe set + +These icons are universally recognized enough to omit a visible text label. **All icon-only usage must be coupled with a11y considerations such as a `aria-label` on the wrapping element.** + +| Icon | Action | +|---|---| +| `Close` | Dismiss / close | +| `Pencil` | Edit | +| `Trash` | Delete | +| `Cog` | Settings | +| `MoreVert` / `MoreHoriz` | Overflow menu | +| `MagnifyingGlass` | Search | +| `ChevronDown` / `ChevronUp` | Expand / collapse | +| `ChevronLeft` / `ChevronRight` | Previous / next | +| `Add` | Create / add | + +Every other icon should be paired with a text label. + +--- + +## Gotchas + +- **Consistency within a group is non-negotiable.** Within a single toolbar, button group, badge row, or menu, every item either has an icon or none does. Mixing is the most common iconography mistake and reads as visual sloppiness. +- **Don't mix solid and outline within the same visual group** (toolbar, menu, tab set, nav). Mixed variants imply a semantic difference where none exists. +- **Icon-only buttons must set `aria-label`.** Screen readers otherwise announce "button" with no context. +- **Don't force an icon when no recognizable metaphor exists.** A wrong metaphor teaches users an incorrect mental model — worse than no icon. Use a text-only label. +- **Don't add an icon to every badge, tag, or table cell.** Icons in every cell become visual noise and lose the emphasis that selective use provides. +- **Prefer plain `Check` over `CheckCircle` in dense contexts** (table cells, small badges). The enclosing circle adds visual weight and reduces readability at small sizes. diff --git a/.agents/skills/kaizen-ui/references/components/Accordion.md b/.agents/skills/kaizen-ui/references/components/Accordion.md new file mode 100644 index 000000000..b2e214440 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Accordion.md @@ -0,0 +1,131 @@ +# Accordion + +A vertically stacked group of sections the user can expand and collapse to show or hide their contents. + +## Notes + +- Prefer default single-open (i.e. not `multiple={true}`). Switch to `multiple` only when users genuinely need to compare or cross-reference content across sections at the same time. + +- Use `defaultValue` to open the most important or most-used section on first render. If no section is clearly more important, leave them all collapsed so the user can scan the trigger labels first. + +- Keep trigger labels concise — users skim them to choose a section. Detailed content belongs inside `slotContent`. + +- Avoid nesting Accordions. If a section needs its own expand/collapse behavior, the information architecture is likely too deep — flatten it instead. + +- If switching between (not collapsing) panels consider Tabs — Tabs show one panel at a time, Accordion can show multiple simultaneously. + +- For an unstyled collapsible section to build your own styles on top of, use Collapsible. + +- Avoid `multiple={true}` with long section content — it can create excessive page length; reconsider whether Accordion is the right pattern or whether the content should be on separate pages/tabs. + +- If triggers need icons, place them consistently on the same side across all items. + +## Examples + +### Basic Accordion + +By default the accordion allows for a single item to be open at a time. Set `multiple` to allow multiple items to be open at a time. + +```tsx + +``` + +### Default Open Multiple Items Accordion + +```tsx + +``` + +### With Icon In Trigger + +Use gap="inherit" to use the Accordion defined gap for the icon and text. + +```tsx + + Item 1 + + ), + slotContent: "...", + value: "1", + }, + { + slotTrigger: ( + + Item 2 + + ), + slotContent: "...", + value: "2", + }, + ]} +/> +``` + +### Composed + +```tsx + + + Section 1 + Content for section 1 + + + + + + Section 2 + + + Content for section 2 + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| **items** * | `{ slotTrigger: ReactNode; slotContent: ReactNode; value: string; disabled?: boolean; chevronPosition?: "start" \| "end" }[]` | - | Section definitions rendered in order as the accordion's collapsible panels. | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| collapsible | `boolean` | `true` | Allows the open item to be clicked to collapse itself; when `false`, the open item stays open until another item opens. Ignored when `multiple` is `true`. | +| defaultValue | `string \| string[]` | - | Item value to open initially when state is uncontrolled. | +| disabled | `boolean` | `false` | Disables the entire accordion and prevents all interactions. | +| multiple | `boolean` | `false` | Allows more than one item to be open at the same time. | +| name | `string` | - | Groups the underlying `
    ` elements under a shared name to enforce single-open behavior. Ignored when `multiple` is `true`. | +| onValueChange | `(value: string) => void \| (value: string[]) => void` | - | Called when the open item changes, with the new value or an empty string when all items close. | +| value | `string \| string[]` | - | Controlled value of the currently open item. Pair with `onValueChange` to manage state externally. | + +`* = required prop` + +## Exports + +- Components: `Accordion`, `AccordionContent`, `AccordionItem`, `AccordionRoot`, `AccordionTrigger` +- Hooks: `useAccordionContext`, `useAccordionItemContext` +- Types: `AccordionContentProps`, `AccordionItemProps`, `AccordionProps`, `AccordionRootProps`, `AccordionRootPropsBase`, `AccordionTriggerProps` + +## See also + +- [Collapsible](./Collapsible.md) +- [Tabs](./Tabs.md) diff --git a/.agents/skills/kaizen-ui/references/components/Anchor.md b/.agents/skills/kaizen-ui/references/components/Anchor.md new file mode 100644 index 000000000..abec8e70a --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Anchor.md @@ -0,0 +1,96 @@ +# Anchor + +Also known as: Link + +Interactive text that navigates the user to another page, section, or resource. + +## Notes + +- Use Anchor for navigation (URL or route). For actions that do not navigate (submit, toggle, open a modal), use Button instead. + +- To open a link in a new tab, use `external` rather than setting `target="_blank"` and `rel` by hand — it also adds the external-link icon and the accessible suffix. + +## Examples + +### Basic Anchor + +```tsx + + + Inline + + + Standalone + + +``` + +### Custom Link Component + +Use asChild to render framework specific link components with our styling + +```tsx + + Home + +``` + +### External Link Anchor + +Use `external` to open the link in a new tab, secure it with `rel="noopener noreferrer"`, and append an external-link icon plus an accessible "(opens in a new tab)" suffix, all in one prop. + +```tsx + + Open documentation + +``` + +### Disabled Anchor + +A disabled anchor will render as a span instead of an anchor tag + +```tsx + + Disabled Link + +``` + +### As Button + +For an Anchor that looks like a link but triggers an action + +```tsx + + + +``` + +## Props + +Typography values use the [typography scale](../design-tokens/TYPOGRAPHY.md). + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| disabled | `boolean` | `false` | Renders the anchor as a non-interactive `` so it no longer navigates. | +| external | `boolean` | `false` | Opens the link in a new tab, secures it with `rel`, and appends an external-link icon and a screen-reader-only "(opens in a new tab)" suffix. An explicit `target` or `rel` still wins, and the suffix is dropped if that target is not `_blank`. Ignored when `disabled`. | +| fontFamily | `"sans" \| "mono"` | - | Overrides the font family inherited from `kind`. | +| fontSize | `"10" \| "12" \| "14" \| "16" \| "18" \| "20" \| "22" \| "24" \| "28" \| "32" \| "36" \| "40" \| "44" \| "48" \| "50" \| "56" \| "60" \| "64" \| "72" \| "80"` | - | Overrides the font size inherited from `kind`, in pixels. | +| fontStyle | `"normal" \| "italic"` | - | Sets the font style. | +| fontWeight | `"bold" \| "light" \| "regular" \| "semibold"` | - | Overrides the font weight inherited from `kind`. | +| kind | `"inline" \| "standalone"` | `"inline"` | The kind of anchor. - "inline" - Embedded within prose; renders with an underline so the link remains distinguishable inside body text. - "standalone" - Rendered outside of prose, such as a navigation link or call to action; renders without an underline. | +| lineHeight | `"100" \| "125" \| "150" \| "175"` | - | Overrides the line height inherited from `kind`, as a percentage of the font size. | +| textKind | `TypographyKindUnion` | `"body/regular/md"` | Typography token applied to the link text. | +| underline | `boolean` | - | Applies an underline to the text. | + +## Exports + +- Components: `Anchor` +- Types: `AnchorProps` + +## See also + +- [Button](./Button.md) +- [Breadcrumbs](./Breadcrumbs.md) diff --git a/.agents/skills/kaizen-ui/references/components/AnimatedChevron.md b/.agents/skills/kaizen-ui/references/components/AnimatedChevron.md new file mode 100644 index 000000000..8dec81b99 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/AnimatedChevron.md @@ -0,0 +1,66 @@ +# AnimatedChevron + +A chevron icon that animates between two orientations to indicate the open or closed state of a disclosure. Use inside expand and collapse triggers when users need a visual cue for the current state. + +## Notes + +- Place AnimatedChevron inside the trigger it describes. Without `state` it follows `data-state="open"` or `data-state="closed"` on an ancestor, which Collapsible and Popover triggers already set, or the open state of the `
    ` whose `` contains it. + +- Pass `state` only when nothing above the chevron sets `data-state`, such as a button that toggles a panel from your own state. `state` takes precedence over any ancestor state. + +- The chevron is hidden from assistive technology, so a trigger you build yourself must expose its state with `aria-expanded` and point `aria-controls` at the content it toggles. + +- Do not add one to Accordion, Select, Combobox, Dropdown, or DatePicker triggers, which render their own, or to a Collapsible trigger that already sets `showChevron`. + +- For an arrow that never changes direction, use a static chevron icon instead. + +## Examples + +### Basic Usage + +Reads open/closed state from `data-state="open"` or `data-state="closed"` from parent. + +```tsx +
    + +
    +``` + +### Controlled Usage + +Manually control the state with the `state` prop. + +```tsx + +``` + +### Usage With Collapsible + +When used in most of our components, it automatically reads the open/closed state from the parent data attribute, such as Collapsible. + +```tsx + + Toggle content + + } +> + ... + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| state | `"open" \| "closed"` | - | Forces the chevron orientation when the parent element does not set a `data-state` attribute. | + +## Exports + +- Components: `AnimatedChevron` +- Types: `AnimatedChevronProps` + +## See also + +- [Collapsible](./Collapsible.md) diff --git a/.agents/skills/kaizen-ui/references/components/AppBar.md b/.agents/skills/kaizen-ui/references/components/AppBar.md new file mode 100644 index 000000000..9520f4a35 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/AppBar.md @@ -0,0 +1,102 @@ +# AppBar + +Also known as: Masthead, Navbar, Navigation Bar + +The top-level masthead for an application, anchoring branding, primary navigation, and global actions. Every application has exactly one. + +## Notes + +- Render exactly one AppBar per application. + +- Keep `slotEnd` to two or three global actions; group overflow inside a Dropdown rather than adding more icons. + +- Place version or product switchers in `slotStart` next to the logo as a Dropdown whose trigger shows the current selection. + +- `slotStart` and `slotEnd` already render as `flex items-center` with the AppBar's `gap` inherited; `slotStart` also applies the masthead label style. Pass children directly as a Fragment (`<>...`) — don't wrap them in your own `flex`/`gap` container, and don't re-style the product label with a styled ``; plain span or Anchor with textKind="inherit" inherits the correct style. + +- Implement AppBar as a sticky top masthead (e.g. `sticky top-0` plus a stacking `z-index`) so brand, nav, and global actions stay visible while the body scrolls. + +- When a sticky VerticalNav sidebar is used alongside AppBar, set its `top-*` offset to match the AppBar block height (via measured height, design token(--nv-app-bar-height), or `calc`) so the sidebar fills the viewport below the masthead instead of sliding under it. + +- Anti-patterns: non-sticky AppBar on long pages, navigation built outside the AppBar (custom header), too many actions in the trailing slot + +## Examples + +### Simple Example + +```tsx + + + + Product + + + } +/> +``` + +### Full Example + +A full example of the AppBar component with a start and end slot. Note the ACME is a placeholder for a brand SVG or logo. + +```tsx + + console.log("toggle")} /> + ACME + + Product Name + + + } + slotEnd={} +> + + +``` + +### With Centered Content + +```tsx + +
    + +
    +     +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| children | `ReactNode` | - | Center-slot content, left-aligned by default and typically used for primary navigation. | +| slotEnd | `ReactNode` | - | Trailing-edge content, typically global actions such as a user menu or settings. | +| slotStart | `ReactNode` | - | Leading-edge content, typically brand area and product name. | + +## Exports + +- Components: `AppBar`, `AppBarExpanderButton` +- Constants: `AppBarHeight`, `AppBarHeightVariable` +- Types: `AppBarProps` + +## See also + +- [HorizontalNav](./HorizontalNav.md) +- [VerticalNav](./VerticalNav.md) diff --git a/.agents/skills/kaizen-ui/references/components/Avatar.md b/.agents/skills/kaizen-ui/references/components/Avatar.md new file mode 100644 index 000000000..540f5625b --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Avatar.md @@ -0,0 +1,101 @@ +# Avatar + +A small visual identifier for a user, account, or entity. Prioritizes an image when available and falls back to initials or an icon otherwise. + +## Notes + +- Always provide a meaningful `fallback` (typically the user's initials) — never leave it empty, since the image may fail to load. + +- Provide descriptive `alt` text whenever an `src` is set. Use `alt=""` only when the user's name is already displayed next to the avatar to avoid screen-reader redundancy. + +- Set `interactive` only when clicking the avatar actually navigates or triggers an action. The hover state otherwise implies an affordance that does not exist. + +- Pick `size` to match the context's density: `"small"` for lists and tables, `"medium"` for standard placements, `"xlarge"` or `"xxlarge"` for profile or hero pages. + +## Examples + +### Basic Avatar + +Always set `fallback` (typically initials) so the avatar still renders if the image fails to load. Pass descriptive `alt` text identifying the user or entity. + +```tsx + +``` + +### With Name + +Set `alt=""` when the user's name is already shown next to the avatar — screen readers would otherwise announce the name twice. + +```tsx +
    + + Alex Morgan +
    +``` + +### Interactive Avatar + +Sometimes Avatars are used as triggers - in that case use the `interactive` prop so the Avatar has hover states. + +```tsx + +``` + +### Icon Avatar + +To render an Avatar with an icon, use the fallback prop with an icon component. + +```tsx +} /> +``` + +### As Link Or Button + +To render an Avatar as a link or button, use the asChild prop on AvatarRoot to render it as its child element. + +```tsx + + + + AM + + +``` + +### Composed + +```tsx + + + AM + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| **fallback** * | `ReactNode` | - | Content rendered when no image source is provided or the image fails to load; typically the user's initials or an icon. | +| alt | `string` | - | Accessible label for the image; prefer the user's full name. Set to an empty string when the name is already displayed nearby to avoid screen reader redundancy. | +| attributes | `{ AvatarImage?: NativeElementAttributes<"img", typeof AvatarImage>; AvatarFallback?: NativeElementAttributes<"div", typeof AvatarFallback> }` | - | Native HTML attributes forwarded to the internal composed components. | +| interactive | `boolean` | - | Enables hover, active, and pointer styling on the avatar. Defaults to `true` when `onClick` is provided, `false` otherwise. | +| kind | `"outline" \| "solid"` | `"outline"` | The visual treatment of the avatar. - "outline" - Adds a border ring; preferred for dense layouts. - "solid" - No outline. | +| size | `"small" \| "medium" \| "large" \| "xlarge" \| "xxlarge"` | `"medium"` | The size of the avatar. - "small" - Lists, tables, and other dense rows. - "medium" - General-purpose default for most layouts. - "large" - Profile detail pages and prominent placements. - "xlarge" - Featured profile or settings headers. - "xxlarge" - Hero or banner placements. | +| src | `string` | - | URL of the image to display. | + +`* = required prop` + +## Exports + +- Components: `Avatar`, `AvatarFallback`, `AvatarImage`, `AvatarRoot` +- Types: `AvatarFallbackProps`, `AvatarImageProps`, `AvatarProps`, `AvatarRootProps` diff --git a/.agents/skills/kaizen-ui/references/components/Badge.md b/.agents/skills/kaizen-ui/references/components/Badge.md new file mode 100644 index 000000000..3034aaee3 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Badge.md @@ -0,0 +1,131 @@ +# Badge + +A small color-coded label that displays status, category, or metadata at a glance. Badges are +non-interactive and convey information rather than trigger actions. + +## Notes + +- Always use Badge (not plain text) to display entity status. Pick a semantic color (e.g. color="green" for healthy, color="yellow" for degraded). + +- Default to kind="solid" almost everywhere — the solid treatment carries the status signal at a glance. Reach for kind="outline" only in deliberate de-emphasized contexts + +- Pair status badges with an icon that reinforces the semantics (check for healthy, warning triangle for degraded, etc.). The icon aids color-blind users and lets the meaning survive at a glance even before the label is read. + +- Use Tag (not Badge) for interactive labels (selectable, dismissible, clickable) and for taxonomic categories (topic tags, tech stack, team name, region). + +- Use StatusIndicator (not Badge) when the status needs only a colored dot without a text label. + +- Establish a single project-wide color-to-status mapping (e.g. green = healthy, red = error, yellow = warning) and use it consistently — do not reuse the same color for different meanings on the same page. + +- Placement: place status badges near the title (top-right of a card, inline with a detail-page title) and never inside a tag row (tags read as interactive, badges do not). If a card has no status, omit the Badge rather than inserting a placeholder for grid symmetry. + +- Limit to 2-3 badges per entity; excess badges become visual noise users ignore. + +- In dense table cells, prefer plain text or `Anchor` over Badge/Tag for taxonomic labels. + +- Prefer no icons - status badges can use color and text to convey meaning. Reserve icons for badges that need to scan at very small sizes, or to disambiguate two states with the same color. + +- Avoid rainbow-coding - use a single color for each status. + +- Set `kind="solid"` for badges in detail views, side panels, and status bars - it provides clear, immediate visual hierarchy for status. `kind="outline"` is visually distracting when it appears alongside other bordered elements(panels, inputs, dividers). + +- Prefer `kind="outline"`(default) when the badge is **secondary** information that should not compete for attention - eg card metadata, labels in a dense list where multiple badges appear per row + +## Examples + +### Basic Badge + +Pick `kind="solid"` for status badges in detail views, side panels, and status bars. Use `kind="outline"` only for de-emphasized contexts like card metadata — outline badges visually resemble Tags and Buttons, creating a false interactive affordance. + +```tsx + + + Badge Text + + Badge Text + + Badge Text + + Badge Text + + Badge Text + + Badge Text + + Badge Text + + Badge Text + +``` + +### Status Badge + +Map `color` to a single project-wide status meaning (green = healthy, red = error, yellow = warning, blue = informational, gray = inactive) and reuse it across the app. Do not assign the same color to different meanings on the same page. + +```tsx + + + Active + + + Failed + + + Warning + + + In Progress + + + Inactive + + +``` + +### With Icon + +Pair an icon with the label when the status benefits from redundant encoding (especially for color-blind accessibility). Omit icons for category badges where color alone is enough — extra glyphs add visual noise in dense layouts. + +```tsx + + Encrypted + +``` + +### As Link + +```tsx + + Badge as a Link + +``` + +### With Truncation + +Badge text should be short and concise. By default, overflowing content wraps so it stays readable. If you must truncate (e.g., a constrained layout with unpredictable content), wrap the text in a span with truncation styles. + +```tsx + + + Long text that will be truncated + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| color | `"blue" \| "green" \| "red" \| "yellow" \| "purple" \| "teal" \| "gray"` | `"blue"` | Semantic color used to convey meaning at a glance. | +| kind | `"outline" \| "solid"` | `"outline"` | Visual treatment of the badge. - "outline" - De-emphasized contexts such as card metadata or secondary information. - "solid" - Prominent status badges in detail views, side panels, and status bars. | +| size | `"small" \| "medium" \| "large"` | `"medium"` | Controls the badge's padding and text size. Reach for `small` in very dense layouts, and `large` for prominent status in detail views. | + +## Exports + +- Components: `Badge` +- Types: `BadgeProps` + +## See also + +- [Tag](./Tag.md) +- [StatusIndicator](./StatusIndicator.md) diff --git a/.agents/skills/kaizen-ui/references/components/Banner.md b/.agents/skills/kaizen-ui/references/components/Banner.md new file mode 100644 index 000000000..f36912cb4 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Banner.md @@ -0,0 +1,149 @@ +# Banner + +A prominent, full-width inline message embedded in the page layout for system-wide announcements, page-level alerts, or important context. + +## Notes + +- Pick `kind` by message scope: `"global"` for app-wide announcements above the AppBar, `"header"` for page-level header alerts below the AppBar, `"inline"` for contextual alerts inside a content area. + +- Banner spans full width and dominates the layout — use Notification (not Banner) for contextual floating messages related to an action. + +- Always set `status` to match the message: `"error"` for outages or failures, `"warning"` for upcoming issues, `"success"` for resolved items, `"info"` for neutral announcements. + +- Omit `onClose` for banners that must remain visible until resolved (active outages, compliance notices, security warnings). Provide `onClose` only when the user can safely dismiss the message. + +- Choose the messaging component by attention and presence: Toast for low-attention transient confirmations, Notification for stacked persistent updates, Banner for page- or app-wide inline alerts, Modal for blocking high-attention decisions. + +- Anti-patterns: using `kind="global"` for page-specific info; stacking multiple banners (causes alert fatigue — users stop reading); making active-outage banners dismissible. + +- Dismissibility: informational announcements, feature promotions, and resolved status updates are safe to dismiss; active outages, compliance notices, and unresolved errors should omit `onClose`. + +## Examples + +### Inline Banner + +Use `kind="inline"` (default) for contextual alerts within a specific content area. Lighter weight than `header` and the right choice for messages scoped to a single section. + +```tsx + + Lorem ipsum dolor sit + Lorem ipsum dolor sit + Lorem ipsum dolor sit + +``` + +### Global Banner + +Use `kind="global"` for app-wide announcements that render at the top of the app, above the AppBar — scheduled maintenance, account expiration, platform-level alerts that affect all pages. + +```tsx + + Your account password is expiring soon + +``` + +### Header Banner With Actions + +Use `kind="header"` for page- or section-level announcements that render below the AppBar. Header banners support heading + subheading + actions for context the current view needs. + +```tsx + + + + + } +> + Service maintenance scheduled + +``` + +### Composed + +```tsx + + + + + + + + Service maintenance scheduled + + + + + + + + + + +``` + +### Composed Header Example + +```tsx + + + + + + + + Maintenance complete + Our system is back online + + + + + + + + + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| **kind** * | `"header" \| "inline" \| "global"` | `"inline"` | The kind of banner based on the scope of the message. - "inline" - Contextual alerts placed inline within a specific content area. - "header" - Page-level header banner with a bolded heading and optional subheading. Renders full width below the AppBar. - "global" - Site-wide announcements rendered full width at the top of the app, above the AppBar. | +| **slotSubheading** * | `ReactNode` | - | Subheading text rendered below the heading. Only applies to `kind="header"` banners. | +| actionsPosition | `"right" \| "bottom"` | - | Forces the actions to a specific position. By default, actions render on the right and move to the bottom when the banner is too narrow. - "right" - Actions stay to the right of the content. - "bottom" - Actions stack below the content. | +| attributes | `{ BannerContent?: NativeElementAttributes<"div", typeof BannerContent>; BannerHeader?: NativeElementAttributes<"div", typeof BannerHeader>; BannerHeading?: NativeElementAttributes<"div", typeof BannerHeading>; BannerSubheading?: NativeElementAttributes<"div", typeof BannerSubheading>; BannerIcon?: NativeElementAttributes<"div", typeof BannerIcon>; BannerCloseButton?: NativeElementAttributes<"button", typeof Button>; BannerLayout?: NativeElementAttributes<"div", typeof BannerLayout>; BannerActionsSection?: NativeElementAttributes<"div", typeof BannerActionsSection>; BannerCloseButtonSection?: NativeElementAttributes<"div", typeof BannerCloseButtonSection> }` | - | Native HTML attributes forwarded to the internal composed components. | +| children | `ReactNode` | - | Heading text rendered at the top of the banner. | +| onClose | `() => void` | - | Dismiss handler. When provided, a close button is rendered; omit for blocking banners (e.g. active outages, compliance notices) that must remain visible until resolved. | +| ref | `Ref` | - | | +| slotActions | `ReactNode` | - | Action buttons rendered next to or below the banner content. | +| slotIcon | `ReactNode` | `status === "info" ? ( ) : status === "warning" ? ( ) : status === "error" ? ( ) : status === "success" ? ( ) : null` | Icon rendered alongside the heading. Defaults to a status-appropriate icon; pass `null` to render no icon. | +| status | `"error" \| "warning" \| "success" \| "info"` | `"info"` | The semantic status of the banner. - "info" - Neutral information (blue). - "warning" - Potentially unwanted outcomes or upcoming issues (yellow). - "error" - A problem that needs attention, such as outages or failures (red). - "success" - A successful action or resolved issue (green). | + +`* = required prop` + +## Exports + +- Components: `Banner`, `BannerActionsSection`, `BannerCloseButtonSection`, `BannerContent`, `BannerHeader`, `BannerHeading`, `BannerIcon`, `BannerLayout`, `BannerRoot`, `BannerSubheading` +- Types: `BannerActionsSectionProps`, `BannerCloseButtonSectionProps`, `BannerContentProps`, `BannerHeaderProps`, `BannerHeadingProps`, `BannerIconProps`, `BannerLayoutProps`, `BannerProps`, `BannerRootProps`, `BannerSubheadingProps` + +## See also + +- [Notification](./Notification.md) +- [Toast](./Toast.md) diff --git a/.agents/skills/kaizen-ui/references/components/Block.md b/.agents/skills/kaizen-ui/references/components/Block.md new file mode 100644 index 000000000..fc86fed24 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Block.md @@ -0,0 +1,58 @@ +# Block + +A general-purpose block-level container for arbitrary content. Use it as a low-level layout primitive when consistent spacing and overflow behavior are needed without writing custom CSS. + +## Notes + +- Use Block for one region that needs padding, overflow control, or text truncation. Arrange several children with Stack, Flex, or Grid instead. + +- Use `truncate` to clip a single line of text with an ellipsis. `textOverflow` on its own does nothing: it also needs `overflow` set to something other than `"visible"` and `textWrap="nowrap"`, which is what `truncate` sets. + +- Prefer density tokens such as `padding="density-lg"` over fixed steps such as `"4"`, so spacing follows the density theme. + +- Use `asChild` to apply Block's padding and overflow to a semantic element such as `
    ` or `
    ` instead of adding a wrapper `
    `. + +## Examples + +### Basic Block + +```tsx + +``` + +## Props + +Spacing values use the [spacing scale](../design-tokens/SPACING_SCALE.md). + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| gap | `SpacingScaleUnion` | - | Sets spacing between flex and grid items. | +| overflow | `"hidden" \| "auto" \| "clip" \| "scroll" \| "visible"` | - | Controls how content that overflows the box is handled on both axes. See https://developer.mozilla.org/en-US/docs/Web/CSS/overflow for more. - "auto" - Adds scrollbars only when content overflows. - "clip" - Clips overflow with no scrolling affordance. - "hidden" - Clips overflow and disables scrolling. - "scroll" - Always shows scrollbars. - "visible" - Renders overflow outside the box. | +| overflowX | `"hidden" \| "auto" \| "clip" \| "scroll" \| "visible"` | - | Controls how content that overflows the box is handled on the horizontal axis. See https://developer.mozilla.org/en-US/docs/Web/CSS/overflow-x for more. - "auto" - Adds a horizontal scrollbar only when content overflows. - "clip" - Clips horizontal overflow with no scrolling affordance. - "hidden" - Clips horizontal overflow and disables horizontal scrolling. - "scroll" - Always shows a horizontal scrollbar. - "visible" - Renders horizontal overflow outside the box. | +| overflowY | `"hidden" \| "auto" \| "clip" \| "scroll" \| "visible"` | - | Controls how content that overflows the box is handled on the vertical axis. See https://developer.mozilla.org/en-US/docs/Web/CSS/overflow-y for more. - "auto" - Adds a vertical scrollbar only when content overflows. - "clip" - Clips vertical overflow with no scrolling affordance. - "hidden" - Clips vertical overflow and disables vertical scrolling. - "scroll" - Always shows a vertical scrollbar. - "visible" - Renders vertical overflow outside the box. | +| padding | `SpacingScaleUnion` | - | Sets padding. | +| paddingBottom | `SpacingScaleUnion` | - | Sets bottom padding. | +| paddingLeft | `SpacingScaleUnion` | - | Sets left padding. | +| paddingRight | `SpacingScaleUnion` | - | Sets right padding. | +| paddingTop | `SpacingScaleUnion` | - | Sets top padding. | +| paddingX | `SpacingScaleUnion` | - | Sets horizontal padding. | +| paddingY | `SpacingScaleUnion` | - | Sets vertical padding. | +| textOverflow | `"clip" \| "ellipsis"` | - | Controls how inline text is signaled when it overflows the box. Requires `overflow` to be set to something other than `visible`. See https://developer.mozilla.org/en-US/docs/Web/CSS/text-overflow for more. - "ellipsis" - Renders an ellipsis (`…`) to indicate clipped text. - "clip" - Truncates the text at the edge of the content area. | +| textWrap | `"wrap" \| "nowrap" \| "balance" \| "pretty"` | - | Controls how text wraps across lines inside the block. See https://developer.mozilla.org/en-US/docs/Web/CSS/text-wrap for more. - "wrap" - Wraps text across as many lines as needed to fit the container. - "nowrap" - Keeps text on a single line; pair with `overflow` and `textOverflow` to clip. - "balance" - Distributes text evenly across lines, suited to short headings. - "pretty" - Optimizes line breaks for readability, suited to body copy. | +| truncate | `boolean` | - | Clips inline text to a single line and appends an ellipsis. Shorthand for `textWrap: "nowrap"`, `overflow: "hidden"`, and `textOverflow: "ellipsis"`. | + +## Exports + +- Components: `Block` +- Types: `BlockProps` + +## See also + +- [Stack](./Stack.md) diff --git a/.agents/skills/kaizen-ui/references/components/Breadcrumbs.md b/.agents/skills/kaizen-ui/references/components/Breadcrumbs.md new file mode 100644 index 000000000..8cd52826c --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Breadcrumbs.md @@ -0,0 +1,85 @@ +# Breadcrumbs + +A navigation trail that shows the user's location within the app hierarchy and links back to parent pages. Use concise labels for each level. + +## Notes + +- Do not render Breadcrumbs on root or first-level pages — there is nothing to navigate back to. + +- All parent entries must be clickable links so users can navigate back; only the final entry (current page) should be a plain label. + +- Keep labels concise — abbreviate or shorten long page titles so the trail fits on one line. + +- Include Breadcrumbs on any page deeper than one level, and place them in PageHeader's `slotBreadcrumbs`. + +## Examples + +### Basic Breadcrumbs + +```tsx +Home }, { children: Category }, "Current Page"]} /> +``` + +### With Icon + +Icons should be used sparingly, but the component supports them. + +```tsx + + + Home + + ), + }, + { children: Category }, + { + children: ( + <> + + Product 123 + + ), + }, + ]} +/> +``` + +### Composed + +```tsx + + + Home + + + + Category + + + Current Page + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| **items** * | `(string \| { children?: ReactNode })[]` | - | The breadcrumb entries, ordered from the root page to the current page. Strings render as plain text labels; objects allow custom content and per-slot native attributes. | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| size | `"small" \| "medium" \| "large"` | `"medium"` | Font size of the breadcrumb items. - "small" - 12px, for dense layouts. - "medium" - 14px, the standard size. - "large" - 16px, for prominent navigation. | +| slotSeparator | `ReactNode` | - | Replaces the default chevron separator between items with custom rendered content. | + +`* = required prop` + +## Exports + +- Components: `Breadcrumbs`, `BreadcrumbsItem`, `BreadcrumbsRoot`, `BreadcrumbsSeparator` +- Types: `BreadcrumbsItemFieldProps`, `BreadcrumbsProps`, `BreadcrumbsRootProps` + +## See also + +- [PageHeader](./PageHeader.md) diff --git a/.agents/skills/kaizen-ui/references/components/Button.md b/.agents/skills/kaizen-ui/references/components/Button.md new file mode 100644 index 000000000..eec4244f0 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Button.md @@ -0,0 +1,189 @@ +# Button + +A clickable element that triggers an action. Use specific verb + noun labels instead of vague text. + +## Notes + +- Unlike native + + + +``` + +### With Icon Button + +```tsx + +``` + +### Icon Only Button + +```tsx + +``` + +### Primary Action Button + +Preferred default for primary actions: primary & brand. Use the primary action button for the most important action in a context. It should be used sparingly and only for the most important actions(submit, create, deploy, save, etc) + +```tsx + +``` + +### Secondary Action Button + +Preferred default for secondary actions: secondary & neutral. Use the secondary action button for actions that are not the most important in a context. It should be used for actions that are not the most important in a context(cancel, close, etc) + +```tsx + +``` + +### Tertiary Or Inline Action Button + +Preferred default for tertiary/inline actions: tertiary & neutral. For example - edit, configure, kebab menu triggers, etc + +```tsx + +``` + +### Destructive Action Button + +Preferred default for destructive actions: primary & danger. Use the destructive action button for actions that are destructive and cannot be undone. For status-changing or reversible actions (remove from list, unassign, archive), use secondary + neutral. Reserve `color="danger"` for irreversible destructive actions. + +```tsx + +``` + +### Button As Link + +```tsx + +``` + +### Button Loading Methods + +Loading buttons with an unchanged label, a changing label, an icon-only action, no indicator, and a custom arrow spinner. + +```tsx + + + Same text, default spinner + + + + Changing text, default spinner + + + + Icon only + + + + Same text, no spinner + + + + Custom arrow spinner + + + +``` + +### Button With Truncation + +Button text should be short and concise, and if not text will wrap to the next line to ensure users can read the text. If you prefer to truncate the text you should set the `title` prop. + +```tsx + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| color | `"brand" \| "neutral" \| "danger"` | `"neutral"` | The color variant of the button. - "brand" - Prominent calls-to-action such as page-level CTAs and modal primary actions. - "neutral" - Regular actions, suitable for most use cases. - "danger" - Destructive actions that cannot be undone, such as delete or remove. | +| disabled | `boolean` | - | Disables the button. | +| kind | `"primary" \| "secondary" \| "tertiary"` | `"primary"` | The kind of button. - "primary" - The most important call-to-action on the page. Only one per context. - "secondary" - Regular actions, suitable for most use cases. - "tertiary" - Low-priority or supplemental actions. | +| loading | `boolean` | `false` | Shows a loading indicator alongside children and applies the pressed treatment while retaining focus and reporting `aria-busy` and `aria-disabled`. Suppresses activation events after hydration; guard native navigation, form submission, and custom child handlers in your application. | +| size | `"small" \| "tiny" \| "medium" \| "large"` | `"medium"` | The size of the button. - "large" - The main call-to-action for a page or section. - "medium" - Suitable for most use cases. - "small" - Compact layouts with limited space or less significant actions. - "tiny" - Dense layouts such as table cells where horizontal space is at a premium. | +| slotLoading | `ReactNode` | - | Noninteractive inline loading indicator; omit or pass `undefined` for the default spinner, or pass `null` for no indicator. Keep loading text in children. With `asChild`, render children directly inside the child's root element. Void elements such as `` cannot contain a custom indicator, so `slotLoading` is ignored for them. | + +## Exports + +- Components: `Button` +- Types: `ButtonProps` + +## See also + +- [ButtonGroup](./ButtonGroup.md) +- [Anchor](./Anchor.md) diff --git a/.agents/skills/kaizen-ui/references/components/ButtonGroup.md b/.agents/skills/kaizen-ui/references/components/ButtonGroup.md new file mode 100644 index 000000000..88a4292a2 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/ButtonGroup.md @@ -0,0 +1,93 @@ +# ButtonGroup + +A row of related buttons presented as a single set of actions with consistent styling. + +## Notes + +- Cap a ButtonGroup at four items. If more actions are needed, move overflow into a Dropdown rather than widening the group. + +- Prefer kind="secondary" or "tertiary" on the group. Multiple kind="primary" buttons in one group compete visually and dilute the page's primary action. + +- Only group actions that operate on the same entity or context. Independent or unrelated actions should be rendered as separate Buttons. + +- Note that SegmentedControl communicates "pick one" (mutually exclusive); ButtonGroup communicates "here are related actions"; individual Buttons are for independent actions. + +- Pick `groupKind` by spacing intent: "gap" for clear visual separation (default for primary), "flush" for tightly coupled actions like split buttons, "border" for a subtle separator without full gap spacing. + +- Action overflow / split button pattern: pair a primary Button with a Dropdown for alternatives — the primary action stays prominent while alternatives remain discoverable. + +## Examples + +### Basic Button Group + +Use when offering a set of related choices. Recommended to show no more than 4 actions. + +```tsx + +``` + +### With Children Button Group + +Use the children API when you need more control over the button group, such as adding dropdowns. Buttons rendered within will use the ButtonGroup's size/kind/color. + +```tsx + + + + + +``` + +### With Icon Button Group + +```tsx +, key: "settings", "aria-label": "Settings" }]} /> +``` + +### Disabled Button Group + +Use when the entire group of actions is unavailable due to a precondition. + +```tsx + +``` + +### Disabled Individual Button + +You can also disable individual buttons in the group by passing a `disabled` prop to the button. We recommend using children in this case so you can provide a tooltip explaining why the button is disabled. + +```tsx + + + + + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| color | `"brand" \| "neutral" \| "danger"` | `"neutral"` | The color variant of the button. - "brand" - Prominent calls-to-action such as page-level CTAs and modal primary actions. - "neutral" - Regular actions, suitable for most use cases. - "danger" - Destructive actions that cannot be undone, such as delete or remove. | +| disabled | `boolean` | - | Disables every button in the group. | +| groupKind | `"gap" \| "flush" \| "border"` | - | Overrides the divider style between buttons. Defaults to "gap" for primary, "flush" for secondary, and "border" for tertiary. | +| items | `ButtonGroupItem[]` | - | Buttons rendered from a configuration array. Mutually exclusive with `children`. | +| kind | `"primary" \| "secondary" \| "tertiary"` | `"primary"` | The kind of button. - "primary" - The most important call-to-action on the page. Only one per context. - "secondary" - Regular actions, suitable for most use cases. - "tertiary" - Low-priority or supplemental actions. | +| size | `"small" \| "tiny" \| "medium" \| "large"` | `"medium"` | The size of the button. - "large" - The main call-to-action for a page or section. - "medium" - Suitable for most use cases. - "small" - Compact layouts with limited space or less significant actions. - "tiny" - Dense layouts such as table cells where horizontal space is at a premium. | + +## Exports + +- Components: `ButtonGroup` +- Types: `ButtonGroupProps` + +## See also + +- [Button](./Button.md) +- [SegmentedControl](./SegmentedControl.md) +- [Dropdown](./Dropdown.md) diff --git a/.agents/skills/kaizen-ui/references/components/Card.md b/.agents/skills/kaizen-ui/references/components/Card.md new file mode 100644 index 000000000..eebc2c857 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Card.md @@ -0,0 +1,193 @@ +# Card + +A container that groups content, actions, and optional media about a single subject. Cards are fluid and grow to fill their container. + +## Notes + +- The Card already has padding, so do not add padding to the Card component or to it's children. + +- Card is for single-subject entity display (one cluster, one user, one model). Do not use Card as a general container on dashboards — use Panel instead. Card's interaction states and internal structure add unintended visual layering when used as a generic wrapper. + +- Cards are fluid and grow to fill their container — do not set fixed widths. Control sizing through the parent layout (e.g. a responsive Grid). Apply `className="h-fit"` when a card should shrink to its content height. + +- Identity test for Card vs Panel: if the container represents "a thing" with its own name and attributes, use Card; if it represents "a region of content", use Panel. + +- Pair `selected` with a visible affordance for what selection means — a bulk-action toolbar, a selection-count indicator, or a primary action — so users understand the consequence of selecting a card. + +- A Card is EITHER a single click target (set `interactive` and render via `asChild` as a link, button or label with containing hidden input) OR carries inline action buttons on its surface — never both. Combining them creates competing click targets and ambiguous focus order. + +- When a Card is `interactive`, do not place other interactive elements (Button, Anchor, Menu, etc.) inside it. Nested click targets break the "whole card is one click" affordance. + +## Examples + +### Basic Card That Fits Its Content + +```tsx + + + Badge + + Header + Lorem ipsum dolor sit amet + +``` + +### With Header Card + +The header slot is rendered absolutely over the media. Use it to add a persistent label or badge above the body content. If there is no media, the header will be rendered above the body content. It is a flex container with a preset gap. + +```tsx + + New + Alt + + } +> + Card body content + +``` + +### With Media Card + +Use when the card needs a visual hero area like an image or video above or alongside the content. If the media rendered is dark or light, and you're using `slotHeader` in conjunction with it, you may want to set `mediaTheme` to ensure text and components are readable. This will set light/dark theme in the header slot to ensure sufficient contrast. + +```tsx +Featured} slotMedia={} mediaTheme="light"> + Card content below media + +``` + +### Model Card Example + +Canonical browse-collection card anatomy for a single entity (model catalog, dataset gallery, template library, related-item strip). Slot order: header `Badge` (resource type) → publisher → title → description → topic `Tag` row (`outline` / `gray`) → footer stats (`text-placeholder`, left-aligned, never `justify="between"`). Every card in the collection must share this same slot structure — see the entity-cards pattern guide. + +```tsx +Model}> + + + NVIDIA + + Nemotron 3 Super 120B + 120B-parameter reasoning model optimized for enterprise RAG. + + + + Reasoning + + + English + + + RAG + + + + Updated 3d · 12.4k · 120B + + +``` + +### Interactive Card + +Use when the entire card should be a clickable target, such as navigation or selection. Render as an `` for navigation or a button for actions. Do not place other interactive elements (Button, Anchor, Menu, etc.) on the card surface — competing click targets are ambiguous. If you need inline actions per card instead, omit `interactive` and use the `With Actions` pattern below. + +```tsx + + + +``` + +### With Actions + +Use when the user needs to perform actions without navigating to a detail view, or when multiple distinct actions are available per card. The card itself is not a click target in this pattern — do not also enable `interactive`. + +```tsx + + Card content... Right aligned actions: + + + + + Or you can have actions stretch across the card: + + + + + Or for just a single action: + + +``` + +### Cards In Responsive Grid + +It's common to use cards in a responsive grid. Using the grid component, set a minimum column width and let the cards automatically resize to fill the available space. + +```tsx + + Card 1 + Card 2 + Card 3 + +``` + +### Composed + +```tsx + + + Featured}> + + + Composed card with media + + + +
    + Featured +
    + Composed card without media +     +     +     +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| attributes | `{ CardContent?: NativeElementAttributes<"div", typeof CardContent>; CardMedia?: NativeElementAttributes<"div", typeof CardMedia> }` | - | Native HTML attributes forwarded to the internal composed components. | +| density | `"compact" \| "standard" \| "spacious"` | `"standard"` | The "density" of the component. This affects the component padding. Set to `compact` for dense layouts, `standard` for general use, and `spacious` for marketing or onboarding surfaces. | +| interactive | `boolean` | `false` | Adds hover and focus affordances to signal the card is clickable. Do not enable on non-clickable cards — it creates false affordance. | +| kind | `"solid" \| "gradient" \| "float"` | `"solid"` | Visual treatment of the card. - "solid" - General-purpose treatment with padded content and a hard border between media and content. Use for most cards. - "gradient" - Same as "solid" but the media fades into the content. Use for promotional or hero-style cards. - "float" - Pairs with `slotMedia` to render the media as a bordered tile while the content sits on the page background without padding. Use when the card should visually lift off the surface. | +| layout | `"horizontal" \| "vertical"` | `"vertical"` | Orientation of the media and content. - "vertical" - Media stacks above the content. Use when the media is the focal point and cards display in a grid. - "horizontal" - Media sits to the left of the content. Use when the content outweighs the media and cards stack in a list. | +| mediaTheme | `"light" \| "dark"` | - | Forces the theme of overlaid `slotHeader` content so it contrasts with the media beneath. - "dark" - For light or bright media; renders overlaid content in the dark theme. - "light" - For dark media; renders overlaid content in the light theme. | +| selected | `boolean` | `false` | Applies the selected visual state, e.g. when the card represents the current choice in a list. | +| slotHeader | `ReactNode` | - | Header content rendered above the body. When paired with `slotMedia`, it overlays the media instead. | +| slotMedia | `ReactNode` | - | Media content rendered above the body, typically an image or video. Pair with `mediaTheme` when also using `slotHeader` so overlaid content remains readable. | + +## Exports + +- Components: `Card`, `CardContent`, `CardMedia`, `CardRoot` +- Types: `CardContentProps`, `CardMediaProps`, `CardProps`, `CardRootProps` + +## See also + +- [Panel](./Panel.md) +- [Grid](./Grid.md) diff --git a/.agents/skills/kaizen-ui/references/components/Carousel.md b/.agents/skills/kaizen-ui/references/components/Carousel.md new file mode 100644 index 000000000..07125b5e7 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Carousel.md @@ -0,0 +1,166 @@ +# Carousel + +A horizontal container that cycles through a series of content items. + +## Notes + +- The required `aria-label` (or `aria-labelledby`) should describe the carousel's content, e.g. aria-label="Featured models". + +- Children can be any content — Cards, images, custom markup. The Carousel wraps each direct child automatically in a CarouselItem. + +- There are no default arrow controls: use for the standard prev/next buttons around a progress indicator. For advanced control you can also use directly. + +- Use `itemsPerView` to size items relative to the viewport(e.g. 1 for single item). Use `itemWidth` for a fixed width per item (e.g. itemWidth="260px"). + +- GOTCHA: `itemsPerView` and `itemWidth` are mutually exclusive. Both should not be used together. + +## Examples + +### Basic Carousel + +Carousel supports any content type. Here is an example using Cards where it defers sizing to the cards. + +```tsx + + + + } +> + {Array.from({ length: 6 }, (_, index) => ({ + value: index, + content: `Item ${index + 1}`, + })).map((item) => ( + + +

    {item.content}

    +     +     + ))} + {/* we could also do images instead, or anything else: */} + {images.map((image) => ( + {image.alt} + ))} +     +``` + +### With Title And View All Carousel + +Fill slotHeader with a title and a navigation action. + +```tsx + + Featured models + + + } + slotFooter={ + + + + } +> + {cards} + +``` + +### Split Controls Carousel + +Place controls in the header when vertical space below the items is limited, or split the arrows from the indicator across both slots — here buttons in the header with the indicator centered below the items. + +```tsx + + Title + + + } + slotFooter={ + + + + } +> + {cards} + +``` + +### Count Indicator Carousel + +Use indicator="count" to show the visible item range (e.g. "1-4 of 12") instead of progress bars. + +```tsx + + + + } +> + {cards} + +``` + +### Items Per View Carousel + +Items are content-sized by default. Pass itemsPerView to fit a set number of items in the viewport — fractional values peek the next item. Use itemWidth instead for a fixed width per slide regardless of container size. + +```tsx + + + + } +> + {cards} + +``` + +### Composed + +```tsx + + + Slide content + Slide content + Slide content + + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| attributes | `{ CarouselItems?: NativeElementAttributes<"div", typeof CarouselItems> }` | - | Native HTML attributes forwarded to the internal composed components. | +| autoplay | `boolean` | `false` | Enables automatic page advancement. | +| autoplayInterval | `number` | `5000` | Time between automatic page advances in milliseconds (5000 ms = 5s). | +| itemsPerView | `number` | - | How many items fit in the viewport. Fractional values peek the next item — i.e. `3.5` shows three and a half, or `1` for single item. Overridden by `itemWidth`. | +| itemWidth | `string \| number` | - | Fixed width of every item. Accepts any CSS length, or numbers which are interpreted as pixels. Overrides `itemsPerView`. | +| loop | `boolean` | `false` | Whether next/previous navigation wraps around at the ends: next on the last page returns to the first, previous on the first jumps to the last, and the arrow buttons never disable. The wrap scrolls across the intermediate slides. | +| onPageChange | `(page: number, meta: { firstVisibleIndex: number; lastVisibleIndex: number; itemCount: number; }) => void` | - | Called when the visible page changes, from scrolling or the controls. | +| renderItem | `false \| ((child: ReactNode, index: number) => ReactNode)` | - | Renders each direct child as a slide. The default wraps each child in a {@link CarouselItem }, leaving children that already are one untouched. Pass `null` to disable the wrapping and render children as-is. | +| slotFooter | `ReactNode` | - | Row below the items, typically a {@link CarouselControls }. The row is unstyled — bring your own layout, e.g. a `Flex` with `justify="center"`. Nothing renders when omitted. | +| slotHeader | `ReactNode` | - | Row above the items, typically a title with an action or a {@link CarouselControls }. The row is unstyled — bring your own layout, e.g. a `Flex` with `justify="between"`. Nothing renders when omitted. | + +## Exports + +- Components: `Carousel`, `CarouselArrowButton`, `CarouselControls`, `CarouselIndicator`, `CarouselItem`, `CarouselItems`, `CarouselRoot` +- Hooks: `useCarouselContext` +- Types: `CarouselArrowButtonProps`, `CarouselContextValue`, `CarouselControlsProps`, `CarouselIndicatorProps`, `CarouselItemProps`, `CarouselItemsProps`, `CarouselProps`, `CarouselRootProps`, `CarouselRootPropsBase` diff --git a/.agents/skills/kaizen-ui/references/components/ChatBubble.md b/.agents/skills/kaizen-ui/references/components/ChatBubble.md new file mode 100644 index 000000000..2b8df5476 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/ChatBubble.md @@ -0,0 +1,115 @@ +# ChatBubble + +The surface of one chat message: a padded, rounded block for a person's words, or flat text for an AI reply. + +## Notes + +- Pick `kind` by author: `"strong"` for the current user, `"subtle"` (the default) for another person, `"flat"` for an AI or system reply, which drops the surface and padding so long text reads like a document. + +- The bubble is fluid: it fills its container and sets no width, max width, height, or alignment of its own. The layout around it, such as a Stack or Flex, owns all of that. + +- Children are the content: usually text, but anything renders. When a reply asks for a decision, pass the `Button`s to `slotActions` so they render last inside the bubble, after the text, and read as part of the reply. `slotActions` also works with `asChild`. Compose `ChatBubbleActions` yourself only when the row must sit somewhere other than last. It sets the space above the row and the gap between items, so do not wrap the actions in a `Flex` or add margins. It has no role; add `role="group"` and an `aria-label` through `attributes.ChatBubbleActions` only when the actions need a name. + +- The bubble has no role. Give a transcript container `role="log"` and an `aria-label`, and put the sender's name beside the bubble as plain text, not a heading. + +- Use `asChild` to render the bubble as another semantic element when needed. + +- ChatBubble is for conversation turns. Use Notification or Toast for system feedback and Card for an entity. + +## Examples + +### Basic Chat Bubble + +The default `kind="subtle"` is another person: a padded bubble on the highlight surface. The bubble fills its container, so the container decides how wide a message can be. + +```tsx +Can you summarize last night's deployment? +``` + +### Agent Chat Bubble With Actions + +`kind="flat"` is an AI or system reply: flat text with no surface or padding, so long generated text reads like a document. Content is any ReactNode, so when the reply asks for a decision, pass the Buttons to `slotActions`. They render in a row after the text, spaced from the text and from each other, so they read as part of the reply. + +```tsx + + + + + } +> + Three services rolled out overnight. Two succeeded and one was held back after a failed health check. Should I retry the failed rollout now or wait for the next maintenance window? + +``` + +### Chat Bubble Conversation + +One way a consumer lays out a back-and-forth conversation. The bubble is fluid and places nothing itself, so everything here beyond the bubbles is the consumer's layout: a Flex row per turn with the Avatar bottom-aligned beside the bubble, `justify="end"` to put the current user on the right, and a Stack capped at 70% so short messages hug their text while long ones wrap. The closing `kind="flat"` turn is the assistant: flat text with no cap, so a long reply reads like a document. The sender's name is plain text beside each bubble, and the transcript carries `role="log"` and an `aria-label` so it is a named live region. + +```tsx + + + + You + How did last night's deployment go? + + + + + + + You + Should we retry it now? + + + + + + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) | +| attributes | `{ ChatBubbleActions?: NativeElementAttributes<"div", typeof ChatBubbleActions> }` | - | Native HTML attributes forwarded to the internal composed components. | +| kind | `"strong" \| "flat" \| "subtle"` | `"subtle"` | The bubble's surface. - "strong" - The emphasis surface, for the current user. - "subtle" - The highlight surface, for another person. - "flat" - Flat text with no surface or padding, for an AI or system reply. | +| slotActions | `ReactNode` | - | Actions rendered in a `ChatBubbleActions` row after the content, such as the Buttons for a decision the reply asks for. With `asChild`, the row renders inside the child element, so the child must accept block content, such as an `
  • ` or `
    `, not a `

    ` or ``. | + +## Exports + +- Components: `ChatBubble`, `ChatBubbleActions` +- Types: `ChatBubbleActionsProps`, `ChatBubbleProps` + +## See also + +- [Stack](./Stack.md) +- [Avatar](./Avatar.md) diff --git a/.agents/skills/kaizen-ui/references/components/Checkbox.md b/.agents/skills/kaizen-ui/references/components/Checkbox.md new file mode 100644 index 000000000..32b54c3b9 --- /dev/null +++ b/.agents/skills/kaizen-ui/references/components/Checkbox.md @@ -0,0 +1,125 @@ +# Checkbox + +A form control that lets users toggle a single option on or off. Supports a mixed state for representing partially-selected groups of related options. + +## Notes + +- Use Checkbox for values submitted with a form ("apply on submit") and for terms or consent acknowledgments. For binary settings that take effect immediately, use Switch. + +- Mental model: Switch communicates "this takes effect immediately"; Checkbox communicates "this will be submitted with the form". + +- Use a group of Checkboxes for multiple independent selections. For mutually exclusive choices, use RadioGroup; for 5+ options, use Select or Combobox with `multiple`. + +- Always provide `slotLabel`, or `aria-label` when surrounding context already names the option, unless labeling is already provided(FormField, wrapped in

    + Use the hook to manage state. +

    +``` + +### Collapsible Code Snippet + +Use when displaying long code that should be collapsed by default to save vertical space. + +```tsx + +``` + +### With Inline Copy Code Snippet + +Use for short snippets, such as a single install command, where a dedicated actions row would take more vertical space than the code. Resolves back to "header" when `collapsible` is set, because the expand toggle needs the actions row. + +```tsx + +``` + +### With Copy Callback Code Snippet + +Use when you need to track or respond to copy events, such as for analytics. + +```tsx + +``` + +### With Custom Actions Code Snippet + +Use when you need to add custom actions to the code snippet. For Buttons, use size="small" so they match the built-in copy and expander buttons. + +```tsx + + Run + + } +/> +``` + +### With Fixed Theme + +By default, the code snippet will render in the current theme. You can fix the theme to light or dark by applying the light or dark class to the CodeSnippet or its parent + +```tsx +
    + +
    +``` + +### With Server Side Rendering Code Snippet + +Use when you need to server-side render the code snippet. + +```tsx +async () => { + // use highlightCode util to server side highlight the code + const prerenderedCode = await highlightCode(sampleCode, "typescript"); + return ; +}; +``` + +### Composed + +```tsx + + Block: + + + + + + + + + Inline: + + + + +``` + +## Props + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| **kind** * | `"inline" \| "block"` | `"block"` | Visual layout of the snippet. - "block" - Stacked block layout that hosts actions and a copy button. - "inline" - Inline span sized to fit within surrounding prose. | +| **value** * | `string` | - | Source code rendered inside the snippet. | +| attributes | `{ CodeSnippetActions?: NativeElementAttributes<"div", typeof CodeSnippetActions>; CodeSnippetCode?: NativeElementAttributes<"div", typeof CodeSnippetCode>; CodeSnippetCopyButton?: NativeElementAttributes<"button", typeof CodeSnippetCopyButton>; CodeSnippetExpanderButton?: NativeElementAttributes<"button", typeof CodeSnippetExpanderButton> }` | - | Native HTML attributes forwarded to the internal composed components. | +| collapsible | `boolean` | - | Adds a chevron toggle and hides overflow until the user expands the snippet. Only honored when `kind="block"`. | +| copyButton | `"header" \| "none" \| "inline"` | `"header"` | Whether the snippet renders a copy button, and where it sits. - "header" - In the actions row above the code, next to any custom actions. - "inline" - Pinned to the top-right corner of the code itself, so the snippet spends no vertical space on an actions row. - "none" - No copy button. The actions row still renders for `slotActions` or the expand toggle. Falls back to "header" when `collapsible` is true, because the expand toggle needs the actions row anyway. Always resolves to "none" when `kind="inline"`, which has no code area to pin a button to. | +| defaultOpen | `boolean` | `false` | Initial open state when uncontrolled. | +| defaultValue | `string` | - | Pre-highlighted HTML used in place of client-side highlighting. Pass server-rendered HTML here for hydration-safe SSR. | +| language | `"html" \| "text" \| "go" \| "typescript" \| "javascript" \| "tsx" \| "jsx" \| "json" \| "css" \| "bash" \| "shell" \| "python" \| "rust" \| "yaml" \| "markdown"` | `"text"` | Language token used for syntax highlighting. | +| onCopySuccess | `() => void` | - | Called after the code is successfully copied to the clipboard. | +| onOpenChange | `(open: boolean) => void` | - | Called whenever the open state changes. | +| open | `boolean` | - | Controlled open state of the collapsible block. | +| rows | `number` | - | Number of code rows visible before scrolling or collapsing. Acts as the collapsed height when `collapsible` is true, otherwise caps the height with overflow scroll. Only honored when `kind="block"`. | +| slotActions | `ReactNode` | - | Additional action elements rendered alongside the built-in controls. | +| structure | `"inline" \| "classic"` | `Derived from `kind`: "block" -> "classic", "inline" -> "inline".` | HTML element structure used for rendering. Override when block-styled code must nest inside a `

    ` tag. - "classic" - Uses block elements (`

    `, `
    `, ``); cannot nest inside `
    ` tags. - "inline" - Uses inline elements (``); safe to nest inside `
    ` tags (e.g. Markdown). |
+
+`* = required prop`
+
+## Exports
+
+- Components: `CodeSnippet`, `CodeSnippetActions`, `CodeSnippetCode`, `CodeSnippetCopyButton`, `CodeSnippetExpanderButton`, `CodeSnippetRoot`
+- Utilities: `assertCodeSnippetLanguage`, `getCodeSnippetLanguage`, `getHighlighter`, `highlightCode`, `initHighlighter`, `isCodeSnippetLanguage`, `sanitizeCodeHtml`
+- Constants: `DEFAULT_LANGUAGES`, `DEFAULT_THEMES`
+- Types: `CodeSnippetActionsProps`, `CodeSnippetCodeProps`, `CodeSnippetCopyButtonProps`, `CodeSnippetExpanderButtonProps`, `CodeSnippetLanguage`, `CodeSnippetProps`, `CodeSnippetRootProps`, `CodeSnippetTheme`, `HighlightCodeOptions`
diff --git a/.agents/skills/kaizen-ui/references/components/Collapsible.md b/.agents/skills/kaizen-ui/references/components/Collapsible.md
new file mode 100644
index 000000000..304170208
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/Collapsible.md
@@ -0,0 +1,120 @@
+# Collapsible
+
+An unstyled disclosure that toggles a single block of content between expanded and collapsed states. Use to hide secondary detail behind a clickable summary so users can reveal more on demand.
+
+## Notes
+
+- For a set of independently togglable sections that share a header treatment, prefer Accordion.
+
+- The trigger must visually read as an interactive control — pair it with a chevron (e.g. `AnimatedChevron`) and ensure it has hover and focus affordances. Plain bold text reads as a section heading and users miss the interaction entirely.
+
+- For a trigger with hover/focus affordances, render a `Button` with `asChild` around a `
    `(do not want to nest interactive elements inside the trigger) and pass that as the `slotTrigger`. The Button supplies hover/focus/keyboard affordance, the chevron signals state, and the combination disambiguates the label from a heading.
+
+- For a noun-phrase label like "Advanced parameters", wrap a `Button asChild kind="tertiary">` in `slotTrigger` and add an `AnimatedChevron` so the affordance is unambiguous.
+
+- Anti-patterns: do not render the trigger as a plain heading; do not hide the chevron behind hover-only visibility; do not use noun-phrase labels like "Advanced parameters" or "More information" without a chevron — the noun-phrase framing compounds the heading-look and users scan past it.
+
+## Examples
+
+### Basic Collapsible
+
+Use when content should be hidden by default to reduce visual noise but available on demand.
+
+```tsx
+Hidden content revealed on expand.
+```
+
+### Default Open Collapsible
+
+```tsx
+
+  Content visible by default.
+
+```
+
+### With Icon In Trigger
+
+```tsx
+
+      
+      Header
+    
+  }
+>
+  Content
+
+```
+
+### With Chevron Trigger
+
+The AnimatedChevron component automatically reads the open/closed state from Collapsible.
+
+```tsx
+
+       Toggle content
+    
+  }
+>
+  Content
+
+```
+
+### With Button Trigger
+
+When using a Button in slotTrigger with asChild, render it as a non-interactive element like 
     to avoid nested interactive elements, allowing the native  element to handle the toggle behavior. The recommended pattern is `Button kind="tertiary"` with a trailing `AnimatedChevron`: the Button supplies hover/focus/keyboard affordance, the chevron signals state, and the combination disambiguates the label from a heading.
+
+```tsx
+
+      
    Toggle content
    
+    
+  }
+>
+  Content
+    
+```
+
+### Composed
+
+Use composed primitives when you want to render the trigger and content apart from each other.
+
+```tsx
+
+  
+    
+  
+  Additional content - visible by default.
+
+```
+
+## Props
+
+| Prop | Type | Default | Description |
+| --- | --- | --- | --- |
+| **children** * | `ReactNode` | - | Content rendered inside the expandable region. |
+| **slotTrigger** * | `ReactNode` | - | Content rendered inside the trigger that toggles the region. |
+| attributes | `{ CollapsibleTrigger?: NativeElementAttributes<"summary", typeof CollapsibleTrigger>; CollapsibleContent?: NativeElementAttributes<"div", typeof CollapsibleContent> }` | - | Native HTML attributes forwarded to the internal composed components. |
+| defaultOpen | `boolean` | `false` | Initial open state when the component is uncontrolled. |
+| disabled | `boolean` | `false` | Prevents the trigger from toggling the open state and blocks programmatic toggle commands. |
+| onOpenChange | `(open: boolean) => void` | - | Called when the open state changes via user interaction or invoker commands. |
+| open | `boolean` | - | Controlled open state. Pair with `onOpenChange` to manage state externally. |
+| showChevron | `boolean` | `false` | Renders a chevron after the trigger content that rotates to reflect the open state. |
+
+`* = required prop`
+
+## Exports
+
+- Components: `Collapsible`, `CollapsibleCommands`, `CollapsibleContent`, `CollapsibleRoot`, `CollapsibleTrigger`
+- Hooks: `useCollapsibleContext`
+- Contexts: `CollapsibleContext`
+- Types: `CollapsibleContentProps`, `CollapsibleContextValue`, `CollapsibleProps`, `CollapsibleRootProps`, `CollapsibleTriggerProps`
diff --git a/.agents/skills/kaizen-ui/references/components/Combobox.md b/.agents/skills/kaizen-ui/references/components/Combobox.md
new file mode 100644
index 000000000..86ee03a01
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/Combobox.md
@@ -0,0 +1,233 @@
+# Combobox
+
+A text input paired with a filterable listbox for selecting one or more values from a list. Use when users need to search through a large set of options.
+
+## Notes
+
+- Use Combobox when the option list is long enough that filtering helps; reach for Select when the list is short and bounded. The boundary isn't a hard count — favor Combobox when the option count is unknown or could grow at runtime, Select when you know the set stays small. For unconstrained free text use TextInput.
+
+- Multi-select Combobox is preferred over multi-select Select when the option list is long — the filter lets users find items without scrolling through dozens of options.
+
+- Write descriptive placeholders ("Search regions...", "Search team...") so users know they can type to filter.
+
+- Group items only when there are 10+ options with natural categories; keep groups to 3–5 and order them by usage frequency.
+
+- Prefer default `size="medium"` for NVIDIA enterprise contexts; do not pass `size="small"` outside dense-layout exceptions.
+
+- For optional fields, set `dismissible: true` so users can clear the selection. Leave it off on required fields — a required field must always carry a value, so the clear button creates a validation conflict.
+
+- Top-level props are split by element kind: recognized `` attributes (`aria-label`, `placeholder`, `name`, `autoComplete`, …) go to the inner input; every other top-level prop (`onPointerEnter`, `onClick`, `data-*`, …) is spread onto `ComboboxContent` (the popover), not the trigger. Use `attributes.ComboboxTrigger` or `attributes.InputShell` when something must live on the trigger chrome.
+
+- Anti-patterns: Combobox for a handful of options when you know the set stays small (use Select or RadioGroup); missing descriptive placeholder text; Combobox for pure free-text entry.
+
+## Examples
+
+### Basic Combobox
+
+```tsx
+
+  
+  
+
+```
+
+### With Default Value Combobox
+
+```tsx
+
+```
+
+### Multiple Combobox
+
+Use when users need to select multiple values with type-ahead filtering. By default, the selected values are rendered as tags in the trigger.
+
+```tsx
+
+```
+
+### Multiple Combobox With Default Values
+
+```tsx
+
+```
+
+### Combobox With Separate Item Values
+
+```tsx
+
+```
+
+### Combobox With Rich Item Children
+
+When setting rich children in `items`, you should also provide a `filterValue` which is a text only representation of `children` that will be used for type-ahead filtering.
+
+```tsx
+
+           Red
+        
+      ),
+    },
+  ]}
+  placeholder="Pick colors..."
+/>
+```
+
+### Multiple Count Mode Combobox
+
+Use count mode when many selections are expected and individual tags would overflow the trigger.
+
+```tsx
+
+```
+
+### Grouped Items Combobox
+
+Use when options logically belong to categories that help users scan the list.
+
+```tsx
+
+```
+
+### Custom Empty State Combobox
+
+```tsx
+
+```
+
+### Dismissible Combobox
+
+Use when users need a quick way to clear their selection.
+
+```tsx
+
+```
+
+### Disabled Combobox
+
+Use when the selection is locked by external conditions.
+
+```tsx
+
+```
+
+### With Slot Start And End Combobox
+
+```tsx
+} slotEnd={} />
+```
+
+### With Async Loading Combobox
+
+```tsx
+ {
+    console.log("Loading more items...");
+  }}
+/>
+```
+
+### Composed
+
+Use the composed primitives when you need full control over trigger rendering, content layout, or custom items.
+
+```tsx
+
+  
+  
+    
+      Apple
+      Banana
+      Cherry
+    
+  
+
+```
+
+## Props
+
+| Prop | Type | Default | Description |
+| --- | --- | --- | --- |
+| **items** * | `(string \| Pick \| ComboboxSectionEntry)[]` | - | Declarative list of options, sections, or string shortcuts shown in the listbox. |
+| allowBackspaceRemoval | `boolean` | - | In multi-select mode, pressing `Backspace` while the input is empty removes the last selected value. Defaults to `true` when selections render as tags with the built-in renderer, and `false` otherwise. |
+| attributes | `{ ComboboxContent?: ComboboxContentProps; ComboboxListbox?: ComboboxListboxProps; ComboboxTrigger?: ComboboxTriggerProps; InputShell?: InputShellProps }` | - | Native HTML attributes forwarded to the internal composed components. |
+| children | `(props: { mode: "count" \| "tags"; removeSelectedValue: (value: string) => void; selectedItems: { label: string; value: string; }[]; summaryLabel: string }) => ReactNode` | - | Custom renderer for the selected values. When provided, `allowBackspaceRemoval` defaults to `false`. |
+| className | `string` | - | Class names applied to the field container that wraps the input, not to the `` itself. |
+| defaultInputValue | `string` | - | Initial input value when uncontrolled. |
+| defaultOpen | `boolean` | `false` | Initial open state when uncontrolled. |
+| defaultValue | `string \| string[]` | - | The selected value when the Combobox is initially rendered. |
+| density | `"compact" \| "standard" \| "spacious"` | `"standard"` | The "density" of the component. This affects the component padding. Set to `compact` for dense layouts, `standard` for general use, and `spacious` for marketing or onboarding surfaces. |
+| disabled | `boolean` | - | Disables interaction with the Combobox. |
+| dismissible | `boolean` | - | Renders a dismiss button that clears the Combobox value. |
+| emptyStateMessage | `string` | `"No results found"` | Message rendered when no items match the current filter. |
+| filteringBehavior | `"after-input" \| "on-open"` | `"after-input"` | Controls when the option list begins filtering. - "after-input" - Shows all options when the popover opens, then filters after the user types. - "on-open" - Filters immediately using the current input value. |
+| filterMatchFn | `"disable" \| ((matchTerm: string, value: string) => boolean)` | - | Custom matcher used to decide whether an item is visible for the current search value. Defaults to a case- and diacritic-insensitive substring match. Set to `"disable"` to disable filtering. |
+| formatSummaryLabel | `(count: number) => string` | - | Optional formatter for the count summary label used when `mode="count"` and by render-prop consumers. |
+| hideOnEscape | `boolean` | `true` | Closes the Combobox when Escape is pressed. |
+| id | `string` | - | Id applied to the input element. |
+| inputValue | `string` | - | Controlled input value. |
+| kind | `"flat" \| "floating"` | `"flat"` | Visual treatment of the shell. `"flat"` has a border and background; `"floating"` is borderless and transparent. |
+| multiple | `boolean` | `false` | Enable multi-select. When `true`, `value` / `defaultValue` are arrays of strings and `onValueChange` receives an array; when omitted or `false` (the default), the Combobox is single-select and those props are plain strings. |
+| multipleMode | `"count" \| "tags"` | `"tags"` | Controls how selected values are rendered in multi-select mode. - "tags" - Render each selection as an individually labelled, removable tag. - "count" - Render a compact summary of how many values are selected. |
+| name | `string` | - | Form field name used by the hidden native `` used for form submission. |
+| side | `"right" \| "bottom" \| "left" \| "top"` | `"bottom"` | Side of the trigger the popover anchors to. |
+| size | `"small" \| "medium" \| "large"` | `"medium"` | Visual size of the Combobox trigger. |
+| slotEnd | `ReactNode` | - | Content rendered after the input. |
+| slotStart | `ReactNode` | - | Content rendered before the input. |
+| status | `"error" \| "success"` | - | The status of the input. Use `withValidation` to automatically apply success/error states based on `:user-valid` and `:user-invalid` pseudo classes. |
+| style | `React.CSSProperties` | - | Inline styles applied to the field container that wraps the input, not to the `` itself. |
+| value | `string \| string[]` | - | Controlled selected value. |
+
+`* = required prop`
+
+## Exports
+
+- Components: `Combobox`, `ComboboxContent`, `ComboboxItem`, `ComboboxListbox`, `ComboboxRoot`, `ComboboxSelectedTags`, `ComboboxToggle`, `ComboboxTrigger`
+- Hooks: `useComboboxContext`
+- Contexts: `ComboboxContext`
+- Types: `ComboboxContentProps`, `ComboboxContextValue`, `ComboboxDefaultItem`, `ComboboxEntry`, `ComboboxFilteringBehavior`, `ComboboxFilterMatchFn`, `ComboboxItemProps`, `ComboboxListboxProps`, `ComboboxProps`, `ComboboxRootProps`, `ComboboxSectionEntry`, `ComboboxSelectedTagsProps`, `ComboboxSelectedTagsRenderProps`, `ComboboxSelectedValuesDisplay`, `ComboboxSingleAndMultipleProps`, `ComboboxToggleProps`, `ComboboxTriggerProps`, `ComboboxValue`
+
+## See also
+
+- [Select](./Select.md)
+- [TextInput](./TextInput.md)
+- [FormField](./FormField.md)
diff --git a/.agents/skills/kaizen-ui/references/components/Command.md b/.agents/skills/kaizen-ui/references/components/Command.md
new file mode 100644
index 000000000..b65469a6c
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/Command.md
@@ -0,0 +1,198 @@
+# Command
+
+A modal menu that lets users search for and run actions or jump to pages from the keyboard. Opens over the page, filters its items as the user types, and closes when an item is chosen.
+
+## Notes
+
+- Use Command for an app-wide palette of actions and destinations, usually opened with a global shortcut such as Meta+K. For a menu attached to a button, use Dropdown. For choosing a form value, use Combobox or Select.
+
+- Command always renders in a modal dialog. To open it from a global shortcut, control it with `open` and `onOpenChange` and listen for the key yourself. Put `aria-keyshortcuts` on any visible trigger.
+
+- Group items into sections under short `slotHeading` labels such as "Suggestions", "Navigation", or "Settings". Separate sections with `{ kind: "divider" }`.
+
+- Show an action's shortcut in `slotEnd` as ``, and name it with `aria-keyshortcuts` in the item's `attributes`. Show it only if the shortcut also works outside the menu.
+
+- Use `href` for items that navigate, and pass `renderLink` so framework routing stays client-side. Use `onSelect` for items that perform an action.
+
+- Set `filterValue` when users might search with words that are not in the label, such as "preferences" for a "Settings" item.
+
+- Within a section, either every item has a `slotStart` icon or none does.
+
+## Examples
+
+### Basic Command
+
+Use for an app-wide palette of actions and destinations. Give actions an `onSelect` and destinations an `href`, and group them under short headings.
+
+```tsx
+Open command menu}
+  items={[
+    {
+      slotHeading: "Suggestions",
+      items: [
+        {
+          children: "New document",
+          // onSelect: createDocument,
+          slotStart: ,
+          slotEnd: ,
+        },
+        {
+          children: "Upload file",
+          // onSelect: uploadFile,
+          slotStart: ,
+          slotEnd: ,
+        },
+        {
+          children: "Refresh data",
+          // onSelect: refreshData,
+          slotStart: ,
+        },
+      ],
+    },
+    { kind: "divider" },
+    {
+      slotHeading: "Navigation",
+      items: [
+        {
+          children: "Calendar",
+          href: "/calendar",
+          slotStart: ,
+        },
+        {
+          children: "Notifications",
+          href: "/notifications",
+          slotStart: ,
+        },
+        {
+          children: "Projects",
+          href: "/projects",
+          slotStart: ,
+        },
+      ],
+    },
+  ]}
+/>
+```
+
+### Keyboard Shortcut Command
+
+Control `open` to toggle the menu from a global shortcut. Listen for the key on the document, and name the shortcut on the visible trigger with `aria-keyshortcuts`.
+
+```tsx
+() => {
+  const [open, setOpen] = useState(false);
+
+  /** @remarks the shortcut works anywhere on the page, so it listens on the document rather than on an element */
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      if (event.key === "k" && (event.metaKey || event.ctrlKey)) {
+        event.preventDefault();
+        setOpen((current) => !current);
+      }
+    };
+    document.addEventListener("keydown", handleKeyDown);
+    return () => document.removeEventListener("keydown", handleKeyDown);
+  }, []);
+
+  return (
+    
+          Search
+          
+        
+      }
+      items={[
+        { children: "Calendar", href: "/calendar" },
+        { children: "Notifications", href: "/notifications" },
+        {
+          children: "Settings",
+          filterValue: "settings preferences",
+          href: "/settings",
+        },
+      ]}
+    />
+  );
+};
+```
+
+### Composed
+
+Use the composed parts when you need to control each region yourself, such as rendering an item as your router's link through `asChild`.
+
+```tsx
+
+  
+    
+  
+  
+    
+      
+      
+        
+          
+                Getting started
+          
+          
+            Theming
+          
+        
+        
+        
+          
+            Modal
+          
+          
+            Side Panel
+          
+        
+      
+    
+  
+
+```
+
+## Props
+
+| Prop | Type | Default | Description |
+| --- | --- | --- | --- |
+| **items** * | `(CommandItemEntry \| { kind?: "section"; slotHeading?: ReactNode; items: CommandItemEntry[] } \| { kind: "divider" })[]` | - | Items, sections, and dividers rendered in the list, in order. |
+| aria-label | `string` | `"Command menu"` | Accessible name of the dialog. Pass `aria-labelledby` instead when a visible heading names it. |
+| attributes | `{ CommandTrigger?: NativeElementAttributes<"button", typeof CommandTrigger>; CommandAnchorTrigger?: NativeElementAttributes<"a", typeof CommandAnchorTrigger>; CommandDialog?: NativeElementAttributes<"dialog", typeof CommandDialog>; CommandInput?: Omit; CommandList?: NativeElementAttributes<"div", typeof CommandList> }` | - | Native HTML attributes forwarded to the internal composed components. |
+| closeOnClickOutside | `boolean` | `true` | Whether clicking the backdrop closes the menu. |
+| defaultFilterValue | `string` | - | Initial search value when the menu is uncontrolled. The search resets each time the menu opens. |
+| defaultOpen | `boolean` | `false` | Initial open state when the dialog is uncontrolled. |
+| emptyStateMessage | `string` | `"No results found"` | Message shown in place of the items when the search matches nothing, and announced to screen readers. |
+| filterMatchFn | `"disable" \| ((matchTerm: string, value: string) => boolean)` | - | Custom matcher used to decide whether an item is visible for the current search value. Defaults to a case- and diacritic-insensitive substring match. Set to `"disable"` to show every item, for example when results come from a server. |
+| filterValue | `string` | - | Controlled search value. Pair with `onFilterChange`, and clear it when the menu closes, since only an uncontrolled search resets on its own. |
+| forceMount | `boolean` | - | Ensures dialog content remains mounted when the dialog is not open. |
+| id | `string` | - | HTML `id` attribute applied to the `` element. |
+| onEscapeKeyDown | `(event: KeyboardEvent) => void` | - | Called when Escape is pressed inside the dialog. Call `event.preventDefault()` to keep the dialog open. |
+| onFilterChange | `(value: string) => void` | - | Called with the new search value as the user types. |
+| onInteractOutside | `(event: MouseEvent) => void` | - | Called when any interaction occurs outside the dialog. Call `event.preventDefault()` to keep the menu open. |
+| onItemSelect | `(event: Event, item: CommandItemEntry) => void` | - | Called when any item is chosen, receiving the underlying event and the entry. Call `event.preventDefault()` to keep the menu open. |
+| onOpenChange | `(open: boolean, request: { cancel: () => void; }) => void` | - | Callback invoked whenever the dialog requests an open state change, in both controlled and uncontrolled modes. A controlled dialog requests `false` before its exit animation, so keeping `open` as `true` keeps it open. When refusing, also call `request.cancel()` so a later close does not reuse the refused return value. Controlled prop changes made by the consumer are not echoed back through this callback. |
+| onPointerDownOutside | `(event: MouseEvent) => void` | - | Called when a pointer event occurs outside the dialog. Call `event.preventDefault()` to keep the menu open. |
+| open | `boolean` | - | Controlled open state. When provided, the hook operates in controlled mode and `defaultOpen` is ignored. The consumer is responsible for updating this value in response to {@link UseDialogOptions.onOpenChange} callbacks, and the dialog stays open until it becomes `false`. |
+| placeholder | `string` | `"Type a command or search..."` | Hint shown while the field is empty. Also the field's accessible name unless `aria-label` is set. |
+| portal | `boolean` | `false` | Renders the floating content in a React portal. |
+| portalContainer | `HTMLElement` | `document.body` | Container for portaled floating content when no nearer portal target exists. |
+| renderLink | `(item: Omit & { href: string; }) => ReactNode` | - | Custom renderer for items with an `href`. Use to swap in a framework-specific link component such as Next.js ``. |
+| slotAnchorTrigger | `ReactNode` | - | Anchor-based trigger that opens the menu on click and falls back to navigation when JavaScript is unavailable. Requires `id`. |
+| slotTrigger | `ReactNode` | - | Trigger that opens the menu when activated. Omit it to open the menu from your own state with `open` and `onOpenChange`. |
+
+`* = required prop`
+
+## Exports
+
+- Components: `Command`, `CommandAnchorTrigger`, `CommandContent`, `CommandDialog`, `CommandDivider`, `CommandHeading`, `CommandInput`, `CommandItem`, `CommandList`, `CommandRoot`, `CommandSection`, `CommandTrigger`
+- Types: `CommandAnchorTriggerProps`, `CommandContentProps`, `CommandDialogProps`, `CommandDividerEntry`, `CommandDividerProps`, `CommandEntry`, `CommandHeadingProps`, `CommandInputProps`, `CommandItemEntry`, `CommandItemProps`, `CommandListProps`, `CommandProps`, `CommandRenderLinkItem`, `CommandRootProps`, `CommandSectionEntry`, `CommandSectionProps`, `CommandTriggerProps`
+
+## See also
+
+- [Dropdown](./Dropdown.md)
+- [Combobox](./Combobox.md)
+- [Kbd](./Kbd.md)
diff --git a/.agents/skills/kaizen-ui/references/components/DatePicker.md b/.agents/skills/kaizen-ui/references/components/DatePicker.md
new file mode 100644
index 000000000..0b3fae660
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/DatePicker.md
@@ -0,0 +1,254 @@
+# DatePicker
+
+An input field paired with a calendar popup for selecting a single date or a date range. Use when users need to enter or browse to a specific date.
+
+## Notes
+
+- Always use DatePicker for date input — never a plain TextInput. Free-text date entry is error-prone, format-inconsistent, and lacks calendar context.
+
+- Use `kind="range"` for start/end date pairs instead of two separate single DatePickers — the linked fields keep the range valid and visually connected.
+
+- Constrain the valid window with `disabledMatcher` (e.g. `{ before: new Date() }` for future-only dates) and `startMonth` / `endMonth` for the navigable calendar.
+
+- Sibling comparison: use DatePicker for any date selection, TextInput only for non-date text or with type="time" for time-of-day selection.
+
+- For optional date fields, set `dismissible: true` so users can clear the selection. Leave it off on required fields — a required field must always carry a value, so the clear button creates a validation conflict.
+
+- For `format`, follow the application's locale conventions; pair with `formatFn` for fully custom rendering.
+
+## Examples
+
+### Basic Date Picker
+
+Use when users need to select a single date.
+
+```tsx
+
+```
+
+### Range Date Picker
+
+Set `kind="range"` when users need to select a start and end date, such as for filtering or scheduling.
+
+```tsx
+
+```
+
+### With Default Value Date Picker
+
+Single pickers accept a `Date` for `defaultValue` / `value` and emit `Date | undefined` from `onValueChange`. Range pickers accept a `DateRange` object with optional `from` / `to` Date properties and emit `DateRange | undefined`. The value can become `undefined` when the user clears the field, so always treat the controlled value as nullable.
+
+```tsx
+
+  
+  
+
+```
+
+### With Range Placeholder Date Picker
+
+Pass an object `{ from, to }` to give the range fields distinct placeholders. A plain string is applied to both inputs.
+
+```tsx
+
+```
+
+### Disable Weekends Date Picker
+
+Use `{ dayOfWeek: [...] }` to disable specific days, where 0 is Sunday and 6 is Saturday. The same matcher shape applies to range pickers.
+
+```tsx
+
+```
+
+### Disable Past Dates Date Picker
+
+Use `{ before }` to forbid dates earlier than a cutoff (here, today), or `{ after }` for the inverse.
+
+```tsx
+
+```
+
+### Combined Disabled Matcher Date Picker
+
+Pass an array to combine matchers; any date matched by _any_ entry is disabled. Other matcher shapes include a single `Date`, an array of `Date`s, a `{ from, to }` range, and a predicate `(date) => boolean`.
+
+```tsx
+
+```
+
+### Constrained Range Date Picker
+
+On a range picker `min` and `max` are the minimum and maximum number of **nights** in the selection — not absolute dates. HTML cannot couple two native date inputs, so these are only enforced by the enhanced widget; re-validate no-JS submissions with `validateDateRangeNights` on the server.
+
+```tsx
+
+```
+
+### Exclude Disabled Dates Range Date Picker
+
+Range-only. With `excludeDisabledDates`, users cannot select a range that spans a disabled date. Without it, the range may straddle disabled dates as long as the endpoints themselves are selectable.
+
+```tsx
+
+```
+
+### Calendar Bounds Date Picker
+
+Use `startMonth` and `endMonth` to constrain the navigable calendar window (defaults: 100 years ago to 5 years from now). These values are also translated into native `min` / `max` for the no-JS fallback. Use `defaultMonth` to set the initially-visible month without selecting a date.
+
+```tsx
+
+```
+
+### Custom Format Date Picker
+
+`format` accepts a `date-fns` token string (e.g. `"yyyy-MM-dd"`, `"dd-MM-yyyy"`, `"MMM d, yyyy"`). Display formatting only — the form value is always submitted as ISO `yyyy-MM-dd`.
+
+```tsx
+
+```
+
+### Custom Format Function Date Picker
+
+Use `formatFn` for full control over display formatting; it overrides `format`. Always set `placeholder` when supplying `formatFn` because the format string can no longer be used as a default hint.
+
+```tsx
+
+    date.toLocaleDateString("en-US", {
+      month: "short",
+      day: "numeric",
+      year: "numeric",
+    })
+  }
+  placeholder="Mon D, YYYY"
+/>
+```
+
+### Time Zone Date Picker
+
+Pass an IANA name (e.g. `"America/New_York"`) or UTC offset (e.g. `"-05:00"`) so the picker interprets and renders the date in that zone instead of the user's local zone.
+
+```tsx
+
+```
+
+### Form Submission Date Picker
+
+Apply the form `name` to `DatePickerNativeFallback`, not to `DatePickerInput`. The enhanced text input is display-only; the hidden native `` is the element that participates in form submission and always submits an ISO `yyyy-MM-dd` value. Use `parseDateFromFormData(formData, "dueDate")` on the server.
+
+```tsx
+
+```
+
+### Form Submission Range Date Picker
+
+Range pickers submit two independent ISO values, so set the `name` on both `DatePickerNativeFallbackFrom` and `DatePickerNativeFallbackTo`. Parse with `parseDateRangeFromFormData(formData, { from: "rolloutFrom", to: "rolloutTo" })`.
+
+```tsx
+
+```
+
+### Custom Input Attributes Date Picker
+
+Use the `attributes` API to set per-input `aria-label`, `autoComplete`, `data-*`, etc. without dropping to composed primitives. Single pickers expose `DatePickerInput`; range pickers expose `DatePickerInputFrom` and `DatePickerInputTo`.
+
+```tsx
+
+```
+
+### Composed
+
+Use the composed primitives when you need full control over trigger, input, calendar layout, or to slot in a custom rendering between any of them.
+
+```tsx
+
+  
+    
+    
+  
+  
+    
+  
+
+```
+
+## Props
+
+| Prop | Type | Default | Description |
+| --- | --- | --- | --- |
+| align | `"center" \| "start" \| "end"` | `"center"` | Alignment of the popover relative to its anchor. |
+| attributes | `{ DatePickerContent?: NativeElementAttributes<"div", typeof DatePickerContent>; DatePickerCalendar?: NativeElementAttributes<"div", typeof DatePickerCalendar>; DatePickerInput?: NativeElementAttributes<"input", typeof DatePickerInput>; DatePickerNativeFallback?: NativeElementAttributes<"input", typeof DatePickerNativeFallback> }` | - | Native HTML attributes forwarded to the internal composed components. |
+| closeOnApply | `boolean` | `true` | Closes the DatePicker when the user applies a selection. |
+| defaultMonth | `Date` | `The current month` | Initial visible month. |
+| defaultOpen | `boolean` | `false` | Initial open state when uncontrolled. |
+| defaultValue | `Date \| { from: Date; to?: Date; }` | - | Initial date when uncontrolled. |
+| disabled | `boolean` | - | Disables interaction with the DatePicker. |
+| disabledMatcher | `Date \| ((date: Date) => boolean) \| Date[] \| { from: Date; to?: Date; } \| { before: Date; } \| { after: Date; } \| { before: Date; after: Date; } \| { dayOfWeek: number \| number[]; } \| ((Date \| ((date: Date) => boolean) \| Date[] \| { from: Date; to?: Date; } \| { before: Date; } \| { after: Date; } \| { before: Date; after: Date; } \| { dayOfWeek: number \| number[]; } \| boolean)[]) \| boolean` | - | Matcher describing which dates are disabled. Accepts a single matcher or an array of matchers. |
+| dismissible | `boolean` | - | Renders a clear button when the input has a value. |
+| endMonth | `Date` | `Five years from today` | Latest month selectable in the calendar. |
+| excludeDisabledDates | `boolean` | `false` | Prevents the user from selecting a range that spans any disabled dates. |
+| format | `string` | `"M/d/yyyy"` | Date format string using `date-fns` syntax. Used for the displayed value and the default placeholder. Ignored when `formatFn` is provided. |
+| formatFn | `(date: Date) => string` | - | Custom formatter for displayed dates. Takes precedence over `format`; pair with an explicit `placeholder` so the expected shape stays clear. |
+| kind | `"single" \| "range"` | `"single"` | Selects single-date mode. |
+| max | `number` | - | Maximum number of nights allowed in the range. |
+| min | `number` | - | Minimum number of nights allowed in the range. |
+| modal | `boolean` | `false` | Renders the popup as a modal that blocks interaction with the rest of the page. |
+| month | `Date` | - | Controlled visible month. Pair with `onMonthChange`. |
+| onCloseAutoFocus | `(event: Event) => void` | - | Called when focus returns to the trigger after the popover closes. Call `event.preventDefault()` to skip the default focus restoration. |
+| onEscapeKeyDown | `(event: KeyboardEvent) => void` | - | Called when Escape is pressed inside the popover. Call `event.preventDefault()` to keep the popover open. |
+| onMonthChange | `(month: Date) => void` | - | Called when the visible month changes through navigation or dropdown selection. |
+| onOpenChange | `(open: boolean) => void` | - | Called when the open state changes. |
+| onValueChange | `(value: Date) => void \| (value: { from: Date; to?: Date; }) => void` | - | Called when the user commits a new date. |
+| open | `boolean` | - | Controlled open state. Pair with `onOpenChange`. |
+| placeholder | `string \| string \| { from?: string; to?: string; }` | - | Placeholder shown when the input is empty. |
+| portal | `boolean` | `false` | Renders the floating content in a React portal. |
+| portalContainer | `HTMLElement` | `document.body` | Container for portaled floating content when no nearer portal target exists. |
+| readOnly | `boolean` | - | Prevents the value from being changed while preserving form submission. |
+| renderDropdown | `(props: RenderDropdownParams) => React.JSX.Element` | - | Custom renderer for the month and year selection dropdowns. |
+| side | `"right" \| "bottom" \| "left" \| "top"` | `"bottom"` | Preferred side of the anchor to render against. Flips automatically when the popover would overflow the viewport. |
+| size | `"small" \| "medium" \| "large"` | `"medium"` | Overall height and typography of the shell. |
+| startMonth | `Date` | - | Earliest month selectable in the calendar. |
+| status | `"error" \| "success"` | - | Visual validation state applied to the input. |
+| timeZone | `string` | - | IANA time-zone name or UTC offset applied when parsing and formatting dates. |
+| value | `Date \| { from: Date; to?: Date; }` | - | Controlled selected date. Pair with `onValueChange`. |
+
+## Exports
+
+- Components: `DatePicker`, `DatePickerCalendar`, `DatePickerCalendarDropdown`, `DatePickerContent`, `DatePickerInput`, `DatePickerNativeFallback`, `DatePickerRangeTrigger`, `DatePickerRangeTriggerField`, `DatePickerRoot`, `DatePickerTrigger`
+- Utilities: `deriveNativeDateBounds`, `formatDate`, `formatDateForNativeInput`, `getCurrentDatePickerValue`, `getPlaceholderForField`, `getZonedTime`, `isDate`, `isDatePickerValueEqual`, `isDateRange`, `parseDate`, `parseDateFromFormData`, `parseDateRangeFromFormData`, `parseNativeDateInput`, `resolveNativeInputId`, `validateDateRangeConstraints`, `validateDateRangeNights`
+- Constants: `DEFAULT_DATE_PICKER_FORMAT`, `NATIVE_DATE_INPUT_FORMAT`
+- Types: `DatePickerCalendarDayProps`, `DatePickerCalendarDropdownProps`, `DatePickerCalendarProps`, `DatePickerContentProps`, `DatePickerInputProps`, `DatePickerNativeFallbackProps`, `DatePickerRangeTriggerFieldProps`, `DatePickerRangeTriggerProps`, `DatePickerRootProps`, `DatePickerTriggerProps`, `DatePickerType`, `DatePickerValue`, `DateRangeNightsValidationError`, `PlaceholderForType`, `RangeDatePickerProps`, `SingleDatePickerProps`, `ValueForType`
+
+## See also
+
+- [FormField](./FormField.md)
diff --git a/.agents/skills/kaizen-ui/references/components/Divider.md b/.agents/skills/kaizen-ui/references/components/Divider.md
new file mode 100644
index 000000000..c2d8c4d0f
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/Divider.md
@@ -0,0 +1,74 @@
+# Divider
+
+A horizontal or vertical line that visually separates groups of content. Use to create breaks between sections, list items, or toolbar regions where a heading or whitespace alone would be too subtle.
+
+## Notes
+
+- Prefer spacing or a section heading over a Divider inside cards and between form sections — lines there read as visual noise. Reserve Divider for breaks between distinct regions (sidebar sections, list groups, panel areas). For form sections, replace dividers with a `Text kind="title/sm"` heading and spacing.
+
+- Use `orientation="vertical"` only for toolbars or side-by-side layouts; default to horizontal.
+
+- Default to `width="small"`; reserve `"medium"` for more visual weight when separating major page sections.
+
+## Examples
+
+### Basic Divider
+
+```tsx
+
+  
+  
+  
+
+```
+
+### With Text
+
+```tsx
+
+  
+  Text
+  
+
+```
+
+### Vertical Divider
+
+```tsx
+
+```
+
+### Composed
+
+```tsx
+
+  
+
+```
+
+## Props
+
+Spacing values use the [spacing scale](../design-tokens/SPACING_SCALE.md).
+
+| Prop | Type | Default | Description |
+| --- | --- | --- | --- |
+| asChild | `boolean` | - | Render-as-child slot (from Radix Primitive) |
+| attributes | `{ DividerRoot?: NativeElementAttributes<"div", typeof DividerRoot>; DividerElement?: NativeElementAttributes<"div", typeof DividerElement> }` | - | Native HTML attributes forwarded to the internal composed components. |
+| orientation | `"horizontal" \| "vertical"` | `"horizontal"` | Axis the separator runs along. |
+| padding | `SpacingScaleUnion` | - | Sets padding. |
+| paddingBottom | `SpacingScaleUnion` | - | Sets bottom padding. |
+| paddingLeft | `SpacingScaleUnion` | - | Sets left padding. |
+| paddingRight | `SpacingScaleUnion` | - | Sets right padding. |
+| paddingTop | `SpacingScaleUnion` | - | Sets top padding. |
+| paddingX | `SpacingScaleUnion` | - | Sets horizontal padding. |
+| paddingY | `SpacingScaleUnion` | - | Sets vertical padding. |
+| width | `"small" \| "medium" \| "large"` | `"small"` | Thickness of the separator line. Step up to `"medium"` only when separating major page sections that need extra visual weight |
+
+## Exports
+
+- Components: `Divider`, `DividerElement`, `DividerRoot`
+- Types: `DividerElementProps`, `DividerProps`, `DividerRootProps`
+
+## See also
+
+- [Stack](./Stack.md)
diff --git a/.agents/skills/kaizen-ui/references/components/Dropdown.md b/.agents/skills/kaizen-ui/references/components/Dropdown.md
new file mode 100644
index 000000000..99c197a21
--- /dev/null
+++ b/.agents/skills/kaizen-ui/references/components/Dropdown.md
@@ -0,0 +1,213 @@
+# Dropdown
+
+A button that opens a menu of actions, links, or selectable options. Use for contextual actions, filters, and overflow menus.
+
+## Notes
+
+- Dropdown is for triggering actions — it is not a value selector.
+
+- Dropdown triggers actions; it does not maintain a stored value. If the user's choice is submitted with a form or persisted to state, use Select (or Combobox for searchable selection) instead.
+
+- Positioning defaults: `side="bottom"` is standard; use `"top"` only when the trigger is near the viewport bottom. `align="start"` aligns to the leading edge.
+
+- Order items by frequency of use, not alphabetically — applies to all dropdowns, including unsectioned ones.
+
+- Pair `danger` items with a confirmation step (Modal or inline) so destructive actions are not committed by a single click.
+
+- Limit sub-menus to one level. If actions need deeper hierarchy, flatten or reach for a different UI pattern.
+
+- Use sections only when there are roughly 6+ items with natural groupings; order items and sections by frequency of use, not alphabetically.
+
+- Radio-in-Dropdown is for mutually exclusive options inside an action menu (e.g. sort direction). It is not a replacement for Select in forms.
+
+- Pass `renderLink` for navigation items so framework routing stays client-side.
+
+- When the dropdown must work from server-rendered HTML or open before hydration, leave `open` unset so `DropdownTrigger` keeps its native `popovertarget`; `onOpenChange` can still observe state changes.
+
+- To control an uncontrolled dropdown programmatically without losing native trigger behavior, attach a ref to `DropdownContent` and call `showPopover()` or `hidePopover()` on the content element.
+
+- `DropdownTrigger` always inject a chevron into the trigger button — including when you use `asChild` to wrap your own `