From 44f50ea09aa59fabdbf55cdc191ca878326f3601 Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Mon, 19 Oct 2026 20:50:20 -0500 Subject: [PATCH 1/5] feat(evaluation): add notebook-driven Brev VANTAGE launchable --- .../cosmos3/reasoner/vlmevalkit/.dockerignore | 15 + .../cosmos3/reasoner/vlmevalkit/README.md | 7 + .../reasoner/vlmevalkit/launchable/.gitignore | 3 + .../reasoner/vlmevalkit/launchable/Dockerfile | 35 ++ .../launchable/Evaluate_Cosmos3.ipynb | 289 +++++++++++++ .../reasoner/vlmevalkit/launchable/README.md | 163 +++++++ .../vlmevalkit/launchable/compose.yaml | 81 ++++ .../vlmevalkit/launchable/jupyter_config.py | 21 + .../vlmevalkit/launchable/launchable.py | 281 ++++++++++++ .../launchable/public-harness.patch | 43 ++ .../vlmevalkit/launchable/public_eval.py | 407 ++++++++++++++++++ .../reasoner/vlmevalkit/launchable/setup.sh | 32 ++ .../vlmevalkit/launchable/test_launchable.py | 77 ++++ 13 files changed, 1454 insertions(+) create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/public-harness.patch create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore b/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore new file mode 100644 index 00000000..de49d07f --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/.dockerignore @@ -0,0 +1,15 @@ +.git +.venv +__pycache__ +*.pyc +*.egg-info +**/__pycache__ +**/.ipynb_checkpoints +launchable/state +launchable/.env +.env +**/.env +**/env.sh +cosmos_eval_out +out +outputs diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/README.md index 76d6514d..76264c99 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/README.md +++ b/evaluation/cosmos3/reasoner/vlmevalkit/README.md @@ -45,6 +45,13 @@ the inputs and reads the outputs. ## 1. Environment setup +For a Brev VM with a notebook-driven Docker deployment, use the +[Cosmos3 VANTAGE launchable](launchable/README.md). Its default **public** path +uses the dataset-linked VANTAGE harness for public data preparation and official +submission generation. Its separate **internal** path retains this NVIDIA fork +and the configs below; the two paths do not share prepared dataset roots. + + Use a clean virtual environment with **Python 3.10+**, run from the **repo root** (the directory that holds this README): diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore new file mode 100644 index 00000000..dd26ad7d --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/.gitignore @@ -0,0 +1,3 @@ +state/ +.env +.ipynb_checkpoints/ diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile new file mode 100644 index 00000000..34cac381 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Dockerfile @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +FROM python:3.11.14-slim-bookworm@sha256:65a93d69fa75478d554f4ad27c85c1e69fa184956261b4301ebaf6dbb0a3543d +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ffmpeg git libgl1 libglib2.0-0 libcairo2 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /opt/vlmevalkit +COPY requirements.txt setup.py setup.cfg README-vlmevalkit.md ./ +# Evaluation is API-based; keep its PyTorch stack off the serving GPU. +RUN pip install --no-cache-dir torch==2.8.0 torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu \ + && pip install --no-cache-dir -r requirements.txt einops accelerate 'setuptools<81' evaluate jupyterlab==4.4.10 + +COPY launchable/public-harness.patch /tmp/public-harness.patch +# Keep the public release and internal NVIDIA fork in separate Python environments. +RUN git init /opt/vantage-public \ + && git -C /opt/vantage-public remote add origin https://github.com/Clemson-Capstone/VANTAGE-Bench.git \ + && git -C /opt/vantage-public fetch --depth 1 origin ec1ebdebb955332a9d7fdd9e98f39627b2822ff5 \ + && git -C /opt/vantage-public checkout --detach FETCH_HEAD \ + && git -C /opt/vantage-public apply /tmp/public-harness.patch \ + && python -m venv /opt/vantage-public/.venv \ + && /opt/vantage-public/.venv/bin/pip install --no-cache-dir torch==2.8.0 torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu \ + && /opt/vantage-public/.venv/bin/pip install --no-cache-dir -r /opt/vantage-public/requirements.txt \ + 'transformers==4.57.1' 'qwen-vl-utils==0.0.14' 'setuptools<81' evaluate gdown \ + && cd /opt/vantage-public \ + && .venv/bin/pip install --no-cache-dir --no-deps -e . \ + && .venv/bin/python -c 'import vlmeval' \ + && .venv/bin/pip freeze > /opt/public-evaluation-packages.txt +COPY . . +RUN pip install --no-cache-dir --no-deps -e . \ + && python -c 'import vlmeval' \ + && pip freeze > /opt/evaluation-packages.txt +ARG COSMOS_COMMIT=unknown +ENV COSMOS_COMMIT=${COSMOS_COMMIT} PYTHONUNBUFFERED=1 HF_HOME=/cache/huggingface +WORKDIR /opt/vlmevalkit/launchable +CMD ["python", "jupyter_config.py"] diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb new file mode 100644 index 00000000..331f9dcd --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -0,0 +1,289 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "welcome", + "metadata": {}, + "source": [ + "# Cosmos3 · VANTAGE evaluation\n", + "\n", + "**Prepare → connect → five real predictions → full submission**\n", + "\n", + "This notebook is a thin front end to `launchable.py`, using the official public VANTAGE harness by default. The full eight-task run is **opt-in**: **Run All runs the five-question smoke only** while `RUN_FULL = False`.\n", + "\n", + "**Before opening:** run `bash setup.sh` on the Brev VM from the launchable directory. Use the authenticated Brev link to port 8888. Docker setup/lifecycle commands run on the VM host, not here; this evaluator has no Docker socket. See [README](README.md) for launch fields and credentials.\n", + "\n", + "**Validation boundary:** GPU serving, runtime, end-to-end full evaluation, portal acceptance, and reference-score parity are not yet GPU-verified. Five predictions are not a leaderboard score." + ] + }, + { + "cell_type": "markdown", + "id": "configure-help", + "metadata": {}, + "source": [ + "## 1 · Configure a fresh run\n", + "\n", + "`HF_TOKEN` normally arrives from the Brev launch secret. A hidden prompt appears only if it is absent; the value stays in the process environment, never notebook source/output. Public mode does **not** need internal AWS/NGC/dataset-service or API judge keys. Existing endpoint/model environment settings are preserved.\n", + "\n", + "Use a new timestamped directory for each run. Re-running this cell creates new output paths; do not mix smoke and full outputs. Change `SOURCE` to `internal` only after configuring the host AWS mount described at the end." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "configure", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import subprocess\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "from getpass import getpass\n", + "from pathlib import Path\n", + "from IPython.display import FileLink, display\n", + "\n", + "SOURCE = 'public'\n", + "RUN_FULL = False # Explicitly opt in before running section 6.\n", + "LAUNCHABLE = Path('/opt/vlmevalkit/launchable/launchable.py')\n", + "CLI = [sys.executable, str(LAUNCHABLE)]\n", + "STAMP = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')\n", + "SMOKE_WORK = Path('/results') / f'{SOURCE}-smoke-{STAMP}'\n", + "FULL_WORK = Path('/results') / f'public-full-{STAMP}'\n", + "\n", + "if not os.environ.get('HF_TOKEN'):\n", + " os.environ['HF_TOKEN'] = getpass('Hugging Face token (hidden): ')\n", + "os.environ.setdefault('COSMOS_API_BASE', 'http://cosmos-endpoint:8080/v1/chat/completions')\n", + "# Do not overwrite COSMOS_MODEL: deployment/BYOM configuration selects it.\n", + "print(f'Source: {SOURCE}; smoke output: {SMOKE_WORK}; full run enabled: {RUN_FULL}')" + ] + }, + { + "cell_type": "markdown", + "id": "byom-help", + "metadata": {}, + "source": [ + "### Optional · Bring your own model\n", + "\n", + "Prefer setting `COSMOS_API_BASE`, `COSMOS_MODEL`, and an optional secret `COSMOS_API_KEY` at **initial deployment**: setup then skips the local GPU endpoint. For a session-only override, set `USE_BYOM = True` below **before** the health check. The URL must end in `/v1/chat/completions` and be reachable from the evaluator container. The service must accept the harness's image/video requests. Do not put credentials in the URL.\n", + "\n", + "Changing these settings here does not stop an already-running local GPU container. Leave this cell disabled to preserve all inherited settings. A blank hidden key prompt preserves the inherited key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "byom", + "metadata": {}, + "outputs": [], + "source": [ + "USE_BYOM = False\n", + "if USE_BYOM:\n", + " os.environ['COSMOS_API_BASE'] = input('Full chat/completions URL (no embedded credentials): ').strip()\n", + " os.environ['COSMOS_MODEL'] = input('Exact served model ID: ').strip()\n", + " byom_key = getpass('Endpoint key (hidden; blank preserves inherited key): ')\n", + " if byom_key:\n", + " os.environ['COSMOS_API_KEY'] = byom_key\n", + " del byom_key" + ] + }, + { + "cell_type": "markdown", + "id": "prepare-help", + "metadata": {}, + "source": [ + "## 2 · Prepare VQA with the selected harness\n", + "\n", + "Public mode uses the official public preparation path and pinned HF revision. VQA-only preparation downloads the VQA subtree rather than the entire benchmark; it still needs real media and can take time. Internal mode instead uses the authorized internal S3/cache constructor. Public and internal data are isolated under `/data/public/LMUData` and `/data/internal/LMUData`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "prepare-vqa", + "metadata": {}, + "outputs": [], + "source": [ + "subprocess.run(CLI + ['prepare', '--source', SOURCE, '--benchmarks', 'VANTAGE_VQA'], check=True)" + ] + }, + { + "cell_type": "markdown", + "id": "endpoint-help", + "metadata": {}, + "source": [ + "## 3 · Wait for the endpoint\n", + "\n", + "Wait up to 30 minutes for `/v1/models` and verify the requested model is served. This checks readiness, not GPU compatibility or benchmark quality. Failures propagate instead of being suppressed. The local model API is available only on the Compose network; do not expose it publicly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "endpoint-check", + "metadata": {}, + "outputs": [], + "source": [ + "subprocess.run(CLI + ['check', '--wait', '1800'], check=True)" + ] + }, + { + "cell_type": "markdown", + "id": "smoke-help", + "metadata": {}, + "source": [ + "## 4 · Infer five real VQA questions\n", + "\n", + "The helper applies a deterministic five-row VQA selection (`limit=5`, `random_state=0`) and runs inference only. It checks actual prediction artifacts because the native harness can catch exceptions and exit zero without producing usable results. This is an integration smoke, **not** a full benchmark, local score, or valid submission." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "smoke-run", + "metadata": {}, + "outputs": [], + "source": [ + "subprocess.run(CLI + ['run', '--source', SOURCE, '--benchmarks', 'VANTAGE_VQA',\n", + " '--mode', 'infer', '--smoke', '--work-dir', str(SMOKE_WORK)], check=True)" + ] + }, + { + "cell_type": "markdown", + "id": "results-help", + "metadata": {}, + "source": [ + "## 5 · Inspect smoke results\n", + "\n", + "Inspect prediction paths and run evidence. Native outputs can be TSV rather than XLSX. Public labels are evaluated by the official server; no local public leaderboard score is claimed. Use Jupyter's `/results` browser for logs, generated configs, and provenance; logs may contain dataset/model content, so share carefully." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "smoke-results", + "metadata": {}, + "outputs": [], + "source": [ + "subprocess.run(CLI + ['results', '--source', SOURCE, '--work-dir', str(SMOKE_WORK)], check=True)" + ] + }, + { + "cell_type": "markdown", + "id": "full-help", + "metadata": {}, + "source": [ + "## 6 · Opt in to the full eight-task public run\n", + "\n", + "**Disabled by default.** Set `RUN_FULL = True` in the configure cell, then run this section. Use `SOURCE = 'public'`. The following cells otherwise do no preparation, inference, or export. Full preparation needs substantial disk/network time: roughly 42 GB public data plus roughly 16 GB SOT source data, in addition to weights, containers, and extraction/cache headroom. SOT requires the official SmartSpaces license/access prerequisites.\n", + "\n", + "All eight tasks run: VQA, EventVerification, 2DGrounding, 2DPointing, Astro2D, Temporal, DVC, and SOT. No runtime or score-parity promise is made. The public harness, dataset classes, prompts, and ID emission differ from the internal fork even where sampling/model parameters are preserved.\n", + "\n", + "This run requests **inference**, then the official submission packager. Public `--mode all` also uses inference because public ground truth is held out; it does not produce a local leaderboard score. Never reuse the five-question directory for this run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "full-prepare", + "metadata": {}, + "outputs": [], + "source": [ + "if RUN_FULL:\n", + " if SOURCE != 'public':\n", + " raise ValueError('This full-submission section requires SOURCE = public.')\n", + " subprocess.run(CLI + ['prepare', '--source', 'public', '--benchmarks', 'all'], check=True)\n", + "else:\n", + " print('Full public run disabled; set RUN_FULL = True to opt in.')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "full-run", + "metadata": {}, + "outputs": [], + "source": [ + "if RUN_FULL:\n", + " if SOURCE != 'public':\n", + " raise ValueError('This full-submission section requires SOURCE = public.')\n", + " subprocess.run(CLI + ['run', '--source', 'public', '--benchmarks', 'all',\n", + " '--mode', 'infer', '--work-dir', str(FULL_WORK)], check=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "full-results", + "metadata": {}, + "outputs": [], + "source": [ + "if RUN_FULL:\n", + " subprocess.run(CLI + ['results', '--source', 'public', '--work-dir', str(FULL_WORK)], check=True)" + ] + }, + { + "cell_type": "markdown", + "id": "export-help", + "metadata": {}, + "source": [ + "## 7 · Export and download the official submission\n", + "\n", + "The public helper validates complete task coverage and uses the official packager, not an invented archive schema. Pillars require complete task sets: semantic = VQA + EventVerification; spatial = Grounding + Pointing + Astro2D; temporal = Temporal + DVC; tracking = SOT. This notebook's submission path runs **all eight**, not a partial pillar or smoke.\n", + "\n", + "Download the generated artifact using the authenticated Jupyter link below, then follow [official submission instructions](https://github.com/Clemson-Capstone/VANTAGE-Bench/blob/ec1ebdebb955332a9d7fdd9e98f39627b2822ff5/SUBMISSION.md) and submit at [vantage-bench.org/submit](https://vantage-bench.org/submit). Server scoring may take up to 24 hours. Nothing is uploaded automatically. Clear notebook outputs before sharing; do not redistribute restricted dataset media or credentials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "full-export", + "metadata": {}, + "outputs": [], + "source": [ + "if RUN_FULL:\n", + " subprocess.run(CLI + ['export', '--source', 'public', '--work-dir', str(FULL_WORK)], check=True)\n", + " archive = FULL_WORK.parent / f'{FULL_WORK.name}-submission.tar.gz'\n", + " display(FileLink(str(archive), url_prefix='/files'))" + ] + }, + { + "cell_type": "markdown", + "id": "internal-help", + "metadata": {}, + "source": [ + "## Optional · Authorized internal evaluation\n", + "\n", + "Public mode needs only the HF token. For internal mode, configure **at initial deployment** an existing host `AWS_CREDENTIALS_DIR` containing AWS `config` and `credentials` with the authorized `team-cosmos` profile (or securely populate the default `state/credentials/aws` before setup). Compose mounts it read-only under `/credentials/aws`, using `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. Credentials must remain valid during preparation. A notebook variable cannot create or change a host bind mount. Never paste AWS values into this notebook.\n", + "\n", + "Then set `SOURCE = 'internal'` in section 1, keep `RUN_FULL = False`, and rerun configuration, VQA preparation, health check, smoke, and results. Those same commands pass `--source internal` to the NVIDIA harness. Internal data stays under `/data/internal/LMUData`; public data stays under `/data/public/LMUData`, with separate Python/import roots as well. Internal `export --source internal` creates an evidence bundle, **not** a public leaderboard submission. See the README for CLI examples.\n", + "\n", + "These eight VANTAGE tasks need no additional dataset-service, NGC, or API judge keys. TAR and other benchmark families are outside scope.\n", + "\n", + "### Stop/restart safely\n", + "Use the **VM terminal**, in the launchable directory: `docker compose --profile local stop` preserves results/caches. For existing local services, `docker compose --profile local start cosmos-endpoint vantage-eval`; for BYOM, `docker compose start vantage-eval`. For changed configuration rerun setup with the intended deployment environment. See README for log commands. Stopping containers does not stop VM billing." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md new file mode 100644 index 00000000..698d8f3c --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -0,0 +1,163 @@ +# Cosmos3 VANTAGE evaluation on Brev + +A Jupyter front end for preparing data, checking an OpenAI-compatible video endpoint, running five real VQA questions, and exporting a full public VANTAGE submission. Notebook cells call `launchable.py`; dataset preparation, inference, and submission conversion stay in the selected upstream harness. + +**Default: public data and the official public harness. Only `HF_TOKEN` is required as a launch secret.** No internal AWS, NGC, NVIDIA dataset-service, or API judge credentials are required for the eight VANTAGE tasks. A separately authenticated BYOM endpoint may require its own `COSMOS_API_KEY`. + +Create a **read** token at [Hugging Face token settings](https://huggingface.co/settings/tokens) and supply it as Brev's secret `HF_TOKEN` launch parameter. Full tracking also requires accepting the [PhysicalAI-SmartSpaces access terms](https://huggingface.co/datasets/nvidia/PhysicalAI-SmartSpaces). No internal NVIDIA key is needed for the public workflow. + +This directory defines a launchable; it does not provision a VM or publish a Brev launchable. GPU serving, end-to-end GPU evaluation, runtime, and leaderboard parity are **not yet GPU-verified**. First successful predictions are not a benchmark score. Do not assume a sub-hour run or reference-score reproduction. + +## Create the Brev launchable + +Use these deployment fields in Brev: + +| Field | Value | +| --- | --- | +| Environment | **VM mode**, Linux x86-64, Docker Engine with Compose v2 and NVIDIA Container Toolkit for local serving | +| Deployment source | This Cosmos repository checkout: `https://github.com/NVIDIA/cosmos.git`, on the branch/commit **containing this launchable**; record the exact deployment commit | +| Setup script command | `bash evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh` from the Cosmos checkout root | +| Managed Jupyter | **Disabled**; Compose starts Jupyter itself | +| Secret environment variable | `HF_TOKEN`, with access to the model/data and accepted applicable licenses | +| Application access | Authenticated/secure Brev link to port **8888**; do not publish the model endpoint | +| Conservative starting hardware | One RTX PRO 6000 with **96 GB VRAM**, at least **256 GB disk**; sizing is an unvalidated starting point, not a compatibility guarantee | + +Do not select an upstream commit that lacks these files. Local edits are not automatically available to Brev: the deployment branch/commit must include them and be accessible to the VM. No branch is published by setup. + +If Brev presents a setup-script text box, paste the setup command above, not just the filename. It must execute from the cloned Cosmos repository root. + +Local serving defaults to `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` (overridable with `VLLM_IMAGE`) and `nvidia/Cosmos3-Nano` at revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`. Both evaluator and serving containers target `linux/amd64`. Confirm the serving image is available to your VM and compatible with its GPU/driver. Disk must accommodate model weights, image layers, downloaded data, extracted media, caches, and results. Public data is roughly 42 GB, with roughly 16 GB more for SOT; headroom and actual resource needs depend on workload. Full SOT preparation also requires access to the SmartSpaces source and acceptance of its license; an HF token alone does not grant that access. + +If running setup manually on the VM after injecting `HF_TOKEN` into its environment: + +```bash +cd /path/to/cosmos/evaluation/cosmos3/reasoner/vlmevalkit/launchable +bash setup.sh +``` + +Setup builds the evaluator, starts Jupyter, and starts the local GPU endpoint unless `COSMOS_API_BASE` was supplied at deployment. It persists a generated Jupyter token in a permission-restricted `.env`; it does not write HF/model credentials there. Use the same environment variables when recreating services. Never paste secrets into the notebook, source files, shell history, or screenshots. + +### Open Jupyter securely + +On the VM, from the launchable directory: + +```bash +docker compose exec vantage-eval jupyter server list +``` + +Treat the displayed login token as a secret. Open Brev's secure port-8888 link and authenticate with that token. The host binding is `127.0.0.1:8888`, not a public listener. If your access method requires a local tunnel, forward port 8888 through Brev/SSH rather than rebinding it to `0.0.0.0`. Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb` in Jupyter. Jupyter's file root is `/`, allowing authenticated downloads from `/results`. + +The evaluator has **no Docker socket**. Container lifecycle commands belong in the VM terminal, not notebook cells. The local model endpoint has no host port; only the Compose network can reach it. + +## Notebook workflow + +1. **Configure:** public source by default, a unique timestamped output directory, and inherited endpoint/model settings. A hidden `getpass` prompt is used only if `HF_TOKEN` was not inherited. Credentials remain in process memory; never save them in cells. +2. **Prepare VQA:** use the selected harness's official preparation. Downloads/preparation may be substantial even for a smoke run. +3. **Check endpoint:** wait for `/v1/models` and verify the configured served model. +4. **Smoke:** infer exactly five seeded `VANTAGE_VQA` questions. This tests actual data/media/model integration; it is not synthetic and is not a reference score. +5. **Inspect results:** the helper checks consumer-visible predictions, not just the native harness's exit status. The native harness can catch a dataset failure and still exit zero. +6. **Full public evaluation:** explicitly set `RUN_FULL = True` before running the full section. Its default is false, so **Run All does not start the full eight-task run**. It prepares all eight tasks and runs inference in a separate fresh directory. +7. **Export/download:** use the official public packager, then download the generated submission artifact via Jupyter. The notebook does not upload anything. Submit at [vantage-bench.org/submit](https://vantage-bench.org/submit); official scoring can take up to 24 hours. + +A full submission covers all eight tasks: `VANTAGE_VQA`, `VANTAGE_EventVerification`, `VANTAGE_2DGrounding`, `VANTAGE_2DPointing`, `VANTAGE_Astro2D`, `VANTAGE_Temporal`, `VANTAGE_DVC`, and `VANTAGE_SOT`. Official pillars require complete task sets: semantic = VQA + EventVerification; spatial = Grounding + Pointing + Astro2D; temporal = Temporal + DVC; tracking = SOT. A five-question smoke or incomplete pillar is not a valid full submission. Export validation rejects incomplete inputs rather than inventing a submission schema. + +### CLI equivalents + +Run inside the evaluator (`docker compose exec vantage-eval bash` from the VM terminal). Work directories must be new; do not reuse smoke output for a full run. + +```bash +cd /opt/vlmevalkit/launchable +python launchable.py prepare --source public --benchmarks VANTAGE_VQA +python launchable.py check --wait 1800 +python launchable.py run --source public --benchmarks VANTAGE_VQA --mode infer --smoke --work-dir /results/public-smoke-unique +python launchable.py results --source public --work-dir /results/public-smoke-unique + +python launchable.py prepare --source public --benchmarks all +python launchable.py run --source public --benchmarks all --mode infer --work-dir /results/public-full-unique +python launchable.py results --source public --work-dir /results/public-full-unique +python launchable.py export --source public --work-dir /results/public-full-unique +``` + +The normal public workflow is inference plus official submission export; leaderboard scoring is separate. Public `--mode all` maps to inference because public ground truth is held out; provenance records the requested and effective modes. Internal `--mode all` uses the NVIDIA harness's local evaluation path. `--dry-run` displays composed commands/configuration without running inference; it does not validate GPU/media compatibility. Predictions may be native TSV files, not spreadsheets. Treat predictions/media/logs as dataset-derived material and observe the source licenses when sharing. + +## Reproducibility and source boundaries + +| Component | Public mode | Internal mode | +| --- | --- | --- | +| Harness | [Clemson-Capstone/VANTAGE-Bench](https://github.com/Clemson-Capstone/VANTAGE-Bench), commit `ec1ebdebb955332a9d7fdd9e98f39627b2822ff5` | NVIDIA harness in this Cosmos checkout | +| Python/import root | `/opt/vantage-public/.venv/bin/python`, `/opt/vantage-public` | Base Python, `/opt/vlmevalkit` | +| Dataset | [nvidia/PhysicalAI-VANTAGE-Bench](https://huggingface.co/datasets/nvidia/PhysicalAI-VANTAGE-Bench), revision `ad5297f645ba90830478a4c6a72a3a7ab077a2f7`, plus official SOT source preparation | Official internal S3/cache preparation | +| Data root | `/data/public/LMUData` | `/data/internal/LMUData` | + +The checkouts, Python environments, and data roots are intentionally separate: an internal TSV/cache must never be silently reused as public data and produce misleading scores. Both share only the persistent HF cache at `/cache/huggingface`. Public configs preserve the committed NVIDIA FPS/frame limits, pixel limits, and model sampling parameters where the official public classes accept them. Dataset classes are translated to the public implementations, Astro2D uses the public `Astro2D` dataset name, and native prompts, dataset construction, ID emission, and server-side scoring differ from the internal harness. Review the generated configs and run provenance for actual settings. No cross-harness score parity is claimed. + +The image applies the tracked `public-harness.patch` to the pinned public source. It preserves TLS certificate verification and removes the `model_config.txt` diagnostic that exposed API credentials. It also aligns video serialization with this NVIDIA fork: lossless H.264 (`crf=0`), scalar FPS, no server-side resampling, and removal of the unsupported `do_resize` vLLM argument. Public dataset prompts and submission IDs remain upstream's; retain the patch with the source pin for reproduction. + +On the host, `LAUNCHABLE_STATE_DIR` defaults to `state/` beside `setup.sh`. It contains `cache/`, `data/public/LMUData`, `data/internal/LMUData`, `results/`, and an optional `credentials/aws/`. `/results` survives evaluator restarts. Data/source pins do not freeze an externally served model; record the endpoint's model revision as well as the deployment commit and generated configuration. + +## Bring your own model (BYOM) + +At **initial deployment**, supply `COSMOS_API_BASE` as a full URL ending in `/v1/chat/completions`, `COSMOS_MODEL` as the exact served model ID, and, only if required, `COSMOS_API_KEY` as a secret. Setup skips the local GPU profile when this URL is set. The endpoint must support the harness's OpenAI-compatible image/video requests and be reachable from inside the evaluator container; `localhost` inside the container is not the VM host. Keep the deployment's network authentication in place. + +The notebook preserves inherited endpoint/model settings. To change only the evaluation target for a session, use the optional BYOM cell **before** the health check. Prompt for any endpoint key rather than inserting it in notebook source. Do not print the environment or embed credentials in URLs. Changing notebook environment does not stop a previously launched local GPU container; use the VM lifecycle commands separately if needed. + +## Optional internal mode + +Public mode needs none of this. Internal mode requires authorized access to NVIDIA's `team-cosmos` S3 profile. **Configure its AWS credential-directory mount at initial deployment**, not by copying credentials into notebook cells. Set `AWS_CREDENTIALS_DIR` to an existing absolute host directory containing AWS `config` and `credentials` files with the `team-cosmos` profile, or populate the default `state/credentials/aws` securely before setup. Short-lived credentials/SSO sessions must be valid for the duration of preparation. Never commit these files. + +Compose mounts that directory read-only at `/credentials/aws` and sets `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. An empty default mount grants no access. No `NVDATASET_TENANTID`, `NGC_API_KEY`, or judge API key is needed for these eight VANTAGE tasks. If a different mount was not configured initially, reconfigure it on the VM and recreate the evaluator using setup; changing a notebook variable cannot create a host mount. + +Set notebook `SOURCE = "internal"` before preparation/smoke, or use: + +```bash +python launchable.py prepare --source internal --benchmarks VANTAGE_VQA +python launchable.py run --source internal --benchmarks VANTAGE_VQA --mode infer --smoke --work-dir /results/internal-smoke-unique +python launchable.py results --source internal --work-dir /results/internal-smoke-unique +python launchable.py export --source internal --work-dir /results/internal-smoke-unique + +# Full internal run with authorized ground truth: infer and score all eight. +python launchable.py prepare --source internal --benchmarks all +python launchable.py run --source internal --benchmarks all --mode all --work-dir /results/internal-full-unique +python launchable.py results --source internal --work-dir /results/internal-full-unique +``` + +The internal export is an evidence bundle, **not** the public leaderboard submission format. Keep public and internal run directories separate. TAR and other non-VANTAGE benchmark families are outside this launchable's scope; do not add their credentials or claim their coverage. + +## Logs, stop, restart + +Run from the launchable directory on the **VM host**: + +```bash +# Status and bounded recent logs (may contain model/data output; share carefully). +docker compose --profile local ps +docker compose logs --tail 100 vantage-eval +docker compose --profile local logs --tail 100 cosmos-endpoint + +# Stop containers without removing caches, results, or volumes. +docker compose --profile local stop + +# Restart existing containers with the same configuration. +# Local serving: +docker compose --profile local start cosmos-endpoint vantage-eval +# BYOM, instead: +docker compose start vantage-eval +``` + +For changed deployment environment/build configuration, rerun `bash setup.sh` with the intended secrets/environment rather than expecting `start` to recreate containers. Do not delete the state directory or use volume-removing commands to recover from a failed run. Check the run's log, fix the prerequisite, then use a new work directory. Stopping containers does not stop VM billing; use Brev's VM controls separately when appropriate. + +## Validation status + +These instructions are **not evidence of a GPU-validated launch**. No GPU hardware/runtime, full dataset preparation, full-eight inference, submission portal acceptance, or reference-score parity is certified here. Consult the accompanying implementation handoff for the specific local checks actually exercised. A successful image build or CPU dry run alone does not establish those claims. + +Local checks exercised: + +- Authenticated JupyterLab rendered this notebook; a Brev-style proxy Host header succeeded with a token and was rejected without one. +- Public preparation downloaded the pinned 2DPointing task and produced 1,005 input rows with linked media through the official preparation script. +- The `linux/amd64` evaluation image built successfully; its isolated public Python environment loaded all prepared pointing rows and resolved unique official submission IDs. +- A real repository video decoded and serialized as MP4 with normalized FPS and processor kwargs. This did not call a model. +- Public eight-task and internal VQA-smoke config generation, pinned public CLI argument parsing, Compose validation, and five artifact-safety regression tests passed. +- On an existing native Linux CPU host, a resource-limited notebook kernel invoked `launchable.py` and generated all eight public dry-run commands successfully. + +No GPU VM was provisioned, and no predictions, official scores, or portal acceptance are claimed by those checks. + +Development-platform caveat: the notebook kernel stalled under local ARM64 Docker's `linux/amd64` emulation, including in a clean upstream Jupyter image with no evaluation code. The same Jupyter stack started a kernel and executed code on native Linux x86-64. Use native Linux x86-64 for container-kernel validation, rather than downgrading dependencies to mask an emulation-specific failure. diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml new file mode 100644 index 00000000..aff62c76 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +name: cosmos3-vantage +services: + cosmos-endpoint: + profiles: [local] + platform: linux/amd64 + image: ${VLLM_IMAGE:-vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55} + restart: unless-stopped + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + ipc: host + environment: + HF_TOKEN: ${HF_TOKEN:-} + HF_HOME: /cache/huggingface + volumes: + - ${LAUNCHABLE_STATE_DIR:-./state}/cache:/cache + - ${LAUNCHABLE_STATE_DIR:-./state}/data:/data:ro + # Deliberately no host port: only the evaluator can reach this API. + command: + - nvidia/Cosmos3-Nano + - --revision + - e59a53c25979a090fa8706c9acc0c254a6e89b92 + - --async-scheduling + - --allowed-local-media-path + - / + - --port + - '8080' + - --max-model-len + - '128000' + - --tensor-parallel-size + - '1' + - --gpu-memory-utilization + - '0.9' + - --enable-chunked-prefill + - --mm-processor-cache-gb + - '0' + - --mm-encoder-tp-mode + - data + - --media-io-kwargs + - '{"video": {"num_frames": -1, "fps": -1}}' + healthcheck: + test: [CMD, python3, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/v1/models', timeout=5)"] + interval: 15s + timeout: 10s + retries: 120 + start_period: 60s + vantage-eval: + platform: linux/amd64 + build: + context: .. + dockerfile: launchable/Dockerfile + args: + COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} + restart: unless-stopped + ports: + - '127.0.0.1:8888:8888' + environment: + HF_TOKEN: ${HF_TOKEN:-} + COSMOS_API_BASE: ${COSMOS_API_BASE:-http://cosmos-endpoint:8080/v1/chat/completions} + COSMOS_MODEL: ${COSMOS_MODEL:-nvidia/Cosmos3-Nano} + COSMOS_API_KEY: ${COSMOS_API_KEY:-EMPTY} + JUPYTER_TOKEN: ${JUPYTER_TOKEN:?Run bash setup.sh to generate a Jupyter token} + LMUData: /data/public/LMUData + AWS_CONFIG_FILE: /credentials/aws/config + AWS_SHARED_CREDENTIALS_FILE: /credentials/aws/credentials + volumes: + - ${LAUNCHABLE_STATE_DIR:-./state}/cache:/cache + - ${LAUNCHABLE_STATE_DIR:-./state}/data:/data + - ${LAUNCHABLE_STATE_DIR:-./state}/results:/results + - ${AWS_CREDENTIALS_DIR:-./state/credentials/aws}:/credentials/aws:ro + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8888/login', timeout=5)"] + interval: 15s + timeout: 10s + retries: 20 diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py new file mode 100644 index 00000000..b0fd3a6e --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/jupyter_config.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Start Jupyter without placing the authentication token in process arguments.""" +import os + +from jupyter_server.serverapp import ServerApp +from traitlets.config import Config + +config = Config() +config.ServerApp.ip = "0.0.0.0" +config.ServerApp.port = 8888 +config.ServerApp.port_retries = 0 +config.ServerApp.open_browser = False +config.ServerApp.allow_root = True +# Brev's reverse proxy preserves a non-local Host header. Token auth remains on. +config.ServerApp.allow_remote_access = True +config.ServerApp.root_dir = "/" +config.ServerApp.default_url = "/lab/tree/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb" +config.ServerApp.jpserver_extensions = {"jupyterlab": True} +config.IdentityProvider.token = os.environ["JUPYTER_TOKEN"] +ServerApp.launch_instance(config=config) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py new file mode 100755 index 00000000..8617531f --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Notebook/CLI orchestration; evaluation and preprocessing remain VLMEvalKit's.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +KIT = ROOT / "cosmos_eval" +DATASET_REVISION = "ad5297f645ba90830478a4c6a72a3a7ab077a2f7" + + +def manifest(): + return json.loads((KIT / "manifest.json").read_text()) + + +def benchmarks(value): + available = [key.split("/", 1)[1] for key in manifest() if "/VANTAGE_" in key] + selected = available if value == "all" else value.split(",") + unknown = set(selected) - set(available) + if unknown: + raise ValueError(f"Unknown VANTAGE benchmark(s): {', '.join(sorted(unknown))}") + # run_all selects in manifest order; keep Astro2D's indexed output consistent. + return [bench for bench in available if bench in selected] + + +def endpoint(wait=0): + base = os.environ.get("COSMOS_API_BASE", "http://cosmos-endpoint:8080/v1/chat/completions") + if not base.endswith("/chat/completions"): + raise ValueError("COSMOS_API_BASE must be the complete /v1/chat/completions URL") + os.environ["COSMOS_API_BASE"] = base + url = base.removesuffix("/chat/completions") + "/models" + key = os.environ.get("COSMOS_API_KEY", "EMPTY") + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}"}) + deadline = time.monotonic() + wait + while True: + try: + with urllib.request.urlopen(request, timeout=10) as response: + models = json.load(response)["data"] + if not models: + raise ValueError("The endpoint returned no served models") + requested = os.environ.get("COSMOS_MODEL") + ids = [model["id"] for model in models] + if requested and requested not in ids: + raise ValueError(f"COSMOS_MODEL {requested!r} is not served; available: {ids}") + if not requested and len(ids) != 1: + raise ValueError(f"Set COSMOS_MODEL to one of {ids}") + os.environ["COSMOS_MODEL"] = requested or ids[0] + print(f"Endpoint ready; model: {os.environ['COSMOS_MODEL']}", flush=True) + return + except urllib.error.HTTPError as exc: + if exc.code not in (502, 503, 504): + raise RuntimeError(f"Model endpoint returned HTTP {exc.code}") from exc + if time.monotonic() >= deadline: + raise RuntimeError(f"Model endpoint not ready after {wait}s (HTTP {exc.code})") from exc + except (urllib.error.URLError, TimeoutError) as exc: + if time.monotonic() >= deadline: + raise RuntimeError(f"Model endpoint not reachable after {wait}s") from exc + time.sleep(min(5, max(0, deadline - time.monotonic()))) + + +def prepare(selected): + # Use the same config composition and dataset constructors as run.py. Do not + # transcode, re-sample, or invent a second dataset conversion convention. + sys.path.insert(0, str(ROOT)) + sys.path.insert(0, str(KIT)) + import run_all + from run import build_dataset_from_config + + layer = run_all.load_model_layer("cosmos") + for bench in selected: + cfg = run_all.build_config(layer, "metropolis", bench, render=False) + dataset = build_dataset_from_config(cfg["data"], bench) + if dataset is None or dataset.data.empty: + raise RuntimeError(f"{bench}: no prepared dataset rows") + print(f"{bench}: prepared {len(dataset.data)} questions", flush=True) + + +def run_command(command, **kwargs): + subprocess.run(command, cwd=ROOT, check=True, **kwargs) + + +def prediction_files(work): + # Only dated run directories, not the symlinks run.py makes at model level. + return sorted(p for p in work.rglob("cosmos_*") + if p.parent.name.startswith("T") and p.suffix in {".tsv", ".xlsx", ".json"} + and p.stem in {f"cosmos_{bench}" for bench in benchmarks("all")}) + + +def verify_predictions(work, bench): + import pandas as pd + + files = [p for p in prediction_files(work) if p.stem == f"cosmos_{bench}"] + if not files: + # run.py catches dataset/model exceptions and may exit 0 without output. + raise RuntimeError(f"{bench}: no predictions were written; inspect the run log") + path = files[-1] + if path.suffix == ".tsv": + frame = pd.read_csv(path, sep="\t") + elif path.suffix == ".json": + frame = pd.read_json(path) + else: + frame = pd.read_excel(path) + if "prediction" not in frame or frame.empty: + raise RuntimeError(f"{bench}: empty or invalid prediction output") + failed = frame["prediction"].isna() | frame["prediction"].astype(str).str.contains( + "Failed to obtain answer via API.", regex=False + ) + if failed.any(): + raise RuntimeError(f"{bench}: {int(failed.sum())} failed/empty predictions; inspect the run log") + print(f"{bench}: {len(frame)} predictions verified", flush=True) + + +def run(selected, work, mode, smoke, dry_run, source): + if smoke and selected != ["VANTAGE_VQA"]: + raise ValueError("--smoke requires --benchmarks VANTAGE_VQA") + if work.exists() and not dry_run: + raise ValueError(f"Use a fresh --work-dir (already exists: {work}) to avoid stale results") + if not dry_run: + endpoint(wait=1800) + if source == "public": + import public_eval + + public_eval.run(selected, work, mode, smoke, dry_run) + return + configs = work / "_configs" + run_command([sys.executable, str(KIT / "run_all.py"), "--benchmarks", ",".join(selected), + "--export-configs", str(configs)]) + if mode == "all" and not smoke: + command = [sys.executable, str(KIT / "run_all.py"), "--benchmarks", ",".join(selected), + "--concurrency", "1", "--work-dir", str(work)] + if dry_run: + command.append("--dry-run") + run_command(command) + if not dry_run: + for bench in selected: + verify_predictions(work, bench) + record_run(work, selected, mode, smoke) + return + # run_all.py only implements inference+scoring. Use its official composed + # configs with run.py's native infer mode for public inputs / a short smoke. + entries = manifest() + for index, bench in enumerate(selected): + path = configs / f"{bench}.json" + cfg = json.loads(path.read_text()) + if smoke: + cfg["data"][bench].update(limit=5, random_state=0) + for model in cfg["model"].values(): + model["api_base"] = os.environ.get("COSMOS_API_BASE", "${COSMOS_API_BASE}") + model["model"] = os.environ.get("COSMOS_MODEL", "${COSMOS_MODEL}") + path.write_text(json.dumps(cfg, indent=2) + "\n") + # Astro2D rewrites its dataset name throughout paths: use name-free dirs. + out = work / f"bench_{index:02d}" + flags = entries[f"metropolis/{bench}"]["run"] + command = [sys.executable, str(ROOT / "run.py"), "--config", str(path), + "--mode", mode, "--work-dir", str(out), + "--api-nproc", str(flags["api_nproc"]), + "--judge-nproc", str(flags["judge_nproc"]), "--verbose", "--save-eval-results"] + if dry_run: + print(" ".join(command)) + continue + out.mkdir(parents=True) + with (out / "run.log").open("w") as log: + run_command(command, stdout=log, stderr=subprocess.STDOUT) + verify_predictions(out, bench) + if not dry_run: + record_run(work, selected, mode, smoke) + + +def record_run(work, selected, mode, smoke): + provenance = { + "cosmos_commit": os.environ.get("COSMOS_COMMIT", "unknown"), + "model": os.environ["COSMOS_MODEL"], "benchmarks": selected, + "source": "internal", "mode": mode, "smoke": smoke, + "work_dirs": { + bench: (f"_concat_{i:02d}" if bench == "VANTAGE_Astro2D" else bench) + if mode == "all" and not smoke else f"bench_{i:02d}" + for i, bench in enumerate(selected) + }, + } + (work / "run.json").write_text(json.dumps(provenance, indent=2) + "\n") + packages = Path("/opt/evaluation-packages.txt") + if packages.exists(): + shutil.copyfile(packages, work / "evaluation-packages.txt") + + +def results(work): + sys.path.insert(0, str(KIT)) + import parse_score + + metadata = json.loads((work / "run.json").read_text()) + for bench, directory in metadata["work_dirs"].items(): + report = parse_score.report(work_dir=str(work / directory), dataset_name=bench) + score = report["overall"] + print(f"{bench:26} {score if score is not None else 'not locally scored'}") + for path in prediction_files(work): + print(f"Predictions: {path}") + + +def export(work): + if not prediction_files(work): + raise ValueError("No predictions found; run inference before exporting") + archive = work.parent / f"{work.name}-predictions.tar.gz" + # Exclude logs and configs: they can contain endpoint URLs or model responses. + # This is an evidence bundle, NOT an undocumented leaderboard conversion. + with tarfile.open(archive, "w:gz") as bundle: + for path in prediction_files(work): + bundle.add(path, arcname=str(path.relative_to(work)), recursive=False) + for name in ("run.json", "evaluation-packages.txt"): + if (work / name).exists(): + bundle.add(work / name, arcname=name, recursive=False) + print(f"Prediction bundle: {archive}") + print("Internal evidence only: may contain private ground truth. Not a public leaderboard submission.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check", help="Wait for /v1/models and verify the requested model") + check.add_argument("--wait", type=int, default=1800) + prep = sub.add_parser("prepare", help="Prepare official public or internal datasets") + prep.add_argument("--source", choices=["public", "internal"], default="public") + prep.add_argument("--benchmarks", default="VANTAGE_VQA") + prep.add_argument("--revision", default=DATASET_REVISION) + runner = sub.add_parser("run", help="Run the selected public or NVIDIA-internal evaluation harness") + runner.add_argument("--benchmarks", default="VANTAGE_VQA") + runner.add_argument("--source", choices=["public", "internal"], default="public") + runner.add_argument("--mode", choices=["infer", "all"], default="infer") + runner.add_argument("--smoke", action="store_true", help="Five seeded VQA samples, not a reference score") + runner.add_argument("--dry-run", action="store_true") + runner.add_argument("--work-dir", type=Path, required=True) + for command in ("results", "export"): + child = sub.add_parser(command) + child.add_argument("--work-dir", type=Path, required=True) + child.add_argument("--source", choices=["public", "internal"], default="public") + args = parser.parse_args() + if hasattr(args, "source"): + data_root = Path(os.environ.get("LAUNCHABLE_DATA_ROOT", "/data")) + os.environ["LMUData"] = str(data_root / args.source / "LMUData") + if args.command in {"prepare", "run"} and not getattr(args, "dry_run", False): + Path(os.environ["LMUData"]).mkdir(parents=True, exist_ok=True) + if args.command == "check": + endpoint(args.wait) + elif args.command == "prepare": + if args.source == "public": + import public_eval + + public_eval.prepare(benchmarks(args.benchmarks), args.revision) + else: + prepare(benchmarks(args.benchmarks)) + elif args.command == "run": + run(benchmarks(args.benchmarks), args.work_dir.resolve(), args.mode, args.smoke, args.dry_run, args.source) + elif args.command == "results": + if args.source == "public": + import public_eval + + public_eval.results(args.work_dir.resolve()) + else: + results(args.work_dir.resolve()) + else: + if args.source == "public": + import public_eval + + public_eval.export(args.work_dir.resolve()) + else: + export(args.work_dir.resolve()) + + +if __name__ == "__main__": + main() diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public-harness.patch b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public-harness.patch new file mode 100644 index 00000000..792750df --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public-harness.patch @@ -0,0 +1,43 @@ +diff --git a/vlmeval/__init__.py b/vlmeval/__init__.py +--- a/vlmeval/__init__.py ++++ b/vlmeval/__init__.py +@@ -1,8 +1,4 @@ +-import ssl +-ssl._create_default_https_context = ssl._create_unverified_context +-# Temporarily bypass SSL certificate verification to download files from oss. +- + try: + import torch + except ImportError: + pass +diff --git a/run.py b/run.py +--- a/run.py ++++ b/run.py +@@ -301,9 +301,2 @@ +- if RANK == 0 and hasattr(model, "__dict__"): +- config_path = os.path.join(pred_root, "model_config.txt") +- with open(config_path, "w") as f: +- for k, v in model.__dict__.items(): +- f.write(f"{k}:\n{v}\n") +- f.write("-" * 40 + "\n") +- + for _, dataset_name in enumerate(args.data): + if WORLD_SIZE > 1: +diff --git a/vlmeval/api/cosmos_reason.py b/vlmeval/api/cosmos_reason.py +--- a/vlmeval/api/cosmos_reason.py ++++ b/vlmeval/api/cosmos_reason.py +@@ -25,3 +25,3 @@ +-def _tensor_video_to_base64(video_tensor, fps, crf=14): ++def _tensor_video_to_base64(video_tensor, fps, crf=0): + """Convert (T,C,H,W) RGB tensor to MP4 base64 using ffmpeg.""" + assert video_tensor.ndim == 4 and video_tensor.shape[1] == 3 +@@ -71,5 +71,7 @@ + video_url = f"data:video/mp4;base64,{video_base64}" + video_kwargs = copy.deepcopy(video_kwargs) +- video_kwargs.update(dict(do_sample_frames=False, do_resize=False)) +- del video_kwargs["fps"] ++ video_kwargs.setdefault("do_sample_frames", False) ++ if isinstance(video_kwargs.get("fps"), list): ++ video_kwargs["fps"] = video_kwargs["fps"][0] ++ video_kwargs.pop("do_resize", None) + return video_url, video_kwargs diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py new file mode 100644 index 00000000..32e4a778 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py @@ -0,0 +1,407 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Public VANTAGE orchestration, isolated from the NVIDIA harness's imports. + +All dataset/model execution, conversion, validation and packaging runs with the +pinned Clemson checkout's Python. This module itself needs only the stdlib. +Public annotations are inference inputs, never local scoring ground truth. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import shlex +import string +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +KIT = ROOT / "cosmos_eval" +HARNESS_REVISION = "ec1ebdebb955332a9d7fdd9e98f39627b2822ff5" +DATASET_REVISION = "ad5297f645ba90830478a4c6a72a3a7ab077a2f7" +DATASET_REPO = "nvidia/PhysicalAI-VANTAGE-Bench" +# NVIDIA benchmark -> (official preparation task, public dataset, public class, +# official submission packager key). +TASKS = { + "VANTAGE_2DGrounding": ("grounding", "VANTAGE_2DGrounding", "VANTAGE_2DGroundingDataset", "grounding"), + "VANTAGE_2DPointing": ("pointing", "VANTAGE_2DPointing", "VANTAGE_2DPointing", "pointing"), + "VANTAGE_Astro2D": ("astro2d", "Astro2D", "Astro2DDetectionDataset", "astro"), + "VANTAGE_DVC": ("dvc", "VANTAGE_DVC", "VANTAGE_DVC", "dvc"), + "VANTAGE_EventVerification": ("event_verification", "VANTAGE_EventVerification", "VANTAGE_EventVerification", "event_verification"), + "VANTAGE_SOT": ("sot", "VANTAGE_SOT", "VANTAGE_SOT", "sot"), + "VANTAGE_Temporal": ("temporal", "VANTAGE_Temporal", "VANTAGE_Temporal", "temporal"), + "VANTAGE_VQA": ("vqa", "VANTAGE_VQA", "VANTAGE_VQA", "vqa"), +} + + +def _harness(): + return Path(os.environ.get("PUBLIC_VLMEVAL_ROOT", "/opt/vantage-public")) + + +def _python(): + return os.environ.get("PUBLIC_VLMEVAL_PYTHON", str(_harness() / ".venv/bin/python")) + + +def _lmu(): + return Path(os.environ.get("LMUData", "/data/public/LMUData")) + + +def _env(): + env = os.environ.copy() + # Never import the NVIDIA checkout via inherited notebook/container settings. + env["PYTHONPATH"] = str(_harness()) + env["LMUData"] = str(_lmu()) + env["PRED_FORMAT"] = env.get("PRED_FORMAT") or "tsv" + # The public runner otherwise silently replaces the explicit --work-dir. + env.pop("MMEVAL_ROOT", None) + return env + + +def _check_harness(): + if not Path(_python()).is_file(): + raise RuntimeError(f"Public harness Python missing: {_python()}; rebuild the evaluation image") + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=_harness(), text=True, + ).strip() + if revision != HARNESS_REVISION: + raise RuntimeError(f"Public harness must be pinned to {HARNESS_REVISION}; found {revision}") + + +def _execute(command, *, log=None): + display = [str(part) for part in command] + if len(display) > 2 and display[1] == "-c": + display[2] = "" + print(shlex.join(display), flush=True) + if log is None: + subprocess.run(command, cwd=_harness(), env=_env(), check=True) + return + print(f"Harness stdout/stderr: {log}", flush=True) + with Path(log).open("w") as stream: + result = subprocess.run(command, cwd=_harness(), env=_env(), stdout=stream, + stderr=subprocess.STDOUT) + if result.returncode: + raise RuntimeError(f"Public harness exited {result.returncode}; see {log}") + + +def _write_json(path, value): + Path(path).write_text(json.dumps(value, indent=2) + "\n") + + +def _selected(selected): + if not selected or len(set(selected)) != len(selected) or set(selected) - TASKS.keys(): + raise ValueError("Select distinct supported VANTAGE benchmarks") + + +# Import the official script's task patterns instead of maintaining a second +# download/layout convention. Markers are required by --local-source validation. +_PREPARE = r''' +import csv, importlib.util, json, os, subprocess, sys +from pathlib import Path +from huggingface_hub import snapshot_download +root, lmu, revision, selected_json, repo = sys.argv[1:] +root, lmu = Path(root), Path(lmu) +selected = json.loads(selected_json) +spec = importlib.util.spec_from_file_location("vantage_public_prep", root / "scripts/run_lmudata.py") +prep = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = prep +spec.loader.exec_module(prep) +patterns = ["README.md", "LICENSE.md"] +for task in selected.values(): + patterns.extend(prep.TASK_CONFIG[task]["hf_patterns"]) +snapshot = snapshot_download(repo, repo_type="dataset", revision=revision, + allow_patterns=sorted(set(patterns)), token=os.environ.get("HF_TOKEN")) +state_path = lmu / "launchable-public.json" +state = json.loads(state_path.read_text()) if state_path.exists() else {"tasks": {}} +for bench in selected: + previous = state["tasks"].get(bench, {}).get("dataset_revision") + if previous and previous != revision: + raise RuntimeError("Changing a prepared task revision requires a fresh LAUNCHABLE_DATA_ROOT; " + "refusing to mix cached public media revisions") +for bench in selected: + state["tasks"].pop(bench, None) +state_path.write_text(json.dumps(state, indent=2) + "\n") +subprocess.run([sys.executable, str(root / "scripts/run_lmudata.py"), + "--local-source", snapshot, "--tasks", ",".join(selected.values()), + "--lmu-root", str(lmu), "--symlink", "--force"], check=True) +for bench, task in selected.items(): + target = lmu / "datasets" / prep.TASK_CONFIG[task]["lmu_name"] + if task == "sot": + source = prep._load_jsonl(Path(snapshot) / "data/tracking/sot_benchmark.jsonl") + names = {item["seq_id"] for item in source} + actual = {p.parent.name for p in target.glob("*/gt.json")} + if actual != names: + raise RuntimeError(f"SOT preparation incomplete: {len(names - actual)} missing and " + f"{len(actual - names)} unexpected sequences") + count = len(names) + elif task == "astro2d": + count = sum(1 for p in (target / "images").iterdir() if p.is_file()) + elif task == "grounding": + count = len(json.loads((target / "annotations.json").read_text())) + else: + with (target / prep.TASK_CONFIG[task]["index_file"]).open() as stream: + count = sum(1 for _ in csv.DictReader(stream, delimiter="\t")) + if not count: + raise RuntimeError(f"{bench}: no prepared rows") + state["tasks"][bench] = {"dataset_revision": revision, "snapshot": snapshot, "expected_rows": count} +state.update(dataset_repo=repo, harness_revision="ec1ebdebb955332a9d7fdd9e98f39627b2822ff5") +state_path.write_text(json.dumps(state, indent=2) + "\n") +''' + + +def prepare(selected: list[str], revision: str): + """Materialize only selected task subtrees, using official preparation.""" + _selected(selected) + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Use an immutable 40-character HF dataset commit SHA") + _check_harness() + _lmu().mkdir(parents=True, exist_ok=True) + _execute([_python(), "-c", _PREPARE, str(_harness()), str(_lmu()), revision, + json.dumps({bench: TASKS[bench][0] for bench in selected}), DATASET_REPO]) + print(f"Public inputs prepared at {_lmu()} (only selected task subtrees downloaded).") + print("SOT additionally requires accepted nvidia/PhysicalAI-SmartSpaces license access; " + "grounding uses the official VisDrone downloader.") + + +def _render(value): + if isinstance(value, str): + return string.Template(value).safe_substitute(os.environ) + if isinstance(value, dict): + return {key: _render(item) for key, item in value.items()} + if isinstance(value, list): + return [_render(item) for item in value] + return value + + +def _config(bench, smoke): + layer = json.loads((KIT / "models/cosmos.json").read_text()) + data = json.loads((KIT / "data/metropolis" / f"{bench}.json").read_text()) + model = {"class": layer["class"], **layer["defaults"], **layer["benchmarks"].get(bench, {})} + data.update(dataset=TASKS[bench][1], **{"class": TASKS[bench][2]}) + # Public image constructors do not accept model_family; SOT does, and uses + # it for xyxy coordinate order. The public run.py filters unsupported kwargs. + if bench == "VANTAGE_SOT": + data["model_family"] = layer["model_family"] + if smoke: + data.update(limit=5, random_state=0) + return {"model": {layer["model_key"]: _render(model)}, "data": {TASKS[bench][1]: data}} + + +# Verify both native table and canonical emitter output: upstream dataset errors +# and canonical-emitter errors can otherwise leave a successful process status. +_VERIFY = r''' +import json, sys +from pathlib import Path +import pandas as pd +from run import build_dataset_from_config +from vlmeval.smp import load +sys.path.insert(0, str(Path.cwd() / "scripts")) +from package_submission import find_submission_files +config_path, out, dataset_name, task, smoke, report_path = sys.argv[1:] +out = Path(out) +cfg = json.loads(Path(config_path).read_text()) +dataset = build_dataset_from_config(cfg["data"], dataset_name) +expected = dataset.data +if expected.empty or (smoke == "1" and len(expected) != 5): + raise RuntimeError("Expected five smoke rows or a nonempty complete dataset") +state = json.loads((Path(__import__("os").environ["LMUData"]) / "launchable-public.json").read_text()) +prepared_bench = "VANTAGE_Astro2D" if dataset_name == "Astro2D" else dataset_name +if smoke != "1" and len(expected) != state["tasks"][prepared_bench]["expected_rows"]: + raise RuntimeError("Dataset row count changed since pinned preparation; refusing incomplete predictions") +source_tsv = Path(__import__("os").environ["LMUData"]) / "datasets" / dataset_name / (dataset_name + ".tsv") +if smoke != "1" and source_tsv.exists() and len(pd.read_csv(source_tsv, sep="\t")) != len(expected): + raise RuntimeError("Dataset dropped prepared rows (usually missing media); refusing incomplete predictions") +candidates = {p.resolve() for suffix in ("tsv", "xlsx", "json") + for p in out.rglob("cosmos_" + dataset_name + "." + suffix) if p.is_file()} +if len(candidates) != 1: + raise RuntimeError("Expected exactly one native prediction table, found: " + str(sorted(map(str, candidates)))) +prediction_path = candidates.pop() +frame = load(str(prediction_path)) +if not isinstance(frame, pd.DataFrame): + frame = pd.DataFrame(frame) +if "prediction" not in frame or len(frame) != len(expected): + raise RuntimeError("Native prediction count/schema does not match the selected dataset") +def bad(value): + text = str(value).strip() + return (value is None or text.lower() in ("", "nan", "none", "null", "") + or "failed to obtain answer" in text.lower()) +if frame["prediction"].map(bad).any(): + raise RuntimeError("Failed/null native predictions; inspect run.log") +if "index" not in frame or frame["index"].duplicated().any() or set(map(str, frame["index"])) != set(map(str, expected["index"])): + raise RuntimeError("Prediction indexes do not match selected dataset indexes") +found = find_submission_files(out) +if set(found) != {task}: + raise RuntimeError("Missing or unexpected official canonical submission files: " + str(found)) +canonical = found[task] +records = [json.loads(line) for line in canonical.read_text().splitlines() if line.strip()] +if len(records) != len(expected) or len({r["id"] for r in records}) != len(expected): + raise RuntimeError("Canonical submission count/unique IDs do not match predictions") +from vlmeval.dataset.utils.vantagebench.emit import _TASK_SPECS +canonical_ids = {_TASK_SPECS[task].id_resolver(row, dataset) for _, row in expected.iterrows()} +if None in canonical_ids or {r["id"] for r in records} != canonical_ids: + raise RuntimeError("Canonical IDs differ from the official emitter's selected dataset IDs") +for record in records: + answers = [c.get("value") for c in record.get("conversations", []) if c.get("from") == "assistant"] + if not answers or any(bad(answer) for answer in answers): + raise RuntimeError("Canonical submission contains failed/null predictions") +Path(report_path).write_text(json.dumps({"rows": len(frame), "prediction": str(prediction_path), + "submission": str(canonical.resolve())}, indent=2) + "\n") +print(f"Verified {dataset_name}: {len(frame)} native predictions and canonical records") +''' + + +def _prepared(selected): + state_path = _lmu() / "launchable-public.json" + if not state_path.is_file(): + raise ValueError("Prepare the selected public tasks before running (no pinned preparation metadata found)") + state = json.loads(state_path.read_text()) + missing = set(selected) - state.get("tasks", {}).keys() + if missing: + raise ValueError(f"Public tasks not prepared: {', '.join(sorted(missing))}") + if state.get("harness_revision") != HARNESS_REVISION: + raise ValueError("Public preparation used a different harness revision; prepare again") + return {bench: state["tasks"][bench]["dataset_revision"] for bench in selected} + + +def run(selected: list[str], work: Path, mode: str, smoke: bool, dry_run: bool): + """Run official public inference; never score missing public ground truth.""" + _selected(selected) + work = Path(work).resolve() + if smoke and (selected != ["VANTAGE_VQA"] or mode != "infer"): + raise ValueError("Public smoke requires VANTAGE_VQA and --mode infer") + if mode not in ("infer", "all"): + raise ValueError("Public mode must be infer or all") + if mode == "all": + print("Public annotations omit scoring ground truth: --mode all runs inference only; score on the official portal.") + revisions = {} if dry_run else _prepared(selected) + if not dry_run: + _check_harness() + configs = work / "_configs" + configs.mkdir(parents=True, exist_ok=True) + entries = json.loads((KIT / "manifest.json").read_text()) + provenance = { + "source": "public", "harness_repo": "https://github.com/Clemson-Capstone/VANTAGE-Bench", + "harness_revision": HARNESS_REVISION, "dataset_repo": DATASET_REPO, + "harness_patch": "public-harness.patch (TLS/credential safety; NVIDIA-aligned lossless video and vLLM kwargs)", + "dataset_revisions": revisions, "cosmos_commit": os.environ.get("COSMOS_COMMIT", "unknown"), + "benchmarks": selected, "model": os.environ.get("COSMOS_MODEL", "${COSMOS_MODEL}"), + "requested_mode": mode, "effective_mode": "infer", "smoke": smoke, + "subset": smoke, "dry_run": dry_run, "status": "planned" if dry_run else "running", + "sampling_note": "Committed NVIDIA data/model settings retained where public constructors accept them; " + "public task classes, prompts, IDs and preparation differ. SOT uses public prepared 8-frame clips " + "(nframe is a compatibility argument). No parity claim or local score.", + "artifacts": {}, + } + _write_json(work / "run.json", provenance) + for index, bench in enumerate(selected): + dataset = TASKS[bench][1] + config = _config(bench, smoke) + path = configs / f"{bench}.json" + _write_json(path, config) + flags = entries[f"metropolis/{bench}"]["run"] + out = work / f"bench_{index:02d}" + command = [_python(), str(_harness() / "run.py"), + "--config", str(path), "--mode", "infer", "--work-dir", str(out), + "--lmudata-root", str(_lmu()), "--api-nproc", str(flags["api_nproc"]), + "--verbose"] + if dry_run: + print(shlex.join(command), flush=True) + continue + out.mkdir(parents=True) + report = out / "predictions.json" + try: + _execute(command, log=out / "run.log") + _execute([_python(), "-c", _VERIFY, str(path), str(out), dataset, TASKS[bench][3], + "1" if smoke else "0", str(report)], log=out / "verify.log") + validate = [_python(), str(_harness() / "scripts/validate_submission.py"), "--work-dir", str(out)] + if not smoke: + validate += ["--lmu-root", str(_lmu())] + _execute(validate, log=out / "validate.log") + provenance["artifacts"][bench] = json.loads(report.read_text()) + # Safe model and sampling provenance: omit endpoint and credentials. + model = dict(config["model"]["cosmos"]) + model.pop("api_base", None) + model.pop("key", None) + provenance.setdefault("configs", {})[bench] = {"model": model, "data": config["data"]} + except Exception: + provenance["status"] = "failed" + _write_json(work / "run.json", provenance) + raise + _write_json(work / "run.json", provenance) + if not dry_run: + provenance["status"] = "complete" + _write_json(work / "run.json", provenance) + results(work) + + +def _run_state(work): + path = Path(work) / "run.json" + if not path.is_file(): + raise ValueError(f"No public run metadata at {path}") + state = json.loads(path.read_text()) + if state.get("source") != "public": + raise ValueError("This work directory is not a public run") + return state + + +def results(work: Path): + """Show verified prediction artifacts, not unavailable local scores.""" + state = _run_state(work) + print(f"Public run: {state['status']}" + (" (five-question smoke subset)" if state["smoke"] else "")) + for bench in state["benchmarks"]: + artifact = state.get("artifacts", {}).get(bench) + if artifact: + print(f"{bench}: {artifact['rows']} verified predictions\n Native: {artifact['prediction']}\n Canonical: {artifact['submission']}") + else: + print(f"{bench}: no verified predictions; inspect bench_*/run.log and verify.log") + print("No local public score: hidden labels are evaluated at https://vantage-bench.org/submit (up to 24 hours).") + + +# The official validator warns but does not fail partial pillars. Enforce that +# policy before calling the official validator and packager, without inventing a +# new archive format or a second filename-discovery implementation. +_COVERAGE = r''' +import json, sys +from pathlib import Path +sys.path.insert(0, str(Path.cwd() / "scripts")) +from package_submission import find_submission_files, PILLAR_TASKS +work, expected_json = sys.argv[1:] +found = find_submission_files(Path(work)) +if set(found) != set(json.loads(expected_json)): + raise RuntimeError("Discovered task files do not match the verified run metadata") +for pillar, tasks in PILLAR_TASKS.items(): + covered = set(tasks) & found.keys() + if covered and covered != set(tasks): + raise RuntimeError(f"Partial pillar {pillar}: missing {sorted(set(tasks) - covered)}. Run the complete pillar before export.") +if not found: + raise RuntimeError("No canonical predictions to package") +''' + + +def export(work: Path): + """Validate and package complete public pillars with upstream tooling.""" + work = Path(work).resolve() + state = _run_state(work) + if state.get("smoke") or state.get("subset") or state.get("dry_run") or state.get("status") != "complete": + raise ValueError("Only completed, full-task public runs can be exported; smoke/dry-run/subset runs cannot be submitted") + _check_harness() + _execute([_python(), "-c", _COVERAGE, str(work), + json.dumps([TASKS[bench][3] for bench in state["benchmarks"]])]) + # Re-read predictions at export: a completed run.json is not proof that files + # still exist or have not been edited. The public venv handles every format. + for index, bench in enumerate(state["benchmarks"]): + out = work / f"bench_{index:02d}" + _execute([_python(), "-c", _VERIFY, str(work / "_configs" / f"{bench}.json"), + str(out), TASKS[bench][1], TASKS[bench][3], "0", str(out / "predictions.json")], + log=out / "export-verify.log") + validator = [_python(), str(_harness() / "scripts/validate_submission.py")] + _execute(validator + ["--work-dir", str(work), "--lmu-root", str(_lmu())], log=work / "export-validate.log") + archive = work.parent / f"{work.name}-submission.tar.gz" + _execute([_python(), str(_harness() / "scripts/package_submission.py"), + "--work-dir", str(work), "--out", str(archive)], log=work / "package.log") + _execute(validator + ["--archive", str(archive), "--lmu-root", str(_lmu())], log=work / "archive-validate.log") + if not archive.is_file() or not archive.stat().st_size: + raise RuntimeError(f"Official packager did not create {archive}; see {work / 'package.log'}") + print(f"Official public submission: {archive}") + print("Upload at https://vantage-bench.org/submit; scoring may take up to 24 hours. No local score is claimed.") + return archive diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh new file mode 100755 index 00000000..c6f20d25 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +umask 077 +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +: "${HF_TOKEN:?Supply HF_TOKEN as a secret Brev launch parameter}" +command -v docker >/dev/null +command -v python3 >/dev/null +docker compose version +docker info >/dev/null +export COSMOS_COMMIT +COSMOS_COMMIT=$(git rev-parse HEAD) +export LAUNCHABLE_STATE_DIR=${LAUNCHABLE_STATE_DIR:-"$PWD/state"} +mkdir -p "$LAUNCHABLE_STATE_DIR"/{cache,data,results,credentials/aws} +mkdir -p "$LAUNCHABLE_STATE_DIR/data/"{public,internal}/LMUData +# Persist only the generated notebook credential, never HF/model/judge credentials. +if [[ ! -f .env ]]; then + python3 -c 'import secrets; print("JUPYTER_TOKEN=" + secrets.token_hex(32))' > .env +fi +chmod 600 .env +if [[ -z "${COSMOS_API_BASE:-}" ]]; then + nvidia-smi + docker compose --profile local pull cosmos-endpoint + docker compose --profile local up -d --build +else + # BYOM does not start or allocate a local model/GPU. + docker compose up -d --build vantage-eval +fi +printf '%s\n' 'Jupyter is starting on 127.0.0.1:8888. Open its Brev secure link.' \ + 'Retrieve the notebook login token with: docker compose exec vantage-eval jupyter server list' \ + 'Run the notebook cells to prepare data, wait for the endpoint, and evaluate.' diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py new file mode 100644 index 00000000..b4afd8b9 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Artifact safety/score isolation; no GPU, credentials, or network required.""" +import contextlib +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest + +import launchable +import public_eval + + +class ArtifactTests(unittest.TestCase): + def test_scores_are_read_from_each_benchmarks_own_output(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + dirs = {"VANTAGE_VQA": "bench_00", "VANTAGE_EventVerification": "bench_01"} + (root / "run.json").write_text(json.dumps({"work_dirs": dirs})) + for bench, score in (("VANTAGE_VQA", 0.75), ("VANTAGE_EventVerification", 0.25)): + output = root / dirs[bench] / "cosmos" / "T20261016" + output.mkdir(parents=True) + (output / f"cosmos_{bench}.dict.eval.json").write_text(json.dumps({"Overall": score})) + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + launchable.results(root) + rows = dict(line.split() for line in printed.getvalue().splitlines()) + self.assertEqual(float(rows["VANTAGE_VQA"]), 75.0) + self.assertEqual(float(rows["VANTAGE_EventVerification"]), 25.0) + + def test_internal_export_excludes_logs_configs_and_duplicate_symlinks(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) / "run" + prediction_dir = work / "bench_00" / "cosmos" / "T20261016" + prediction_dir.mkdir(parents=True) + prediction = prediction_dir / "cosmos_VANTAGE_VQA.tsv" + prediction.write_text("index\tprediction\n0\tA\n") + (prediction_dir.parent / prediction.name).symlink_to(prediction) + (work / "run.log").write_text("sensitive diagnostic") + (work / "_configs").mkdir() + (work / "_configs" / "secret.json").write_text('{"key":"secret"}') + with contextlib.redirect_stdout(io.StringIO()): + launchable.export(work) + with tarfile.open(work.parent / "run-predictions.tar.gz") as bundle: + self.assertEqual(bundle.getnames(), ["bench_00/cosmos/T20261016/cosmos_VANTAGE_VQA.tsv"]) + self.assertEqual(bundle.extractfile(bundle.getmembers()[0]).read(), prediction.read_bytes()) + + def test_existing_output_is_rejected_before_network_or_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + with self.assertRaisesRegex(ValueError, "fresh --work-dir"): + launchable.run(["VANTAGE_VQA"], Path(temp), "infer", True, False, "internal") + + def test_public_export_rejects_smoke_before_packaging(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) + (work / "run.json").write_text(json.dumps({ + "source": "public", "status": "complete", "smoke": True, + })) + with self.assertRaisesRegex(ValueError, "smoke/dry-run/subset"): + public_eval.export(work) + + def test_public_export_rejects_failed_and_planned_runs(self): + with tempfile.TemporaryDirectory() as temp: + work = Path(temp) + for status in ("failed", "planned"): + with self.subTest(status=status): + (work / "run.json").write_text(json.dumps({ + "source": "public", "status": status, "smoke": False, + })) + with self.assertRaisesRegex(ValueError, "Only completed"): + public_eval.export(work) + + +if __name__ == "__main__": + unittest.main() From 1b506b29cd371503bb4e36423d90d3085c9311cd Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Fri, 23 Oct 2026 23:23:17 -0500 Subject: [PATCH 2/5] Fix public launchable access and reject invalid smoke predictions --- .../launchable/Evaluate_Cosmos3.ipynb | 14 ++++++---- .../reasoner/vlmevalkit/launchable/README.md | 16 ++++++----- .../vlmevalkit/launchable/compose.yaml | 1 + .../vlmevalkit/launchable/launchable.py | 14 ++++++---- .../vlmevalkit/launchable/public_eval.py | 5 +++- .../reasoner/vlmevalkit/launchable/setup.sh | 1 - .../vlmevalkit/launchable/test_launchable.py | 27 +++++++++++++++++++ 7 files changed, 59 insertions(+), 19 deletions(-) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index 331f9dcd..fd9cded4 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -23,7 +23,7 @@ "source": [ "## 1 · Configure a fresh run\n", "\n", - "`HF_TOKEN` normally arrives from the Brev launch secret. A hidden prompt appears only if it is absent; the value stays in the process environment, never notebook source/output. Public mode does **not** need internal AWS/NGC/dataset-service or API judge keys. Existing endpoint/model environment settings are preserved.\n", + "The pinned Nano model and public VQA data support anonymous access; no HF token or hidden prompt is needed for this smoke. An inherited secret `HF_TOKEN` remains available for gated assets. Full SOT preparation requires a read token with accepted [PhysicalAI-SmartSpaces access terms](https://huggingface.co/datasets/nvidia/PhysicalAI-SmartSpaces). Public mode does **not** need internal AWS/NGC/dataset-service or API judge keys. Existing endpoint/model environment settings are preserved.\n", "\n", "Use a new timestamped directory for each run. Re-running this cell creates new output paths; do not mix smoke and full outputs. Change `SOURCE` to `internal` only after configuring the host AWS mount described at the end." ] @@ -51,8 +51,6 @@ "SMOKE_WORK = Path('/results') / f'{SOURCE}-smoke-{STAMP}'\n", "FULL_WORK = Path('/results') / f'public-full-{STAMP}'\n", "\n", - "if not os.environ.get('HF_TOKEN'):\n", - " os.environ['HF_TOKEN'] = getpass('Hugging Face token (hidden): ')\n", "os.environ.setdefault('COSMOS_API_BASE', 'http://cosmos-endpoint:8080/v1/chat/completions')\n", "# Do not overwrite COSMOS_MODEL: deployment/BYOM configuration selects it.\n", "print(f'Source: {SOURCE}; smoke output: {SMOKE_WORK}; full run enabled: {RUN_FULL}')" @@ -192,6 +190,12 @@ "if RUN_FULL:\n", " if SOURCE != 'public':\n", " raise ValueError('This full-submission section requires SOURCE = public.')\n", + " if not os.environ.get('HF_TOKEN'):\n", + " token = getpass('HF read token with accepted SmartSpaces access (hidden): ').strip()\n", + " if not token:\n", + " raise ValueError('Full SOT preparation requires authorized SmartSpaces access and HF_TOKEN.')\n", + " os.environ['HF_TOKEN'] = token\n", + " del token\n", " subprocess.run(CLI + ['prepare', '--source', 'public', '--benchmarks', 'all'], check=True)\n", "else:\n", " print('Full public run disabled; set RUN_FULL = True to opt in.')" @@ -254,7 +258,7 @@ "source": [ "## Optional · Authorized internal evaluation\n", "\n", - "Public mode needs only the HF token. For internal mode, configure **at initial deployment** an existing host `AWS_CREDENTIALS_DIR` containing AWS `config` and `credentials` with the authorized `team-cosmos` profile (or securely populate the default `state/credentials/aws` before setup). Compose mounts it read-only under `/credentials/aws`, using `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. Credentials must remain valid during preparation. A notebook variable cannot create or change a host bind mount. Never paste AWS values into this notebook.\n", + "Public VQA smoke supports anonymous HF access. For internal mode, configure **at initial deployment** an existing host `AWS_CREDENTIALS_DIR` containing AWS `config` and `credentials` with the authorized `team-cosmos` profile (or securely populate the default `state/credentials/aws` before setup). Compose mounts it read-only under `/credentials/aws`, using `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. Credentials must remain valid during preparation. A notebook variable cannot create or change a host bind mount. Never paste AWS values into this notebook.\n", "\n", "Then set `SOURCE = 'internal'` in section 1, keep `RUN_FULL = False`, and rerun configuration, VQA preparation, health check, smoke, and results. Those same commands pass `--source internal` to the NVIDIA harness. Internal data stays under `/data/internal/LMUData`; public data stays under `/data/public/LMUData`, with separate Python/import roots as well. Internal `export --source internal` creates an evidence bundle, **not** a public leaderboard submission. See the README for CLI examples.\n", "\n", @@ -286,4 +290,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 698d8f3c..10c122f3 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -2,11 +2,11 @@ A Jupyter front end for preparing data, checking an OpenAI-compatible video endpoint, running five real VQA questions, and exporting a full public VANTAGE submission. Notebook cells call `launchable.py`; dataset preparation, inference, and submission conversion stay in the selected upstream harness. -**Default: public data and the official public harness. Only `HF_TOKEN` is required as a launch secret.** No internal AWS, NGC, NVIDIA dataset-service, or API judge credentials are required for the eight VANTAGE tasks. A separately authenticated BYOM endpoint may require its own `COSMOS_API_KEY`. +**Default: public data and the official public harness. The pinned Nano model and public VQA smoke support anonymous Hugging Face access.** No launch secret is required for that path. No internal AWS, NGC, NVIDIA dataset-service, or API judge credentials are required for the eight VANTAGE tasks. A separately authenticated BYOM endpoint may require its own `COSMOS_API_KEY`. -Create a **read** token at [Hugging Face token settings](https://huggingface.co/settings/tokens) and supply it as Brev's secret `HF_TOKEN` launch parameter. Full tracking also requires accepting the [PhysicalAI-SmartSpaces access terms](https://huggingface.co/datasets/nvidia/PhysicalAI-SmartSpaces). No internal NVIDIA key is needed for the public workflow. +For gated assets, create a **read** token at [Hugging Face token settings](https://huggingface.co/settings/tokens) and supply it as Brev's secret `HF_TOKEN` launch parameter. Full SOT/tracking additionally requires accepting the [PhysicalAI-SmartSpaces access terms](https://huggingface.co/datasets/nvidia/PhysicalAI-SmartSpaces); a token alone does not grant access. The full notebook section prompts privately if that token was not inherited. No internal NVIDIA key is needed for the public workflow. -This directory defines a launchable; it does not provision a VM or publish a Brev launchable. GPU serving, end-to-end GPU evaluation, runtime, and leaderboard parity are **not yet GPU-verified**. First successful predictions are not a benchmark score. Do not assume a sub-hour run or reference-score reproduction. +This directory defines a launchable; it does not itself provision a VM or publish a Brev launchable. A native single-H200 public VQA smoke has produced five real predictions. That does **not** establish Brev notebook readiness, full benchmark scores, or leaderboard parity. Do not assume a sub-hour run or reference-score reproduction. ## Create the Brev launchable @@ -18,7 +18,7 @@ Use these deployment fields in Brev: | Deployment source | This Cosmos repository checkout: `https://github.com/NVIDIA/cosmos.git`, on the branch/commit **containing this launchable**; record the exact deployment commit | | Setup script command | `bash evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh` from the Cosmos checkout root | | Managed Jupyter | **Disabled**; Compose starts Jupyter itself | -| Secret environment variable | `HF_TOKEN`, with access to the model/data and accepted applicable licenses | +| Optional secret environment variable | `HF_TOKEN` for gated assets; required with accepted SmartSpaces access for full SOT, not for the default public VQA smoke | | Application access | Authenticated/secure Brev link to port **8888**; do not publish the model endpoint | | Conservative starting hardware | One RTX PRO 6000 with **96 GB VRAM**, at least **256 GB disk**; sizing is an unvalidated starting point, not a compatibility guarantee | @@ -28,7 +28,7 @@ If Brev presents a setup-script text box, paste the setup command above, not jus Local serving defaults to `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` (overridable with `VLLM_IMAGE`) and `nvidia/Cosmos3-Nano` at revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`. Both evaluator and serving containers target `linux/amd64`. Confirm the serving image is available to your VM and compatible with its GPU/driver. Disk must accommodate model weights, image layers, downloaded data, extracted media, caches, and results. Public data is roughly 42 GB, with roughly 16 GB more for SOT; headroom and actual resource needs depend on workload. Full SOT preparation also requires access to the SmartSpaces source and acceptance of its license; an HF token alone does not grant that access. -If running setup manually on the VM after injecting `HF_TOKEN` into its environment: +If running setup manually on the VM (supply `HF_TOKEN` in the environment only when needed for gated assets): ```bash cd /path/to/cosmos/evaluation/cosmos3/reasoner/vlmevalkit/launchable @@ -51,10 +51,10 @@ The evaluator has **no Docker socket**. Container lifecycle commands belong in t ## Notebook workflow -1. **Configure:** public source by default, a unique timestamped output directory, and inherited endpoint/model settings. A hidden `getpass` prompt is used only if `HF_TOKEN` was not inherited. Credentials remain in process memory; never save them in cells. +1. **Configure:** public source by default, a unique timestamped output directory, and inherited endpoint/model settings. The default public VQA smoke does not prompt for an HF token. The opt-in full/SOT section requires an inherited token or a hidden `getpass` prompt; accepted SmartSpaces access is still required. Credentials remain in process memory; never save them in cells. 2. **Prepare VQA:** use the selected harness's official preparation. Downloads/preparation may be substantial even for a smoke run. 3. **Check endpoint:** wait for `/v1/models` and verify the configured served model. -4. **Smoke:** infer exactly five seeded `VANTAGE_VQA` questions. This tests actual data/media/model integration; it is not synthetic and is not a reference score. +4. **Smoke:** infer exactly five seeded `VANTAGE_VQA` questions. Verification rejects missing/extra rows, empty or whitespace-only answers, API failure markers, and `VIDEO_LOAD_FAILED`. This tests actual data/media/model integration; it is not synthetic and is not a reference score. 5. **Inspect results:** the helper checks consumer-visible predictions, not just the native harness's exit status. The native harness can catch a dataset failure and still exit zero. 6. **Full public evaluation:** explicitly set `RUN_FULL = True` before running the full section. Its default is false, so **Run All does not start the full eight-task run**. It prepares all eight tasks and runs inference in a separate fresh directory. 7. **Export/download:** use the official public packager, then download the generated submission artifact via Jupyter. The notebook does not upload anything. Submit at [vantage-bench.org/submit](https://vantage-bench.org/submit); official scoring can take up to 24 hours. @@ -107,6 +107,8 @@ Public mode needs none of this. Internal mode requires authorized access to NVID Compose mounts that directory read-only at `/credentials/aws` and sets `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. An empty default mount grants no access. No `NVDATASET_TENANTID`, `NGC_API_KEY`, or judge API key is needed for these eight VANTAGE tasks. If a different mount was not configured initially, reconfigure it on the VM and recreate the evaluator using setup; changing a notebook variable cannot create a host mount. +A read-only credentials-file mount does **not** make S3 operations read-only: the internal video cache may upload on a miss. Enforce an actual read-only storage policy before invoking the internal CLI directly; a read-only file mount alone is insufficient. + Set notebook `SOURCE = "internal"` before preparation/smoke, or use: ```bash diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml index aff62c76..6168010b 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -62,6 +62,7 @@ services: - '127.0.0.1:8888:8888' environment: HF_TOKEN: ${HF_TOKEN:-} + COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} COSMOS_API_BASE: ${COSMOS_API_BASE:-http://cosmos-endpoint:8080/v1/chat/completions} COSMOS_MODEL: ${COSMOS_MODEL:-nvidia/Cosmos3-Nano} COSMOS_API_KEY: ${COSMOS_API_KEY:-EMPTY} diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py index 8617531f..9b425fe2 100755 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchable.py @@ -98,7 +98,7 @@ def prediction_files(work): and p.stem in {f"cosmos_{bench}" for bench in benchmarks("all")}) -def verify_predictions(work, bench): +def verify_predictions(work, bench, expected_rows=None): import pandas as pd files = [p for p in prediction_files(work) if p.stem == f"cosmos_{bench}"] @@ -114,9 +114,13 @@ def verify_predictions(work, bench): frame = pd.read_excel(path) if "prediction" not in frame or frame.empty: raise RuntimeError(f"{bench}: empty or invalid prediction output") - failed = frame["prediction"].isna() | frame["prediction"].astype(str).str.contains( - "Failed to obtain answer via API.", regex=False - ) + if expected_rows is not None and len(frame) != expected_rows: + raise RuntimeError(f"{bench}: expected {expected_rows} predictions, found {len(frame)}") + text = frame["prediction"].astype(str).str.strip().str.lower() + failed = (frame["prediction"].isna() + | text.isin(("", "nan", "none", "null", "")) + | text.str.contains("failed to obtain answer", regex=False) + | text.str.contains("video_load_failed", regex=False)) if failed.any(): raise RuntimeError(f"{bench}: {int(failed.sum())} failed/empty predictions; inspect the run log") print(f"{bench}: {len(frame)} predictions verified", flush=True) @@ -173,7 +177,7 @@ def run(selected, work, mode, smoke, dry_run, source): out.mkdir(parents=True) with (out / "run.log").open("w") as log: run_command(command, stdout=log, stderr=subprocess.STDOUT) - verify_predictions(out, bench) + verify_predictions(out, bench, expected_rows=5 if smoke else None) if not dry_run: record_run(work, selected, mode, smoke) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py index 32e4a778..5b6421a4 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py @@ -152,6 +152,8 @@ def _selected(selected): def prepare(selected: list[str], revision: str): """Materialize only selected task subtrees, using official preparation.""" _selected(selected) + if "VANTAGE_SOT" in selected and not os.environ.get("HF_TOKEN"): + raise ValueError("SOT requires HF_TOKEN with accepted nvidia/PhysicalAI-SmartSpaces access terms") if not re.fullmatch(r"[0-9a-f]{40}", revision): raise ValueError("Use an immutable 40-character HF dataset commit SHA") _check_harness() @@ -224,7 +226,8 @@ def _config(bench, smoke): def bad(value): text = str(value).strip() return (value is None or text.lower() in ("", "nan", "none", "null", "") - or "failed to obtain answer" in text.lower()) + or "failed to obtain answer" in text.lower() + or "video_load_failed" in text.lower()) if frame["prediction"].map(bad).any(): raise RuntimeError("Failed/null native predictions; inspect run.log") if "index" not in frame or frame["index"].duplicated().any() or set(map(str, frame["index"])) != set(map(str, expected["index"])): diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh index c6f20d25..506112d4 100755 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -4,7 +4,6 @@ set -euo pipefail umask 077 cd -- "$(dirname -- "${BASH_SOURCE[0]}")" -: "${HF_TOKEN:?Supply HF_TOKEN as a secret Brev launch parameter}" command -v docker >/dev/null command -v python3 >/dev/null docker compose version diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py index b4afd8b9..649b8964 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/test_launchable.py @@ -72,6 +72,33 @@ def test_public_export_rejects_failed_and_planned_runs(self): with self.assertRaisesRegex(ValueError, "Only completed"): public_eval.export(work) + def test_internal_predictions_reject_empty_and_failure_markers(self): + for value in (" ", "VIDEO_LOAD_FAILED", "prefix VIDEO_LOAD_FAILED", + "Failed to obtain answer via API.", "null"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as temp: + work = Path(temp) + output = work / "cosmos" / "T20261023" + output.mkdir(parents=True) + (output / "cosmos_VANTAGE_VQA.tsv").write_text( + "index\tprediction\n0\t" + value + "\n") + with self.assertRaisesRegex(RuntimeError, "failed/empty"): + launchable.verify_predictions(work, "VANTAGE_VQA") + + def test_internal_smoke_requires_exactly_five_predictions(self): + for count in (4, 5, 6): + with self.subTest(count=count), tempfile.TemporaryDirectory() as temp: + work = Path(temp) + output = work / "cosmos" / "T20261023" + output.mkdir(parents=True) + (output / "cosmos_VANTAGE_VQA.tsv").write_text( + "index\tprediction\n" + "".join(f"{i}\tA\n" for i in range(count))) + if count == 5: + with contextlib.redirect_stdout(io.StringIO()): + launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) + else: + with self.assertRaisesRegex(RuntimeError, "expected 5 predictions"): + launchable.verify_predictions(work, "VANTAGE_VQA", expected_rows=5) + if __name__ == "__main__": unittest.main() From cef89742d8b86eddea3df57b3dc3fc0480d29eb1 Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Fri, 23 Oct 2026 23:35:38 -0500 Subject: [PATCH 3/5] Bind Brev notebooks to the authenticated private mesh --- .../launchable/Evaluate_Cosmos3.ipynb | 2 +- .../reasoner/vlmevalkit/launchable/README.md | 11 +++-- .../vlmevalkit/launchable/compose.yaml | 2 +- .../reasoner/vlmevalkit/launchable/setup.sh | 47 +++++++++++++++++-- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index fd9cded4..978d7540 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -11,7 +11,7 @@ "\n", "This notebook is a thin front end to `launchable.py`, using the official public VANTAGE harness by default. The full eight-task run is **opt-in**: **Run All runs the five-question smoke only** while `RUN_FULL = False`.\n", "\n", - "**Before opening:** run `bash setup.sh` on the Brev VM from the launchable directory. Use the authenticated Brev link to port 8888. Docker setup/lifecycle commands run on the VM host, not here; this evaluator has no Docker socket. See [README](README.md) for launch fields and credentials.\n", + "**Before opening:** run `bash setup.sh --brev` on the Brev VM from the launchable directory. It binds Jupyter to the private mesh address for the authenticated Brev link on port 8888. Docker setup/lifecycle commands run on the VM host, not here; this evaluator has no Docker socket. See [README](README.md) for launch fields and credentials.\n", "\n", "**Validation boundary:** GPU serving, runtime, end-to-end full evaluation, portal acceptance, and reference-score parity are not yet GPU-verified. Five predictions are not a leaderboard score." ] diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 10c122f3..7b9943cb 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -16,15 +16,16 @@ Use these deployment fields in Brev: | --- | --- | | Environment | **VM mode**, Linux x86-64, Docker Engine with Compose v2 and NVIDIA Container Toolkit for local serving | | Deployment source | This Cosmos repository checkout: `https://github.com/NVIDIA/cosmos.git`, on the branch/commit **containing this launchable**; record the exact deployment commit | -| Setup script command | `bash evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh` from the Cosmos checkout root | +| Setup script command | `bash evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh --brev` from the Cosmos checkout root | | Managed Jupyter | **Disabled**; Compose starts Jupyter itself | | Optional secret environment variable | `HF_TOKEN` for gated assets; required with accepted SmartSpaces access for full SOT, not for the default public VQA smoke | | Application access | Authenticated/secure Brev link to port **8888**; do not publish the model endpoint | +| Notebook host binding | `--brev` discovers the private `wt0` mesh IPv4 address; non-Brev setup defaults to `127.0.0.1`, with an explicit private `JUPYTER_BIND_ADDRESS` override available | | Conservative starting hardware | One RTX PRO 6000 with **96 GB VRAM**, at least **256 GB disk**; sizing is an unvalidated starting point, not a compatibility guarantee | Do not select an upstream commit that lacks these files. Local edits are not automatically available to Brev: the deployment branch/commit must include them and be accessible to the VM. No branch is published by setup. -If Brev presents a setup-script text box, paste the setup command above, not just the filename. It must execute from the cloned Cosmos repository root. +If Brev presents a setup-script text box, paste the setup command above, not just the filename. It must execute from the cloned Cosmos repository root. Brev's authenticated proxy connects through the VM's private mesh interface, not loopback: `--brev` discovers exactly one global `wt0` IPv4 address in its `100.64.0.0/10` mesh range and fails clearly if unavailable. An explicit private IPv4 `JUPYTER_BIND_ADDRESS` takes precedence. There is no public/wildcard fallback. Do not expose a raw cloud/firewall port for 8888. Local serving defaults to `vllm/vllm-openai:cosmos3@sha256:db0bb920b0b54e82ea96a98659bbd21921f87d0dcfc86feffdafa2db3f08be55` (overridable with `VLLM_IMAGE`) and `nvidia/Cosmos3-Nano` at revision `e59a53c25979a090fa8706c9acc0c254a6e89b92`. Both evaluator and serving containers target `linux/amd64`. Confirm the serving image is available to your VM and compatible with its GPU/driver. Disk must accommodate model weights, image layers, downloaded data, extracted media, caches, and results. Public data is roughly 42 GB, with roughly 16 GB more for SOT; headroom and actual resource needs depend on workload. Full SOT preparation also requires access to the SmartSpaces source and acceptance of its license; an HF token alone does not grant that access. @@ -32,10 +33,10 @@ If running setup manually on the VM (supply `HF_TOKEN` in the environment only w ```bash cd /path/to/cosmos/evaluation/cosmos3/reasoner/vlmevalkit/launchable -bash setup.sh +bash setup.sh --brev ``` -Setup builds the evaluator, starts Jupyter, and starts the local GPU endpoint unless `COSMOS_API_BASE` was supplied at deployment. It persists a generated Jupyter token in a permission-restricted `.env`; it does not write HF/model credentials there. Use the same environment variables when recreating services. Never paste secrets into the notebook, source files, shell history, or screenshots. +Setup builds the evaluator, starts Jupyter, and starts the local GPU endpoint unless `COSMOS_API_BASE` was supplied at deployment. It persists a generated Jupyter token and the selected non-secret `JUPYTER_BIND_ADDRESS` in a permission-restricted `.env`; it does not write HF/model credentials there. Later Compose lifecycle commands retain that binding. Use `bash setup.sh` without `--brev` for a non-Brev host/SSH tunnel; its initial default is loopback. Use the same environment variables when recreating services. Never paste secrets into the notebook, source files, shell history, or screenshots. ### Open Jupyter securely @@ -45,7 +46,7 @@ On the VM, from the launchable directory: docker compose exec vantage-eval jupyter server list ``` -Treat the displayed login token as a secret. Open Brev's secure port-8888 link and authenticate with that token. The host binding is `127.0.0.1:8888`, not a public listener. If your access method requires a local tunnel, forward port 8888 through Brev/SSH rather than rebinding it to `0.0.0.0`. Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb` in Jupyter. Jupyter's file root is `/`, allowing authenticated downloads from `/results`. +Treat the displayed login token as a secret. Open Brev's secure port-8888 link and authenticate with that token. The default non-Brev host binding is `127.0.0.1:8888` for SSH tunnels. Brev's authenticated proxy instead requires `bash setup.sh --brev` to bind its private mesh IPv4 address; binding only loopback makes that proxy return connection-refused/503. Setup persists the selected address for later Compose commands. Keep 8888 closed on the public cloud firewall, preserve provider login and Jupyter token authentication, and never bind to `0.0.0.0` or a public host address. Open `/opt/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb` in Jupyter. Jupyter's file root is `/`, allowing authenticated downloads from `/results`. The evaluator has **no Docker socket**. Container lifecycle commands belong in the VM terminal, not notebook cells. The local model endpoint has no host port; only the Compose network can reach it. diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml index 6168010b..922f8766 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/compose.yaml @@ -59,7 +59,7 @@ services: COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} restart: unless-stopped ports: - - '127.0.0.1:8888:8888' + - '${JUPYTER_BIND_ADDRESS:-127.0.0.1}:8888:8888' environment: HF_TOKEN: ${HF_TOKEN:-} COSMOS_COMMIT: ${COSMOS_COMMIT:-unknown} diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh index 506112d4..bee361b0 100755 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/setup.sh @@ -13,10 +13,46 @@ COSMOS_COMMIT=$(git rev-parse HEAD) export LAUNCHABLE_STATE_DIR=${LAUNCHABLE_STATE_DIR:-"$PWD/state"} mkdir -p "$LAUNCHABLE_STATE_DIR"/{cache,data,results,credentials/aws} mkdir -p "$LAUNCHABLE_STATE_DIR/data/"{public,internal}/LMUData -# Persist only the generated notebook credential, never HF/model/judge credentials. -if [[ ! -f .env ]]; then - python3 -c 'import secrets; print("JUPYTER_TOKEN=" + secrets.token_hex(32))' > .env -fi +# Persist notebook authentication and its non-secret bind address, never model/judge credentials. +export JUPYTER_BIND_ADDRESS +JUPYTER_BIND_ADDRESS=$(python3 - "$@" <<'PY' +import ipaddress +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys + +if sys.argv[1:] not in ([], ["--brev"]): + raise SystemExit("Usage: bash setup.sh [--brev]") +path = Path(".env") +lines = path.read_text().splitlines() if path.exists() else [ + "JUPYTER_TOKEN=" + secrets.token_hex(32) +] +stored = dict(line.split("=", 1) for line in lines if line.startswith("JUPYTER_BIND_ADDRESS=")) +address = os.environ.get("JUPYTER_BIND_ADDRESS") or stored.get("JUPYTER_BIND_ADDRESS", "127.0.0.1") +if sys.argv[1:] == ["--brev"] and not os.environ.get("JUPYTER_BIND_ADDRESS"): + try: + interfaces = json.loads(subprocess.check_output( + ["ip", "-j", "-4", "addr", "show", "dev", "wt0"], timeout=5)) + addresses = [item["local"] for interface in interfaces for item in interface["addr_info"] + if item["family"] == "inet" and item.get("scope") == "global"] + if len(addresses) != 1: + raise ValueError("expected exactly one global wt0 IPv4 address") + address = addresses[0] + if ipaddress.IPv4Address(address) not in ipaddress.IPv4Network("100.64.0.0/10"): + raise ValueError("wt0 is not in the private Brev mesh range") + except (OSError, subprocess.SubprocessError, ValueError, KeyError) as error: + raise SystemExit(f"Brev private notebook binding unavailable: {error}") from error +allowed = ("127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10") +if not any(ipaddress.IPv4Address(address) in ipaddress.IPv4Network(network) for network in allowed): + raise SystemExit("JUPYTER_BIND_ADDRESS must be a loopback or private IPv4 address, never a public/wildcard address") +lines = [line for line in lines if not line.startswith("JUPYTER_BIND_ADDRESS=")] +path.write_text("\n".join(lines + ["JUPYTER_BIND_ADDRESS=" + address]) + "\n") +print(address) +PY +) chmod 600 .env if [[ -z "${COSMOS_API_BASE:-}" ]]; then nvidia-smi @@ -26,6 +62,7 @@ else # BYOM does not start or allocate a local model/GPU. docker compose up -d --build vantage-eval fi -printf '%s\n' 'Jupyter is starting on 127.0.0.1:8888. Open its Brev secure link.' \ +printf 'Jupyter is starting on %s:8888. Open its authenticated secure link or SSH tunnel.\n' "${JUPYTER_BIND_ADDRESS:-127.0.0.1}" +printf '%s\n' \ 'Retrieve the notebook login token with: docker compose exec vantage-eval jupyter server list' \ 'Run the notebook cells to prepare data, wait for the endpoint, and evaluate.' From 23cfa62968f0a395481b52c4fd91bae9ce1106dd Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Fri, 23 Oct 2026 23:44:05 -0500 Subject: [PATCH 4/5] Isolate public evaluation workers from notebook plotting backends --- evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md | 2 ++ .../cosmos3/reasoner/vlmevalkit/launchable/public_eval.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 7b9943cb..93f9bcfd 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -92,6 +92,8 @@ The normal public workflow is inference plus official submission export; leaderb The checkouts, Python environments, and data roots are intentionally separate: an internal TSV/cache must never be silently reused as public data and produce misleading scores. Both share only the persistent HF cache at `/cache/huggingface`. Public configs preserve the committed NVIDIA FPS/frame limits, pixel limits, and model sampling parameters where the official public classes accept them. Dataset classes are translated to the public implementations, Astro2D uses the public `Astro2D` dataset name, and native prompts, dataset construction, ID emission, and server-side scoring differ from the internal harness. Review the generated configs and run provenance for actual settings. No cross-harness score parity is claimed. +Public subprocesses explicitly use Matplotlib's headless `Agg` backend. They do not inherit Jupyter's inline plotting backend, which belongs to the notebook kernel rather than the isolated harness environment. + The image applies the tracked `public-harness.patch` to the pinned public source. It preserves TLS certificate verification and removes the `model_config.txt` diagnostic that exposed API credentials. It also aligns video serialization with this NVIDIA fork: lossless H.264 (`crf=0`), scalar FPS, no server-side resampling, and removal of the unsupported `do_resize` vLLM argument. Public dataset prompts and submission IDs remain upstream's; retain the patch with the source pin for reproduction. On the host, `LAUNCHABLE_STATE_DIR` defaults to `state/` beside `setup.sh`. It contains `cache/`, `data/public/LMUData`, `data/internal/LMUData`, `results/`, and an optional `credentials/aws/`. `/results` survives evaluator restarts. Data/source pins do not freeze an externally served model; record the endpoint's model revision as well as the deployment commit and generated configuration. diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py index 5b6421a4..4dbf10a6 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/public_eval.py @@ -52,6 +52,8 @@ def _env(): # Never import the NVIDIA checkout via inherited notebook/container settings. env["PYTHONPATH"] = str(_harness()) env["LMUData"] = str(_lmu()) + # Public CLI workers are headless; the notebook's inline backend is not installed there. + env["MPLBACKEND"] = "Agg" env["PRED_FORMAT"] = env.get("PRED_FORMAT") or "tsv" # The public runner otherwise silently replaces the explicit --work-dir. env.pop("MMEVAL_ROOT", None) From 0297efb4ee50584055f5ba4f95293d433f71c0d0 Mon Sep 17 00:00:00 2001 From: Jon Patterson Date: Fri, 23 Oct 2026 23:57:30 -0500 Subject: [PATCH 5/5] Add validated private single-GPU LaunchPad evaluation workflow --- .../launchable/Evaluate_Cosmos3.ipynb | 2 +- .../reasoner/vlmevalkit/launchable/README.md | 8 +- .../vlmevalkit/launchable/launchpad/README.md | 181 ++++++++++++++++++ .../launchable/launchpad/launchpad.py | 166 ++++++++++++++++ .../launchable/launchpad/provision_profile.py | 62 ++++++ .../launchable/launchpad/runtime.py | 114 +++++++++++ .../launchable/launchpad/stage_internal.py | 102 ++++++++++ 7 files changed, 631 insertions(+), 4 deletions(-) create mode 100644 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/README.md create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/launchpad.py create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/provision_profile.py create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/runtime.py create mode 100755 evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/stage_internal.py diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb index 978d7540..76883ea7 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/Evaluate_Cosmos3.ipynb @@ -13,7 +13,7 @@ "\n", "**Before opening:** run `bash setup.sh --brev` on the Brev VM from the launchable directory. It binds Jupyter to the private mesh address for the authenticated Brev link on port 8888. Docker setup/lifecycle commands run on the VM host, not here; this evaluator has no Docker socket. See [README](README.md) for launch fields and credentials.\n", "\n", - "**Validation boundary:** GPU serving, runtime, end-to-end full evaluation, portal acceptance, and reference-score parity are not yet GPU-verified. Five predictions are not a leaderboard score." + "**Validation boundary:** the public five-question VQA notebook passed on a Brev 96 GB RTX PRO 6000 Blackwell GPU at commit `23cfa62968f0a395481b52c4fd91bae9ce1106dd`, using a resumed VM and retained cache. An uninterrupted fresh-VM cold launch, full evaluation, portal acceptance, and reference-score parity remain unverified. Five predictions are not a leaderboard score." ] }, { diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md index 93f9bcfd..a6044c92 100644 --- a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/README.md @@ -6,7 +6,9 @@ A Jupyter front end for preparing data, checking an OpenAI-compatible video endp For gated assets, create a **read** token at [Hugging Face token settings](https://huggingface.co/settings/tokens) and supply it as Brev's secret `HF_TOKEN` launch parameter. Full SOT/tracking additionally requires accepting the [PhysicalAI-SmartSpaces access terms](https://huggingface.co/datasets/nvidia/PhysicalAI-SmartSpaces); a token alone does not grant access. The full notebook section prompts privately if that token was not inherited. No internal NVIDIA key is needed for the public workflow. -This directory defines a launchable; it does not itself provision a VM or publish a Brev launchable. A native single-H200 public VQA smoke has produced five real predictions. That does **not** establish Brev notebook readiness, full benchmark scores, or leaderboard parity. Do not assume a sub-hour run or reference-score reproduction. +This directory defines a launchable; it does not itself provision a VM or publish a Brev launchable. The public five-question VQA notebook passed on a Brev RTX PRO 6000 Blackwell Server Edition at commit `23cfa62968f0a395481b52c4fd91bae9ce1106dd`; a native single-H200 public VQA smoke also passed. Brev validation reused a stopped/resumed VM and retained cache. An uninterrupted fresh-VM cold launch, full benchmark scores, portal acceptance, and leaderboard parity remain unverified. Five predictions are not a benchmark score. + +For an existing SSH-accessible host instead of a Brev VM, use the [repeatable LaunchPad setup](launchpad/README.md). It reuses this launchable's serving configuration, evaluator, controller, and native dataset layout. ## Create the Brev launchable @@ -21,7 +23,7 @@ Use these deployment fields in Brev: | Optional secret environment variable | `HF_TOKEN` for gated assets; required with accepted SmartSpaces access for full SOT, not for the default public VQA smoke | | Application access | Authenticated/secure Brev link to port **8888**; do not publish the model endpoint | | Notebook host binding | `--brev` discovers the private `wt0` mesh IPv4 address; non-Brev setup defaults to `127.0.0.1`, with an explicit private `JUPYTER_BIND_ADDRESS` override available | -| Conservative starting hardware | One RTX PRO 6000 with **96 GB VRAM**, at least **256 GB disk**; sizing is an unvalidated starting point, not a compatibility guarantee | +| Observed VQA smoke hardware | One RTX PRO 6000 Blackwell Server Edition with **96 GB VRAM**, **64 GiB RAM**, and **256 GiB disk**; this is not a minimum-size or full-benchmark compatibility guarantee | Do not select an upstream commit that lacks these files. Local edits are not automatically available to Brev: the deployment branch/commit must include them and be accessible to the VM. No branch is published by setup. @@ -110,7 +112,7 @@ Public mode needs none of this. Internal mode requires authorized access to NVID Compose mounts that directory read-only at `/credentials/aws` and sets `AWS_CONFIG_FILE=/credentials/aws/config` and `AWS_SHARED_CREDENTIALS_FILE=/credentials/aws/credentials`. An empty default mount grants no access. No `NVDATASET_TENANTID`, `NGC_API_KEY`, or judge API key is needed for these eight VANTAGE tasks. If a different mount was not configured initially, reconfigure it on the VM and recreate the evaluator using setup; changing a notebook variable cannot create a host mount. -A read-only credentials-file mount does **not** make S3 operations read-only: the internal video cache may upload on a miss. Enforce an actual read-only storage policy before invoking the internal CLI directly; a read-only file mount alone is insufficient. +A read-only credentials-file mount does **not** make S3 operations read-only: the internal video cache may upload on a miss. The [LaunchPad setup](launchpad/README.md) separates authorized preparation from credential-free, network-isolated inference and supports data-only workstation staging when the host cannot reach PDX. Otherwise, enforce an actual read-only storage policy before invoking the internal CLI directly. Set notebook `SOURCE = "internal"` before preparation/smoke, or use: diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/README.md b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/README.md new file mode 100644 index 00000000..4894f9b9 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/README.md @@ -0,0 +1,181 @@ +# Private single-GPU LaunchPad setup + +This Linux CLI reuses the parent launchable's pinned serving image/model command, +controller, manifest, internal dataset preparation and native VQA scorer. It does +not create or terminate an instance, expose a host model port, start a notebook, +or modify existing Jupyter/Traefik services. Use your provider's authorized SSH +connection. No instance locator or credentials belong in this source tree. + +## Requirements + +- Linux x86-64 host, NVIDIA driver/container runtime, Docker with Compose v2+, + Python 3.9+, and one free GPU with sufficient memory for the pinned Nano model. +- This VLMEvalKit source tree, plus an existing `linux/amd64` evaluator image + built from the parent `launchable/Dockerfile`. Setup does not rebuild images. + Dependency input hashes must match the source tree; mismatches fail explicitly. +- For private internal data, authorized `team-cosmos` PDX S3 access. Public/HF + model access is anonymous unless the actual model requires `HF_TOKEN`. + +If a matching evaluator image does not already exist, build it explicitly from +VLMEvalKit (not during another launchable build): + +```bash +docker build -f launchable/Dockerfile \ + --build-arg COSMOS_COMMIT="$(git rev-parse HEAD)" \ + -t cosmos3-vantage-eval:local . +``` + +The image supplies dependencies; the current source tree is bind-mounted +read-only into each evaluator/preparation container. Provenance records both the +base Git commit and a SHA-256 source-tree digest, alongside the exact dependency +image ID and model image/command. The image's baked commit is not passed off as +runtime source provenance. For an exported tree without `.git`, supply +`--commit FULL_BASE_REVISION`; the separate digest still identifies local changes. + +The source root must be source-only: setup refuses `.env*`, runtime `state` or +`credentials` directories, `.aws`, `.ssh`, `.cache`, `.venv`, `.git`, key-like +files, and symlinks escaping the source tree. These files are never hashed or +mounted into inference. If an old launchable wrote `.env` or `state` under its +source, use a new clean export (for example a Git archive plus your reviewed +source changes), not the old directory. Do not delete or mutate retained secrets +or state to make an old source directory pass. + +## Configure and start + +Run from the VLMEvalKit root. Keep state outside the source tree, use a unique +project, and choose exactly one free GPU index or UUID: + +```bash +LP=(python3 launchable/launchpad/launchpad.py \ + --state-dir "$HOME/cosmos-launchpad-state" \ + --project cosmos-launchpad --gpu 0 --image cosmos3-vantage-eval:local) +"${LP[@]}" up +"${LP[@]}" status +"${LP[@]}" check +``` + +`up` preserves weights, datasets and outputs, starts only the project's pinned +endpoint and waits for the existing controller's model readiness check. Running +it again resumes the same project without rebuilding. Neither the model API nor +an evaluator/notebook port is published. GPU selection overrides the parent's +single-device reservation without changing its serving flags. The private +inference network has Docker `internal: true`; the serving container also has a +download network for first-time model retrieval. + +## Internal data: choose the route that actually has PDX access + +Some LaunchPad hosts cannot reach `pdx.s8k.io:443`, even when their SSH and +Hugging Face access work. A connection timeout is not evidence of bad credentials. +Use the workstation-staging route below for such hosts. Direct remote preparation +requires a working network route to PDX; provisioning a profile does not add one. + +### Direct preparation on a PDX-reachable host + +On the local workstation with the working AWS CLI profile, run: + +```bash +python3 launchable/launchpad/provision_profile.py \ + --host YOUR_SSH_HOST --user YOUR_SSH_USER \ + --remote-dir /absolute/state/path/credentials/team-cosmos +``` + +`--remote-dir` must be the chosen state directory followed by +`/credentials/team-cosmos`. If your SSH gateway needs an authenticated master +session, establish it normally first, then supply the same `--control-path`. +The script captures only `aws configure export-credentials --profile team-cosmos` +in private process memory and sends it over SSH stdin. It never copies `~/.aws`, +uses the workspace `.env`, prints values, or puts them in command arguments. +The remote directory is mode 0700; its two isolated files are mode 0600. + +```bash +"${LP[@]}" probe +"${LP[@]}" prepare +``` + +`probe` performs one bounded VQA prefix listing and a one-byte ranged object +read. `prepare` calls the original internal VQA dataset constructor using that +profile. Preparation can be repeated against its existing native local dataset +layout. No S3 upload operation is invoked by either command. Interruptions during +a first download may require inspection of the native dataset's partial local +files; this wrapper does not delete data or claim atomic upstream downloads. + +**Credential isolation:** only the one-shot preparation/probe service mounts the +isolated credentials. Inference/scoring have no credential mount, use `/dev/null` +for both AWS files, disable instance metadata credentials, and have no outbound +network. This matters because the upstream video cache attempts S3 cache uploads +on misses. Private inference can generate local cache entries but cannot +access/write S3. Do not mount a home directory or add AWS credentials to the +inference service; a read-only credentials mount is not a read-only S3 policy. + +### Workstation preparation and data-only transfer + +On a PDX-reachable workstation with AWS CLI, Docker, rsync and the matching +evaluator image, run from a source-only VLMEvalKit tree: + +```bash +python3 launchable/launchpad/stage_internal.py \ + --local-state-dir "$HOME/private-cosmos-vqa-staging" \ + --image cosmos3-vantage-eval:local \ + --host YOUR_SSH_HOST --user YOUR_SSH_USER \ + --remote-state-dir /absolute/launchpad/state/path +``` + +The remote state path must be the same one passed to `launchpad.py`. Supply +`--control-path` when reusing an authenticated SSH gateway session. This helper +reads the official adapter's VQA S3 dispatch, uses native `aws s3 sync` with only +the authorized `team-cosmos` profile to download/resume its unchanged directory +layout, then invokes the original adapter's annotation-to-TSV method and original +controller preparation in a credential-free, network-disabled evaluator +container. It checks all referenced native video paths and retains its private +native preparation log under the private local state directory. + +Only `data/internal/LMUData/datasets/VANTAGE_VQA/` is transferred by rsync. No +credentials, home directory, cache credential files, results or public data are +transferred. Local state is mode0700; transferred directories/files are +mode0700/0600. The helper has no `--delete`: rerunning the same command resumes +downloads/transfers and preserves existing remote results. No proxy, public +data service, S3 upload or network bypass is installed. + +After transfer, on LaunchPad run `"${LP[@]}" prepare` to exercise the original +controller against its native local dataset layout, then run the smoke below. +This is local-cache preparation, not a claim that LaunchPad downloaded from PDX. +The remote `probe` still requires PDX connectivity and is expected to time out on +a host without that route; the staged route does not require remote credentials. + +## Five-question internal smoke and native score + +After preparation and endpoint readiness: + +```bash +"${LP[@]}" smoke --run-name internal-vqa-smoke-001 --score +``` + +The original controller selects five seeded VQA questions and performs native +inference. Independent acceptance rejects empty/API/video-load failures, requires +exactly five unique predictions with the expected native sample IDs, and checks +that scoring labels match the prepared native dataset. `--score` invokes its +native local scorer only when all five answers actually exist, and verifies the +native metrics account for all five samples. Omit `--score` for inference-only. + +Outputs remain private under `state/results/RUN_NAME`: native predictions, +logs, configs, native scoring files when requested, `run.json`, +`launchpad-acceptance.json`, and `launchpad-provenance.json`. Labels and model +responses must not be publicly exported. A five-question smoke is not a full +benchmark score or published-reference parity. Existing output directories are +refused; choose a fresh run name instead of overwriting or merging results. + +## Stop and resume safely + +```bash +"${LP[@]}" stop +"${LP[@]}" status +"${LP[@]}" up +"${LP[@]}" smoke --run-name internal-vqa-smoke-002 --score +``` + +`stop` targets only this project's endpoint container. It does not remove +containers, state, cached weights, data, predictions, unrelated services or the +instance. An allocated LaunchPad instance can continue accruing provider costs +when its containers are stopped. Keep the same project/state/GPU arguments on +resume. `status` and `stop` use the retained config and do not require an image +rebuild or source-provenance check. There is deliberately no delete/purge command. diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/launchpad.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/launchpad.py new file mode 100755 index 00000000..3c58e6d2 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/launchpad.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Private single-GPU LaunchPad lifecycle using the existing launchable controller.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +HERE = Path(__file__).resolve().parent + + +def call(args, **kwargs): + return subprocess.run(args, check=True, **kwargs) + + +def capture(args, **kwargs): + return subprocess.check_output(args, text=True, **kwargs).strip() + + +def source_digest(root): + digest = hashlib.sha256() + forbidden = {'.git', '.aws', '.ssh', '.cache', '.venv', 'state', 'credentials'} + for path in sorted(root.rglob('*')): + relative = path.relative_to(root) + if any(part in forbidden or part.startswith('.env') for part in relative.parts): + raise ValueError(f'Source contains runtime/private artifacts: {relative}; use a fresh source-only export, without modifying retained state') + if path.is_symlink() and root not in path.resolve().parents: + raise ValueError(f'Source symlink escapes the source-only tree: {relative}') + if path.suffix in {'.pem', '.key'}: + raise ValueError(f'Private-key-like source file refused: {relative}') + if not path.is_file() or any(part in {'__pycache__', '.ipynb_checkpoints'} for part in relative.parts): + continue + if path.suffix in {'.pyc', '.pyo'}: + continue + digest.update(str(relative).encode() + b'\0') + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def configure(args): + root, state = args.source_root.resolve(), args.state_dir.resolve() + if state == root or root in state.parents: + raise ValueError('Keep --state-dir outside the source tree') + if not re.fullmatch(r'[a-z0-9][a-z0-9_-]+', args.project): + raise ValueError('Use a lowercase Compose project name of at least two characters') + commit = args.commit or capture(['git', '-C', str(root), 'rev-parse', 'HEAD']) + if not re.fullmatch(r'[0-9a-f]{40}', commit): + raise ValueError('--commit must identify the full base source revision; runtime source is additionally hashed') + digest = source_digest(root) + inventory = [tuple(value.strip() for value in line.split(',')) for line in capture( + ['nvidia-smi', '--query-gpu=index,uuid', '--format=csv,noheader,nounits']).splitlines()] + selected = [uuid for index, uuid in inventory if args.gpu in {index, uuid}] + if len(selected) != 1: + raise ValueError('--gpu must resolve to exactly one actual nvidia-smi index or UUID; all/lists are forbidden') + gpu = selected[0] + image = json.loads(capture(['docker', 'image', 'inspect', args.image]))[0] + if image['Architecture'] != 'amd64': + raise ValueError('The evaluator image must be linux/amd64') + # Reuse dependencies only when their declared inputs match. Runtime source is + # mounted explicitly; an old image's baked COSMOS_COMMIT is never trusted. + inputs = ['requirements.txt', 'setup.py', 'setup.cfg', 'launchable/Dockerfile', 'launchable/public-harness.patch'] + program = 'import hashlib,json,pathlib; print(json.dumps({p:hashlib.sha256(pathlib.Path("/opt/vlmevalkit",p).read_bytes()).hexdigest() for p in ' + repr(inputs) + '}))' + baked = json.loads(capture(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', 'python', args.image, '-c', program])) + for name in inputs: + if hashlib.sha256((root / name).read_bytes()).hexdigest() != baked[name]: + raise ValueError(f'Prebuilt dependency input differs: {name}; build a matching evaluator image explicitly') + env = dict(os.environ, JUPYTER_TOKEN=os.urandom(32).hex(), LAUNCHABLE_STATE_DIR=str(state), COSMOS_COMMIT=commit) + base = json.loads(capture(['docker', 'compose', '--env-file', '/dev/null', '--profile', 'local', '-f', str(root / 'launchable/compose.yaml'), 'config', '--format', 'json'], env=env)) + model = base['services']['cosmos-endpoint'] + model.pop('profiles', None) + model['deploy']['resources']['reservations']['devices'] = [{'driver': 'nvidia', 'device_ids': [gpu], 'capabilities': ['gpu']}] + model['networks'] = ['private', 'download'] + model['volumes'] = [f'{state}/cache:/cache', f'{state}/data:/data:ro'] + common = { + 'image': args.image, 'platform': 'linux/amd64', + 'working_dir': '/opt/vlmevalkit/launchable', + 'environment': {'COSMOS_COMMIT': commit, 'COSMOS_SOURCE_SHA256': digest, + 'HF_HOME': '/cache/huggingface', 'HOME': '/tmp/cosmos-home', + 'PYTHONUNBUFFERED': '1', 'AWS_EC2_METADATA_DISABLED': 'true', + 'COSMOS_API_BASE': 'http://cosmos-endpoint:8080/v1/chat/completions', + 'COSMOS_MODEL': base['services']['vantage-eval']['environment']['COSMOS_MODEL'], + 'COSMOS_API_KEY': 'EMPTY'}, + 'volumes': [f'{root}:/opt/vlmevalkit:ro', f'{state}/cache:/cache', f'{state}/data:/data', f'{state}/results:/results'], + 'entrypoint': ['python', 'launchable.py'], + } + evaluate = dict(common, networks=['private']) + evaluate['environment'] = dict(common['environment'], AWS_CONFIG_FILE='/dev/null', AWS_SHARED_CREDENTIALS_FILE='/dev/null') + prepare = dict(common, networks=['download']) + prepare['environment'] = dict(common['environment'], AWS_CONFIG_FILE='/credentials/config', AWS_SHARED_CREDENTIALS_FILE='/credentials/credentials', AWS_PROFILE='team-cosmos') + prepare['volumes'] = common['volumes'] + [f'{state}/credentials/team-cosmos:/credentials:ro'] + config = {'services': {'cosmos-endpoint': model, 'evaluate': evaluate, 'prepare': prepare}, + 'networks': {'private': {'internal': True}, 'download': {}}} + state.mkdir(mode=0o700, parents=True, exist_ok=True) + for name in ['cache', 'data', 'results', 'credentials/team-cosmos']: + (state / name).mkdir(mode=0o700, parents=True, exist_ok=True) + path = state / 'launchpad-compose.json' + path.write_text(json.dumps(config, indent=2) + '\n') + path.chmod(0o600) + evidence = {'cosmos_commit': commit, 'source_sha256': common['environment']['COSMOS_SOURCE_SHA256'], + 'source_root': str(root), 'evaluator_image': args.image, 'evaluator_image_id': image['Id'], + 'dependency_inputs_match': True, 'runtime_source_bind_mounted_read_only': True, + 'serving_image': model['image'], 'serving_command': model['command'], + 'gpu': gpu, 'project': args.project, 'state_dir': str(state), + 'inference_credentials_mounted': False, 'inference_network_internal': True} + (state / 'launchpad-provenance.json').write_text(json.dumps(evidence, indent=2) + '\n') + return state, ['docker', 'compose', '--project-name', args.project, '-f', str(path)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--state-dir', required=True, type=Path) + parser.add_argument('--project', default='cosmos-launchpad') + parser.add_argument('--gpu', default='0', help='One GPU index or UUID') + parser.add_argument('--image', default='cosmos3-vantage-eval:local', help='Existing, dependency-compatible evaluator image') + parser.add_argument('--source-root', type=Path, default=HERE.parents[1], help='VLMEvalKit root in a Git checkout') + parser.add_argument('--commit', help='Full base Git revision for an exported tree; working changes are identified by the separate source digest') + sub = parser.add_subparsers(dest='command', required=True) + for name in ['up', 'status', 'stop', 'check', 'probe', 'prepare']: + sub.add_parser(name) + smoke = sub.add_parser('smoke') + smoke.add_argument('--run-name', required=True, help='Fresh directory name under state/results') + smoke.add_argument('--score', action='store_true', help='Native local scoring; requires actual internal ground truth') + args = parser.parse_args() + if args.command in {'status', 'stop'}: + config = args.state_dir.resolve() / 'launchpad-compose.json' + if not config.is_file(): + raise ValueError('No configured project at this state path; run up first') + compose = ['docker', 'compose', '--project-name', args.project, '-f', str(config)] + call(compose + (['ps', '--all'] if args.command == 'status' else ['stop', 'cosmos-endpoint'])) + return + state, compose = configure(args) + if args.command == 'up': + call(compose + ['up', '-d', '--no-build', 'cosmos-endpoint']) + call(compose + ['run', '--rm', '--no-deps', 'evaluate', 'check', '--wait', '1800']) + elif args.command == 'check': + call(compose + ['run', '--rm', '--no-deps', 'evaluate', 'check', '--wait', '1800']) + elif args.command == 'probe': + call(compose + ['run', '--rm', '--no-deps', '--entrypoint', 'python', 'prepare', 'launchpad/runtime.py', 'probe']) + elif args.command == 'prepare': + call(compose + ['run', '--rm', '--no-deps', 'prepare', 'prepare', '--source', 'internal', '--benchmarks', 'VANTAGE_VQA']) + else: + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', args.run_name): + raise ValueError('Use a simple run directory name') + if (state / 'results' / args.run_name).exists(): + raise ValueError('Run directory already exists; select a fresh --run-name') + work = '/results/' + args.run_name + call(compose + ['run', '--rm', '--no-deps', 'evaluate', 'run', '--source', 'internal', '--benchmarks', 'VANTAGE_VQA', '--smoke', '--mode', 'infer', '--work-dir', work]) + call(compose + ['run', '--rm', '--no-deps', '--entrypoint', 'python', 'evaluate', 'launchpad/runtime.py', 'accept', '--work-dir', work] + (['--score'] if args.score else [])) + # Attach exact runtime source/dependency provenance to private outputs. + provenance = json.loads((state / 'launchpad-provenance.json').read_text()) + call(compose + ['run', '--rm', '--no-deps', '--entrypoint', 'python', 'evaluate', '-c', 'import pathlib,sys; pathlib.Path(sys.argv[1]).write_text(sys.argv[2])', work + '/launchpad-provenance.json', json.dumps(provenance, indent=2) + '\n']) + + +if __name__ == '__main__': + try: + main() + except (ValueError, subprocess.CalledProcessError) as exc: + print(f'LaunchPad setup failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/provision_profile.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/provision_profile.py new file mode 100755 index 00000000..6bf127f2 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/provision_profile.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export only team-cosmos privately through SSH stdin; never copy ~/.aws.""" +import argparse +import json +import shlex +import subprocess + + +RECEIVER = r''' +import configparser, json, os, pathlib, sys +payload = json.load(sys.stdin) +directory = pathlib.Path(payload['directory']).expanduser() +if not directory.is_absolute(): + raise ValueError('Remote credentials directory must be absolute') +if directory.name != 'team-cosmos' or directory.parent.name != 'credentials': + raise ValueError('Remote target must end with credentials/team-cosmos; refusing unrelated profile directories') +if any(path.is_symlink() for path in (directory, *directory.parents)): + raise ValueError('Remote credentials path must not contain symlinks') +directory.mkdir(mode=0o700, parents=True, exist_ok=True) +directory.chmod(0o700) +credentials = configparser.ConfigParser(interpolation=None) +values = payload['credentials'] +credentials['team-cosmos'] = {'aws_access_key_id': values['AccessKeyId'], 'aws_secret_access_key': values['SecretAccessKey']} +if values.get('SessionToken'): + credentials['team-cosmos']['aws_session_token'] = values['SessionToken'] +config = configparser.ConfigParser(interpolation=None) +config['profile team-cosmos'] = {'region': 'us-east-1', 'endpoint_url': 'https://pdx.s8k.io', 's3': '\nsignature_version = s3v4\npayload_signing_enabled = true\naddressing_style = path'} +for name, document in [('credentials', credentials), ('config', config)]: + path = directory / name + # Refuse symlinks and create with restrictive mode before writing bytes. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + os.fchmod(fd, 0o600) + with os.fdopen(fd, 'w') as stream: + document.write(stream) +print('Provisioned only team-cosmos; directory 0700, files 0600; no secret output.') +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', required=True, help='SSH host, from your authorized instance locator') + parser.add_argument('--user', help='SSH user if not in SSH config') + parser.add_argument('--remote-dir', required=True, help='Absolute state-dir/credentials/team-cosmos on the host') + parser.add_argument('--control-path', help='Existing SSH ControlPath, if used') + args = parser.parse_args() + # Do not print/export credential values or pass them in argv/environment. + exported = subprocess.run(['aws', 'configure', 'export-credentials', '--profile', 'team-cosmos', '--format', 'process'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + credentials = json.loads(exported.stdout) + payload = json.dumps({'directory': args.remote_dir, 'credentials': credentials}).encode() + command = ['ssh'] + if args.user: + command += ['-l', args.user] + if args.control_path: + command += ['-o', 'ControlMaster=auto', '-o', 'ControlPersist=10m', '-o', 'ControlPath=' + args.control_path] + command += [args.host, 'python3 -c ' + shlex.quote(RECEIVER)] + subprocess.run(command, input=payload, check=True) + + +if __name__ == '__main__': + main() diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/runtime.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/runtime.py new file mode 100755 index 00000000..c6bb7f8e --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/runtime.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded S3 read probe and independent five-question internal smoke acceptance.""" +import argparse +import contextlib +import json +import os +from pathlib import Path +import sys + +LAUNCHABLE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(LAUNCHABLE)) + + +def probe(): + import boto3 + from botocore.config import Config + + client = boto3.Session(profile_name='team-cosmos').client( + 's3', endpoint_url='https://pdx.s8k.io', region_name='us-east-1', + config=Config(signature_version='s3v4', s3={'addressing_style': 'path', 'payload_signing_enabled': True}, + connect_timeout=10, read_timeout=30, retries={'max_attempts': 2})) + response = client.list_objects_v2(Bucket='cosmos_understanding', Prefix='benchmark/vantage_benchmark_hf_release_annotations/VQA/', MaxKeys=16) + objects = [item for item in response.get('Contents', []) if item['Size'] > 0] + if not objects: + raise RuntimeError('Bounded VQA listing returned no nonempty object') + body = client.get_object(Bucket='cosmos_understanding', Key=objects[0]['Key'], Range='bytes=0-0')['Body'] + try: + if len(body.read()) != 1: + raise RuntimeError('Ranged read did not return exactly one byte') + finally: + body.close() + print(json.dumps({'vqa_bounded_listing': True, 'one_byte_get': True, 's3_writes': False})) + + +def accept(work, score): + import pandas as pd + import launchable + + # No credentials, metadata provider, or non-local network are available to + # this service. The upstream cache may attempt S3 initialization, but cannot + # authenticate or reach S3; generated entries stay in the local cache. + import boto3 + if boto3.Session().get_credentials() is not None: + raise RuntimeError('Inference unexpectedly has AWS credentials') + if os.environ.get('AWS_CONFIG_FILE') != '/dev/null' or os.environ.get('AWS_SHARED_CREDENTIALS_FILE') != '/dev/null': + raise RuntimeError('Inference credential files are not disabled') + paths = [path for path in launchable.prediction_files(work) if path.stem == 'cosmos_VANTAGE_VQA'] + if len(paths) != 1: + raise RuntimeError(f'Expected one fresh native prediction file; found {len(paths)}') + path = paths[0] + frame = pd.read_csv(path, sep='\t') if path.suffix == '.tsv' else pd.read_json(path) if path.suffix == '.json' else pd.read_excel(path) + if len(frame) != 5 or 'prediction' not in frame or 'index' not in frame: + raise RuntimeError('Smoke must contain exactly five indexed predictions') + values = frame['prediction'].fillna('').astype(str).str.strip() + markers = ['VIDEO_LOAD_FAILED', 'Failed to obtain answer via API', 'Traceback (most recent call last)', 'Internal Server Error', 'ConnectionError'] + if values.eq('').any() or any(values.str.contains(marker, regex=False, case=False).any() for marker in markers): + raise RuntimeError('Empty, video-load or API-failure prediction rejected') + for log in work.rglob('run.log'): + content = log.read_text(errors='replace') + if any(marker.lower() in content.lower() for marker in markers): + raise RuntimeError('Failure marker in native run log') + sys.path.insert(0, str(launchable.ROOT)) + sys.path.insert(0, str(launchable.KIT)) + import run_all + from run import build_dataset_from_config + + cfg = run_all.build_config(run_all.load_model_layer('cosmos'), 'metropolis', 'VANTAGE_VQA', render=False) + cfg['data']['VANTAGE_VQA'].update(limit=5, random_state=0) + # Dataset initialization and native scorer may print private labels. Retain + # that output only alongside private results, never in the console summary. + with (work / 'native-score.log').open('w') as log, contextlib.redirect_stdout(log), contextlib.redirect_stderr(log): + os.environ['LMUData'] = '/data/internal/LMUData' + dataset = build_dataset_from_config(cfg['data'], 'VANTAGE_VQA') + expected = [str(value) for value in dataset.data['index']] + actual = [str(value) for value in frame['index']] + if len(expected) != 5 or len(set(actual)) != 5 or set(expected) != set(actual): + raise RuntimeError('Missing, duplicate, dropped or unexpected sample IDs') + native = None + if score: + if 'answer' not in frame or frame['answer'].isna().any() or frame['answer'].astype(str).str.strip().eq('').any(): + raise RuntimeError('Native scoring requires real ground truth for all five samples') + expected_answers = dataset.data.set_index('index')['answer'].astype(str).to_dict() + if any(str(row['answer']) != expected_answers[row['index']] for _, row in frame.iterrows()): + raise RuntimeError('Prediction ground truth differs from the prepared native dataset') + summary = dataset.evaluate(str(path), verbose=False) + metrics = pd.read_csv(path.with_name(path.stem + '_metrics.csv')) + overall = metrics.loc[metrics['Category'] == 'Overall'].iloc[0] + native = {'acc': float(summary['Overall']), 'correct': int(overall['Correct']), 'total': int(overall['Total'])} + if native['total'] != 5: + raise RuntimeError('Native scorer did not account for all five samples') + evidence = {'accepted': True, 'questions': 5, 'sample_indexes': sorted(actual), + 'source': 'internal', 'cosmos_commit': os.environ['COSMOS_COMMIT'], + 'source_sha256': os.environ['COSMOS_SOURCE_SHA256'], 'aws_credentials_available': False, + 'inference_network_internal': True, 'native_overall': native, + 'scope': 'five-question smoke only; not published-reference parity'} + (work / 'launchpad-acceptance.json').write_text(json.dumps(evidence, indent=2) + '\n') + print(json.dumps(evidence, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + sub.add_parser('probe') + check = sub.add_parser('accept') + check.add_argument('--work-dir', type=Path, required=True) + check.add_argument('--score', action='store_true') + args = parser.parse_args() + probe() if args.command == 'probe' else accept(args.work_dir, args.score) + + +if __name__ == '__main__': + main() diff --git a/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/stage_internal.py b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/stage_internal.py new file mode 100755 index 00000000..229f5b72 --- /dev/null +++ b/evaluation/cosmos3/reasoner/vlmevalkit/launchable/launchpad/stage_internal.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Prepare internal VQA on a PDX-reachable workstation; transfer only native data.""" +from __future__ import annotations + +import argparse +import ast +import json +import os +from pathlib import Path, PurePosixPath +import shlex +import subprocess + +from launchpad import source_digest + + +NATIVE_PREPARE = r''' +import json, pathlib, sys +sys.path.insert(0, '/opt/vlmevalkit') +sys.path.insert(0, '/opt/vlmevalkit/cosmos_eval') +from vlmeval.dataset.metropolis_vqa import MetropolisVQA +import launchable +import run_all +from run import build_dataset_from_config +root = pathlib.Path('/data/internal/LMUData/datasets/VANTAGE_VQA') +converter = MetropolisVQA(dataset='VANTAGE_VQA', test_mode=True) +converter._generate_tsv_from_annotations(None, '', str(root), 'VANTAGE_VQA') +launchable.prepare(['VANTAGE_VQA']) +cfg = run_all.build_config(run_all.load_model_layer('cosmos'), 'metropolis', 'VANTAGE_VQA', render=False) +dataset = build_dataset_from_config(cfg['data'], 'VANTAGE_VQA') +for _, row in dataset.data.iterrows(): + videos = [part['value'] for part in dataset.build_prompt(row, video_llm=True) if part['type'] == 'video'] + if len(videos) != 1 or not pathlib.Path(videos[0]).is_file(): + raise RuntimeError('Native dataset references an unavailable video') +print(json.dumps({'native_prepared_questions':len(dataset.data),'native_video_count':len(dataset.videos),'all_referenced_videos_present':True})) +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--local-state-dir', required=True, type=Path, help='Private workstation data/cache/log directory outside source') + parser.add_argument('--source-root', type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument('--image', default='cosmos3-vantage-eval:local') + parser.add_argument('--host', required=True) + parser.add_argument('--user') + parser.add_argument('--control-path') + parser.add_argument('--remote-state-dir', required=True, help='Absolute isolated LaunchPad state directory') + args = parser.parse_args() + root, state = args.source_root.resolve(), args.local_state_dir.resolve() + source_digest(root) # Refuse secret-bearing source before any container mount. + if state == root or root in state.parents: + raise ValueError('Local state must be outside source') + remote_state = PurePosixPath(args.remote_state_dir) + if not remote_state.is_absolute() or str(remote_state) == '/' or '..' in remote_state.parts: + raise ValueError('Use an absolute isolated remote state directory') + # Read canonical dispatch without importing the heavyweight evaluation stack + # on the workstation or maintaining a second benchmark-source table. + tree = ast.parse((root / 'vlmeval/dataset/metropolis_vqa.py').read_text()) + cls = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'MetropolisVQA') + table = next(node.value for node in cls.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == '_S3_PATHS' for target in node.targets)) + bucket, prefix, profile = ast.literal_eval(table)['VANTAGE_VQA'] + if profile != 'team-cosmos': + raise ValueError('Only the explicitly authorized team-cosmos profile is supported') + state.mkdir(mode=0o700, parents=True, exist_ok=True) + state.chmod(0o700) + data = state / 'data' + dataset = data / 'internal/LMUData/datasets/VANTAGE_VQA' + dataset.mkdir(mode=0o700, parents=True, exist_ok=True) + cache = state / 'cache' + cache.mkdir(mode=0o700, exist_ok=True) + subprocess.run(['aws', '--profile', profile, '--endpoint-url', 'https://pdx.s8k.io', 's3', 'sync', + f's3://{bucket}/{prefix}/', str(dataset), '--only-show-errors'], check=True) + log_path = state / 'native-staged-prepare.log' + fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w') as log: + subprocess.run(['docker', 'run', '--rm', '--platform', 'linux/amd64', '--network', 'none', + '-v', f'{root}:/opt/vlmevalkit:ro', '-v', f'{data}:/data', '-v', f'{cache}:/cache', + '-e', 'LMUData=/data/internal/LMUData', '-e', 'HOME=/tmp/cosmos-home', + '-e', 'AWS_CONFIG_FILE=/dev/null', '-e', 'AWS_SHARED_CREDENTIALS_FILE=/dev/null', + '-e', 'AWS_EC2_METADATA_DISABLED=true', '--entrypoint', 'python', args.image, + '-c', NATIVE_PREPARE], stdout=log, stderr=subprocess.STDOUT, check=True) + if any(path.is_symlink() for path in dataset.rglob('*')): + raise ValueError('Refusing symlinks in the private dataset transfer') + ssh = ['ssh'] + if args.user: + ssh += ['-l', args.user] + if args.control_path: + ssh += ['-o', 'ControlMaster=auto', '-o', 'ControlPersist=10m', '-o', 'ControlPath=' + args.control_path] + target = str(remote_state / 'data/internal/LMUData/datasets/VANTAGE_VQA') + subprocess.run(ssh + [args.host, 'umask 077; mkdir -p -- ' + shlex.quote(target) + ' && chmod 700 -- ' + shlex.quote(target)], check=True) + # No --delete, no credentials/cache/home directory and no result paths. + subprocess.run(['rsync', '-rlt', '--partial', '--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=', '-e', shlex.join(ssh), + str(dataset) + '/', args.host + ':' + shlex.quote(target) + '/'], check=True) + print(json.dumps({'native_preparation_passed': True, 'private_log': str(log_path), + 'transferred_only': 'data/internal/LMUData/datasets/VANTAGE_VQA', + 'remote_state_dir': str(remote_state), 'credentials_transferred': False, + 's3_writes': False, 'resume': 'Rerun the same command; aws sync and rsync preserve completed files; no --delete'})) + + +if __name__ == '__main__': + main()