From 53e3a1527bd6a3e23e2388956de6cacf60c445e3 Mon Sep 17 00:00:00 2001 From: JIN-strong <1228766517@qq.com> Date: Wed, 14 Oct 2026 17:50:48 +0800 Subject: [PATCH 1/2] fix(paibench): count shard rows correctly in paibench_c generation The shard JSONLs and input.jsonl were written without a trailing newline and pending tasks were counted with wc -l, which counts newline characters, so a shard holding exactly one pending row counted as zero. The documented 4-task quick start interleaves one task per GPU shard and therefore generated no videos at all, and a resumed 600-task run silently dropped the final task on every GPU whose shard was left with a single row. Write every JSONL with a trailing newline and count pending rows with awk END{NR}, which is correct for unterminated and empty files alike. Also guard the worker-pid loop expansion so a fully-complete resume cannot abort under set -u on bash < 4.4. Add test_run_paibench_c.sh, which extracts the real build_jsonl_and_run function from the script and drives it with a stub inference entrypoint; it reproduces the zero-video quick start before the fix and passes afterwards. Signed-off-by: JIN-strong <1228766517@qq.com> --- .../generator/paibench_c/run_paibench_c.sh | 17 +- .../paibench_c/test_run_paibench_c.sh | 180 ++++++++++++++++++ 2 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh diff --git a/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh b/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh index 793aedbb..8777a0d1 100755 --- a/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh +++ b/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh @@ -540,7 +540,9 @@ for tid in task_ids: dst = output_dir / "input.jsonl" dst.parent.mkdir(parents=True, exist_ok=True) -dst.write_text("\n".join(json.dumps(r) for r in rows)) +# Trailing newline: wc -l and line-oriented tools count the final row only +# when the file ends with one. +dst.write_text("".join(json.dumps(r) + "\n" for r in rows)) print(f"Wrote {len(rows)} rows → {dst}") PY @@ -609,7 +611,8 @@ for row in shard_rows: vision_mp4.unlink() pending.append(row) -shard_jsonl.write_text("\n".join(json.dumps(r) for r in pending)) +# Trailing newline so line counting (wc -l) sees every pending row. +shard_jsonl.write_text("".join(json.dumps(r) + "\n" for r in pending)) skipped = len(shard_rows) - len(pending) print(f"Shard {shard_idx}/{num_shards}: {len(pending)} to generate, {skipped} already done") PY @@ -620,7 +623,9 @@ PY for (( gi=0; gi<_num_gpus; gi++ )); do local _shard_jsonl="${_shard_jsonls[$gi]}" local _pending_count - _pending_count="$(wc -l < "$_shard_jsonl" | tr -d ' ')" + # NR counts the last row even without a trailing newline (and unlike + # wc -l, awk exits 0 on an empty file under set -o pipefail). + _pending_count="$(awk 'END{print NR}' "$_shard_jsonl")" local _gpu_id="${_gpu_ids[$gi]}" if [[ "$_pending_count" -eq 0 ]]; then @@ -645,9 +650,11 @@ PY _worker_pids+=("$!") done - # Wait for all GPU workers to finish. + # Wait for all GPU workers to finish. The ${var+...} expansion keeps the + # loop a no-op (instead of an "unbound variable" abort under set -u on + # bash < 4.4) when every shard was already complete. local _fail=0 - for _pid in "${_worker_pids[@]}"; do + for _pid in ${_worker_pids[@]+"${_worker_pids[@]}"}; do wait "$_pid" || _fail=1 done [[ "$_fail" -eq 0 ]] || die "One or more GPU workers failed for modality=$MODALITY" diff --git a/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh b/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh new file mode 100644 index 00000000..7da921ac --- /dev/null +++ b/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 +# +# Regression tests for run_paibench_c.sh generation-shard bookkeeping. +# +# The tests extract the real functions from run_paibench_c.sh (no logic is +# duplicated here) and drive them with a stub python that fakes the +# cosmos-framework inference entrypoint. No GPU, network, or venv is needed. +# +# Run: bash test_run_paibench_c.sh +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_UNDER_TEST="$HERE/run_paibench_c.sh" +[[ -f "$SCRIPT_UNDER_TEST" ]] || { echo "FAIL: $SCRIPT_UNDER_TEST not found" >&2; exit 1; } + +PASS=0 +FAIL=0 + +assert_eq() { # assert_eq + if [[ "$1" == "$2" ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $3" + echo " expected: $2" + echo " actual: $1" + fi +} + +assert_contains() { # assert_contains + if [[ "$1" == *"$2"* ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $3" + echo " haystack does not contain: $2" + fi +} + +# Extract one function definition verbatim from the script under test and +# evaluate it in this shell so the tests run the shipped code, not a copy. +load_fn() { + local _fn="$1" + local _src + _src="$(sed -n "/^${_fn}()/,/^}/p" "$SCRIPT_UNDER_TEST")" + [[ -n "$_src" ]] || { echo "FAIL: function ${_fn}() not found in $SCRIPT_UNDER_TEST" >&2; exit 1; } + eval "$_src" +} + +log() { echo "[test-log] $*"; } +die() { echo "[test-die] $*" >&2; exit 9; } + +# Stub venv: a bin dir whose python passes heredoc scripts through to the real +# python3 and fakes the framework inference entrypoint by materializing one +# valid (>50 KB) vision.mp4 per JSONL row. +make_stub_env() { + local _env="$1" + mkdir -p "$_env/bin" + cat > "$_env/bin/python" <<'STUB' +#!/usr/bin/env bash +if [[ "${1:-}" == "-" ]]; then + exec python3 - +fi +if [[ "${1:-}" == "-m" ]]; then + stub_shard=""; stub_raw="" + while [[ $# -gt 0 ]]; do + case "$1" in + -i) stub_shard="$2"; shift 2 ;; + -o) stub_raw="$2"; shift 2 ;; + *) shift ;; + esac + done + export stub_shard stub_raw + python3 - <<'FAKE' +import json, os, pathlib +shard, raw = pathlib.Path(os.environ["stub_shard"]), pathlib.Path(os.environ["stub_raw"]) +for line in shard.read_text().splitlines(): + if not line.strip(): + continue + name = json.loads(line)["name"] + d = raw / name + d.mkdir(parents=True, exist_ok=True) + (d / "vision.mp4").write_bytes(b"\x00" * 52000) +FAKE + exit 0 +fi +exec python3 "$@" +STUB + chmod +x "$_env/bin/python" +} + +# Shared fixture: 4 tasks on 4 GPUs through the real build_jsonl_and_run(). +run_generation() { # run_generation + local _out="$1" _logfile="$2" + PAIBENCH_C_ROOT="$HERE" \ + PAIBENCH_C_DATASET_ROOT="$PAIBENCH_C_OUTPUT_ROOT_FIXTURE_DATASET" \ + PAIBENCH_C_OUTPUT_ROOT="$_out" \ + PAIBENCH_C_CHECKPOINT="Cosmos3-Nano" \ + PAIBENCH_C_NUM_SAMPLES="4" \ + COSMOS3_UV_ENV="$PAIBENCH_C_OUTPUT_ROOT_FIXTURE_STUB" \ + COSMOS3_REPO="$PAIBENCH_C_OUTPUT_ROOT_FIXTURE_REPO" \ + TRITON_CACHE_DIR="$_out/triton" \ + CUDA_VISIBLE_DEVICES="0,1,2,3" \ + build_jsonl_and_run edge >"$_logfile" 2>&1 +} + +# --------------------------------------------------------------------------- +# Scenario 1: fresh 4-task / 4-GPU run — every shard must report 1 pending +# task (the wc -l bug counted 0 for a 1-row file without a trailing newline) +# and all four videos must be generated. +# --------------------------------------------------------------------------- +test_fresh_run_counts_single_row_shards() { + local _tmp; _tmp="$(mktemp -d)" + make_stub_env "$_tmp/stub-venv" + mkdir -p "$_tmp/dataset" "$_tmp/repo" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_DATASET="$_tmp/dataset" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_STUB="$_tmp/stub-venv" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_REPO="$_tmp/repo" + + load_fn build_jsonl_and_run + + local _rc=0 + run_generation "$_tmp/out" "$_tmp/run.log" || _rc=$? + + assert_eq "$_rc" "0" "fresh run exits 0" + assert_eq "$(wc -l < "$_tmp/out/Cosmos3-Nano/edge/input.jsonl" | tr -d ' ')" "4" \ + "input.jsonl has a trailing newline so wc -l counts all 4 rows" + for gpu in 0 1 2 3; do + assert_contains "$(cat "$_tmp/run.log")" "GPU $gpu: 1 tasks" \ + "GPU $gpu shard with exactly 1 pending row is counted as 1 task" + done + assert_eq "$(find "$_tmp/out/Cosmos3-Nano/edge/videos" -name '*.mp4' | wc -l)" "4" \ + "all 4 shard tasks were generated (none skipped as already done)" + assert_eq "$(find "$_tmp/out/Cosmos3-Nano/edge" -name 'shard_*.jsonl' | wc -l)" "0" \ + "worker shard files are cleaned up" + + rm -rf "$_tmp" +} + +# --------------------------------------------------------------------------- +# Scenario 2: fully-complete resume — every shard reports 0 pending, the +# worker-pids array stays empty, and the wait loop must not trip set -u +# (unbound variable) semantics on bash < 4.4. +# --------------------------------------------------------------------------- +test_resume_with_no_pending_tasks() { + local _tmp; _tmp="$(mktemp -d)" + make_stub_env "$_tmp/stub-venv" + mkdir -p "$_tmp/dataset" "$_tmp/repo" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_DATASET="$_tmp/dataset" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_STUB="$_tmp/stub-venv" + PAIBENCH_C_OUTPUT_ROOT_FIXTURE_REPO="$_tmp/repo" + + load_fn build_jsonl_and_run + + run_generation "$_tmp/out" "$_tmp/run1.log" + local _rc=0 + run_generation "$_tmp/out" "$_tmp/run2.log" || _rc=$? + + assert_eq "$_rc" "0" "fully-complete resume exits 0 under set -euo pipefail" + local _skipped + _skipped="$(grep -c 'all tasks done, skipping' "$_tmp/run2.log" || true)" + assert_eq "$_skipped" "4" "all four GPUs report nothing pending on resume" + assert_eq "$(find "$_tmp/out/Cosmos3-Nano/edge/videos" -name '*.mp4' | wc -l)" "4" \ + "resume regenerates nothing and keeps the four videos" + + rm -rf "$_tmp" +} + +# --------------------------------------------------------------------------- +run_tests() { + test_fresh_run_counts_single_row_shards + test_resume_with_no_pending_tasks + echo + echo "passed: $PASS failed: $FAIL" + [[ "$FAIL" -eq 0 ]] +} + +run_tests From 450506cfd50b6bb5da8b4f2ef138a9749ff8a716 Mon Sep 17 00:00:00 2001 From: JIN-strong <1228766517@qq.com> Date: Wed, 14 Oct 2026 17:56:34 +0800 Subject: [PATCH 2/2] fix(paibench): refuse incomplete paibench_c evaluation coverage Evaluation ran on whatever videos happened to exist under the output tree and only failed when none were present, so a run interrupted near the end (or one whose final shard tasks were dropped) computed metrics over the subset and printed them as the benchmark result with no signal. Wrap the per-modality evaluation step in a function and gate it on the requested task set: the collected video count must match the row count of that modality's input.jsonl (falling back to PAIBENCH_C_NUM_SAMPLES for trees built before this file existed), and any mismatch aborts with both numbers and the recovery paths. Document the resume and coverage behavior in the README. Extend test_run_paibench_c.sh to drive the real evaluate_modality function with a stub torchrun: full coverage evaluates and reports metrics, while a missing video is rejected before any evaluation work starts. Signed-off-by: JIN-strong <1228766517@qq.com> --- .../cosmos3/generator/paibench_c/README.md | 2 + .../generator/paibench_c/run_paibench_c.sh | 29 ++++-- .../paibench_c/test_run_paibench_c.sh | 92 ++++++++++++++++++- 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/evaluation/cosmos3/generator/paibench_c/README.md b/evaluation/cosmos3/generator/paibench_c/README.md index 190526e3..993b0f7d 100644 --- a/evaluation/cosmos3/generator/paibench_c/README.md +++ b/evaluation/cosmos3/generator/paibench_c/README.md @@ -52,6 +52,8 @@ SKIP_EVAL=1 bash run_paibench_c.sh SKIP_GEN=1 bash run_paibench_c.sh ``` +Generation is resume-safe: tasks with a valid video are skipped on re-run, and evaluation refuses to score a tree whose video count does not match the requested task set, so a partial run can never be reported as a complete benchmark result. + Alternatively, open `run_with_cosmos_framework.ipynb` for an interactive step-by-step walkthrough with inline video previews and metric display. diff --git a/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh b/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh index 8777a0d1..6f15c978 100755 --- a/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh +++ b/evaluation/cosmos3/generator/paibench_c/run_paibench_c.sh @@ -722,10 +722,12 @@ _py_for_summary() { fi } -for _mod in $PAIBENCH_C_MODALITIES; do - _videos_parent="$PAIBENCH_C_OUTPUT_ROOT/${_ckpt_slug}/$_mod" - _videos_dir="$_videos_parent/videos" - _metrics_out="$_videos_parent/metrics.json" +evaluate_modality() { + local _mod="$1" + local _videos_parent="$PAIBENCH_C_OUTPUT_ROOT/${_ckpt_slug}/$_mod" + local _videos_dir="$_videos_parent/videos" + local _metrics_out="$_videos_parent/metrics.json" + local _eval_ngpu _eval_ngpu="$(int_min "$PAIBENCH_C_NUM_SAMPLES" "$COSMOS3_NUM_GPUS")" # Verify videos exist before launching torchrun to surface errors early. @@ -734,9 +736,18 @@ for _mod in $PAIBENCH_C_MODALITIES; do Run without SKIP_GEN=1, or point PAIBENCH_C_OUTPUT_ROOT at an existing output tree." fi _nvideos="$(find "$_videos_dir" -name '*.mp4' | wc -l)" - if [[ "$_nvideos" -eq 0 ]]; then - die "No .mp4 files found in $_videos_dir - Re-run generation (remove SKIP_GEN=1) to produce videos." + + # Evaluation must cover exactly the requested task set — scoring a subset + # would silently report the subset metric as the benchmark result. + local _expected_tasks + if [[ -f "$_videos_parent/input.jsonl" ]]; then + _expected_tasks="$(awk 'END{print NR}' "$_videos_parent/input.jsonl")" + else + _expected_tasks="$PAIBENCH_C_NUM_SAMPLES" + fi + if [[ "$_nvideos" -ne "$_expected_tasks" ]]; then + die "Coverage mismatch for modality=$_mod: found $_nvideos video(s), expected $_expected_tasks. + Evaluation refuses to score a partial benchmark. Re-run generation (remove SKIP_GEN=1) to produce the missing videos, or point PAIBENCH_C_OUTPUT_ROOT at the output tree that matches this input set." fi log " Found $_nvideos video(s) in $_videos_dir" @@ -772,6 +783,10 @@ m = json.loads(pathlib.Path(os.environ["_mout"]).read_text()) for k, v in m["global"].items(): print(f" {k}: {v:.4f}") PY +} + +for _mod in $PAIBENCH_C_MODALITIES; do + evaluate_modality "$_mod" done log "=== PAI-Bench-C complete ===" diff --git a/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh b/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh index 7da921ac..db276ad7 100644 --- a/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh +++ b/evaluation/cosmos3/generator/paibench_c/test_run_paibench_c.sh @@ -41,11 +41,25 @@ assert_contains() { # assert_contains # Extract one function definition verbatim from the script under test and # evaluate it in this shell so the tests run the shipped code, not a copy. +# Handles both one-line definitions ( ... ; }) and multi-line ones closed by +# a lone "}" at column zero. load_fn() { local _fn="$1" local _src - _src="$(sed -n "/^${_fn}()/,/^}/p" "$SCRIPT_UNDER_TEST")" - [[ -n "$_src" ]] || { echo "FAIL: function ${_fn}() not found in $SCRIPT_UNDER_TEST" >&2; exit 1; } + _src="$(awk -v fn="^${_fn}[(]" ' + $0 ~ fn { + print + if ($0 ~ /}[[:space:]]*$/) { exit } + in_body = 1 + next + } + in_body { + print + if ($0 == "}") { exit } + } + ' "$SCRIPT_UNDER_TEST")" + [[ -n "$_src" && "$_src" == *}* ]] \ + || { echo "FAIL: function ${_fn}() not found in $SCRIPT_UNDER_TEST" >&2; exit 1; } eval "$_src" } @@ -168,10 +182,84 @@ test_resume_with_no_pending_tasks() { rm -rf "$_tmp" } +# --------------------------------------------------------------------------- +# Scenario 3: evaluation coverage gate — metrics are only computed when the +# collected video count equals the requested task count in input.jsonl; a +# partial tree must be rejected instead of silently scored as the benchmark +# result. +# --------------------------------------------------------------------------- +test_evaluation_coverage_gate() { + local _tmp; _tmp="$(mktemp -d)" + make_stub_env "$_tmp/stub-venv" + # Stub eval venv: torchrun writes a metrics.json and succeeds; the real + # call never happens when the gate rejects the tree. + mkdir -p "$_tmp/eval-venv/bin" "$_tmp/eval-root" + cat > "$_tmp/eval-venv/bin/torchrun" <<'STUB' +#!/usr/bin/env bash +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output_path) out="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '{"global": {"dummy_metric": 1.0}}\n' > "$out" +STUB + chmod +x "$_tmp/eval-venv/bin/torchrun" + + local _tree="$_tmp/out/Cosmos3-Nano/edge" + mkdir -p "$_tree/videos" + for i in 0 1 2 3; do + printf 'x' > "$_tree/videos/task_000$i.mp4" + printf '{"name": "task_000%i"}\n' "$i" >> "$_tree/input.jsonl" + done + + load_fn int_min + load_fn _py_for_summary + load_fn evaluate_modality + + local _rc + _run_eval() { + PAIBENCH_C_OUTPUT_ROOT="$_tmp/out" \ + PAIBENCH_C_CHECKPOINT="Cosmos3-Nano" \ + _ckpt_slug="Cosmos3-Nano" \ + PAIBENCH_C_NUM_SAMPLES="4" \ + COSMOS3_NUM_GPUS="2" \ + COSMOS3_UV_ENV="$_tmp/stub-venv" \ + PAIBENCH_UV_ENV="$_tmp/eval-venv" \ + PAIBENCH_EVAL_ROOT="$_tmp/eval-root" \ + PAIBENCH_C_DATASET_ROOT="$_tmp/dataset" \ + CUDA_VISIBLE_DEVICES="0,1" \ + evaluate_modality edge + } + + # Case A: full coverage — evaluation runs and reports the metrics. + local _rc=0 + _run_eval >"$_tmp/eval.log" 2>&1 || _rc=$? + assert_eq "$_rc" "0" "evaluation with full coverage exits 0" + assert_contains "$(cat "$_tmp/eval.log")" "dummy_metric: 1.0000" \ + "full-coverage evaluation reports the computed metrics" + + # Case B: one video missing — evaluation must refuse to score. + rm -f "$_tree/videos/task_0002.mp4" "$_tree/metrics.json" + _rc=0 + ( _run_eval ) >"$_tmp/eval2.log" 2>&1 || _rc=$? + assert_eq "$_rc" "9" "partial-coverage evaluation is rejected" + assert_contains "$(cat "$_tmp/eval2.log")" "found 3 video(s), expected 4" \ + "rejection message states the missing coverage" + [[ ! -f "$_tmp/out/Cosmos3-Nano/edge/metrics.json" ]] && PASS=$((PASS + 1)) || { + FAIL=$((FAIL + 1)) + echo "FAIL: partial-coverage evaluation must not produce metrics.json" + } + + rm -rf "$_tmp" +} + # --------------------------------------------------------------------------- run_tests() { test_fresh_run_counts_single_row_shards test_resume_with_no_pending_tasks + test_evaluation_coverage_gate echo echo "passed: $PASS failed: $FAIL" [[ "$FAIL" -eq 0 ]]