diff --git a/.github/workflows/result-server-tests.yml b/.github/workflows/result-server-tests.yml
index 1c8f033..7998ca1 100644
--- a/.github/workflows/result-server-tests.yml
+++ b/.github/workflows/result-server-tests.yml
@@ -5,7 +5,9 @@ on:
paths:
- "result_server/**"
- "scripts/bk_functions.sh"
+ - "scripts/collect_environment_snapshot.sh"
- "scripts/collect_timing.sh"
+ - "scripts/matrix_generate.sh"
- "scripts/result.sh"
- "scripts/result_server/**"
- "scripts/site/setup_trigger_runner.sh"
@@ -39,7 +41,9 @@ on:
paths:
- "result_server/**"
- "scripts/bk_functions.sh"
+ - "scripts/collect_environment_snapshot.sh"
- "scripts/collect_timing.sh"
+ - "scripts/matrix_generate.sh"
- "scripts/result.sh"
- "scripts/result_server/**"
- "scripts/site/setup_trigger_runner.sh"
diff --git a/docs/guides/portal-execution-profiles-handoff.md b/docs/guides/portal-execution-profiles-handoff.md
index d5f1067..e44a33d 100644
--- a/docs/guides/portal-execution-profiles-handoff.md
+++ b/docs/guides/portal-execution-profiles-handoff.md
@@ -54,13 +54,21 @@ Completed:
- Received benchmark and estimation JSON metadata is indexed into
`result_metadata_index` at ingest time. JSON/tgz artifacts remain the raw
records and existing result pages remain file-backed.
+- Generated GitLab CI jobs collect lightweight environment snapshots in the
+ common build/run/build_run wrappers, without requiring application
+ `build.sh` or `run.sh` changes. Result submission combines those artifacts
+ into a Result JSON `environment_snapshot` reference block. The Portal indexes
+ the snapshot payload by hash in `environment_snapshots` and links received
+ results through `environment_snapshot_results`. Result detail pages show the
+ snapshot hash, allocation project ID, scheduler, runner, and BenchKit commit.
Remaining follow-up:
-1. Add environment snapshot storage after deciding which host/runtime metadata
- should define an environment identity.
-2. Review which result and estimate views should move from file-backed scans to
+1. Review which result and estimate views should move from file-backed scans to
indexed lookup once the operational view requirements are stable.
+2. Expand environment snapshot collectors only when a specific site needs more
+ runtime detail. The v1 collector intentionally avoids full environment dumps
+ and secret-bearing CI variables.
GitLab schedules should not be the primary governance point. The Portal should
own periodic and event-triggered execution decisions, then trigger GitLab CI
@@ -83,6 +91,14 @@ results count run time only, because build and run are separated. Systems that
build and run in one scheduler job are recorded as `native`; `native` counts
build time plus run time.
+Environment snapshots are result-time evidence, not profile policy. Profiles
+describe the intended scope, allocation, approval state, and trigger bindings;
+snapshots describe what the CI job actually observed when it packaged the
+result. The v1 snapshot identity is the SHA-256 hash of the canonical snapshot
+payload assembled from `results/environment_snapshot_build.json`,
+`results/environment_snapshot_run.json`, or
+`results/environment_snapshot_build_run.json`.
+
## GitLab Pipeline Trigger Configuration
Dry-run payload rendering requires:
diff --git a/result_server/routes/api.py b/result_server/routes/api.py
index 207c688..f4a4d74 100644
--- a/result_server/routes/api.py
+++ b/result_server/routes/api.py
@@ -15,6 +15,7 @@
from utils.auth import verify_ingest_key, verify_trusted_proxy_auth
from utils.audit_logging import audit_event
+from utils.environment_snapshots import index_environment_snapshot
from utils.rate_limit import rate_limited
from utils.result_metadata_index import index_result_metadata
@@ -145,6 +146,27 @@ def _index_saved_json(record_type, saved):
return indexed
+def _index_environment_snapshot(saved):
+ """Index an embedded environment snapshot when present."""
+ try:
+ indexed = index_environment_snapshot(
+ db_path=current_app.config.get("EXECUTION_PROFILE_DB_PATH"),
+ payload=saved.get("payload", {}),
+ json_file=saved.get("json_file", ""),
+ )
+ except (sqlite3.Error, OSError, ValueError) as exc:
+ current_app.logger.exception("environment snapshot index update failed")
+ audit_event(
+ "environment_snapshot_index_failed",
+ target=saved.get("json_file", ""),
+ result="failure",
+ level=logging.ERROR,
+ details={"error": str(exc)},
+ )
+ return False
+ return indexed
+
+
def _saved_json_response(saved):
"""Return the public API response fields for a saved JSON payload."""
return {
@@ -317,6 +339,7 @@ def ingest_result():
prefix="result",
out_dir=current_app.config["RECEIVED_DIR"],
)
+ _index_environment_snapshot(saved)
_index_saved_json("result", saved)
audit_event(
"ingest_accepted",
diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html
index 6358761..6cecc39 100644
--- a/result_server/templates/result_detail.html
+++ b/result_server/templates/result_detail.html
@@ -74,6 +74,10 @@
Quality
{% endif %}
+{% if environment_rows %}
+{{ render_titled_key_value_table("Environment Snapshot", environment_rows, "meta-table") }}
+{% endif %}
+
{% if vector_metrics %}
Vector Metrics - Graph
diff --git a/result_server/tests/test_api_routes.py b/result_server/tests/test_api_routes.py
index 73818d6..a97d563 100644
--- a/result_server/tests/test_api_routes.py
+++ b/result_server/tests/test_api_routes.py
@@ -140,6 +140,76 @@ def test_post_valid_json_indexes_metadata(self, tmp_dirs, tmp_path):
"3152",
)
+ def test_post_valid_json_indexes_environment_snapshot(self, tmp_dirs, tmp_path):
+ """Accepted result payloads should index embedded environment snapshots."""
+ received, received_padata, received_estimation_artifacts, estimated = tmp_dirs
+ db_path = tmp_path / "cx_portal.sqlite3"
+ app = build_api_route_app(
+ received_dir=received,
+ received_padata_dir=received_padata,
+ received_estimation_artifacts_dir=received_estimation_artifacts,
+ estimated_dir=estimated,
+ execution_profile_db_path=str(db_path),
+ )
+ app.config["INGEST_KEYS"] = {API_KEY: "test-runner"}
+
+ with app.test_client() as client:
+ resp = client.post(
+ "/api/ingest/result",
+ data=json.dumps({
+ "code": "qws",
+ "system": "Fugaku",
+ "Exp": "CASE1",
+ "pipeline_id": 3270,
+ "environment_snapshot": {
+ "schema_version": 1,
+ "hash": "sha256:test",
+ "summary": {
+ "system": "Fugaku",
+ "allocation_project_id": "rkp00010",
+ "scheduler": "pbs",
+ },
+ "payload": {
+ "schema_version": 1,
+ "system": {
+ "name": "Fugaku",
+ "allocation_project_id": "rkp00010",
+ },
+ "scheduler": {"kind": "pbs"},
+ },
+ },
+ }),
+ headers={
+ "X-API-Key": API_KEY,
+ "Content-Type": "application/json",
+ },
+ )
+
+ assert resp.status_code == 200
+ body = resp.get_json()
+ import sqlite3
+
+ with sqlite3.connect(db_path) as conn:
+ snapshot = conn.execute(
+ "SELECT snapshot_hash, result_count FROM environment_snapshots"
+ ).fetchone()
+ link = conn.execute(
+ """
+ SELECT result_uuid, snapshot_hash, json_file, code, system, exp, pipeline_id
+ FROM environment_snapshot_results
+ """
+ ).fetchone()
+ assert snapshot == ("sha256:test", 1)
+ assert link == (
+ body["id"],
+ "sha256:test",
+ body["json_file"],
+ "qws",
+ "Fugaku",
+ "CASE1",
+ "3270",
+ )
+
def test_valid_key_logs_runner_id(self, client, caplog):
"""Accepted API requests should include the resolved runner id in logs."""
with caplog.at_level(logging.INFO):
diff --git a/result_server/tests/test_environment_snapshot_ci_scripts.py b/result_server/tests/test_environment_snapshot_ci_scripts.py
new file mode 100644
index 0000000..cbd3c9c
--- /dev/null
+++ b/result_server/tests/test_environment_snapshot_ci_scripts.py
@@ -0,0 +1,27 @@
+"""Static checks for environment snapshot CI integration."""
+
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_matrix_generator_collects_snapshots_in_common_wrappers():
+ matrix_generate = (REPO_ROOT / "scripts" / "matrix_generate.sh").read_text(
+ encoding="utf-8"
+ )
+
+ assert "BK_SNAPSHOT_STAGE=build bash scripts/collect_environment_snapshot.sh" in matrix_generate
+ assert "BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh" in matrix_generate
+ assert (
+ "BK_SNAPSHOT_STAGE=build_run bash scripts/collect_environment_snapshot.sh"
+ in matrix_generate
+ )
+
+
+def test_send_results_process_does_not_collect_send_stage_snapshot():
+ process_script = (
+ REPO_ROOT / "scripts" / "result_server" / "process_and_send_results.sh"
+ ).read_text(encoding="utf-8")
+
+ assert "collect_environment_snapshot.sh" not in process_script
diff --git a/result_server/tests/test_environment_snapshots.py b/result_server/tests/test_environment_snapshots.py
new file mode 100644
index 0000000..3a64565
--- /dev/null
+++ b/result_server/tests/test_environment_snapshots.py
@@ -0,0 +1,110 @@
+"""Tests for environment snapshot indexing."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from utils.environment_snapshots import ( # noqa: E402
+ extract_environment_snapshot_record,
+ index_environment_snapshot,
+ list_environment_snapshots,
+)
+
+
+def _payload(snapshot_hash="sha256:abc123"):
+ return {
+ "code": "qws",
+ "system": "Fugaku",
+ "Exp": "CASE1",
+ "_server_uuid": "11111111-2222-3333-4444-555555555555",
+ "pipeline_id": 3270,
+ "environment_snapshot": {
+ "schema_version": 1,
+ "hash": snapshot_hash,
+ "summary": {
+ "system": "Fugaku",
+ "allocation_project_id": "rkp00010",
+ "scheduler": "pbs",
+ "runner": "fugaku-runner",
+ "benchkit_commit": "abcdef",
+ },
+ "payload": {
+ "schema_version": 1,
+ "system": {
+ "name": "Fugaku",
+ "allocation_project_id": "rkp00010",
+ },
+ "scheduler": {"kind": "pbs"},
+ },
+ },
+ }
+
+
+def test_extract_environment_snapshot_record():
+ record = extract_environment_snapshot_record(_payload())
+
+ assert record is not None
+ assert record["snapshot_hash"] == "sha256:abc123"
+ assert record["schema_version"] == 1
+ assert record["result_uuid"] == "11111111-2222-3333-4444-555555555555"
+ assert record["pipeline_id"] == "3270"
+ assert json.loads(record["summary_json"])["allocation_project_id"] == "rkp00010"
+
+
+def test_index_environment_snapshot_deduplicates_payloads(tmp_path):
+ db_path = tmp_path / "cx_portal.sqlite3"
+
+ assert index_environment_snapshot(
+ db_path=str(db_path),
+ payload=_payload(),
+ json_file="result-a.json",
+ )
+ payload = _payload()
+ payload["_server_uuid"] = "22222222-3333-4444-5555-666666666666"
+ assert index_environment_snapshot(
+ db_path=str(db_path),
+ payload=payload,
+ json_file="result-b.json",
+ )
+
+ rows = list_environment_snapshots(str(db_path))
+ assert len(rows) == 1
+ assert rows[0]["snapshot_hash"] == "sha256:abc123"
+ assert rows[0]["result_count"] == 2
+
+ import sqlite3
+
+ with sqlite3.connect(db_path) as conn:
+ links = conn.execute(
+ "SELECT result_uuid, snapshot_hash FROM environment_snapshot_results ORDER BY result_uuid"
+ ).fetchall()
+ assert links == [
+ ("11111111-2222-3333-4444-555555555555", "sha256:abc123"),
+ ("22222222-3333-4444-5555-666666666666", "sha256:abc123"),
+ ]
+
+
+def test_index_environment_snapshot_moves_existing_result_link(tmp_path):
+ db_path = tmp_path / "cx_portal.sqlite3"
+
+ index_environment_snapshot(
+ db_path=str(db_path),
+ payload=_payload("sha256:old"),
+ json_file="result-a.json",
+ )
+ index_environment_snapshot(
+ db_path=str(db_path),
+ payload=_payload("sha256:new"),
+ json_file="result-a.json",
+ )
+
+ rows = {
+ row["snapshot_hash"]: row["result_count"]
+ for row in list_environment_snapshots(str(db_path))
+ }
+ assert rows["sha256:old"] == 0
+ assert rows["sha256:new"] == 1
diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py
index 25a28d1..4aefc9d 100644
--- a/result_server/tests/test_result_detail_template.py
+++ b/result_server/tests/test_result_detail_template.py
@@ -63,6 +63,22 @@ def app():
"events": ["pa1"],
"report_kinds": ["summary_text"],
},
+ "environment_snapshot": {
+ "schema_version": 1,
+ "hash": "sha256:abcdef",
+ "summary": {
+ "system": "RC_GH200",
+ "allocation_project_id": "rccs-cloud",
+ "scheduler": "slurm",
+ "runner": "gh200-runner",
+ "benchkit_commit": "1234567",
+ },
+ "payload": {
+ "schema_version": 1,
+ "ci": {"job_name": "qws_RC_GH200_run"},
+ "toolchain": {"modules": ["gcc/11.5.0", "openmpi/4.1.7"]},
+ },
+ },
}
FULL_QUALITY = {
@@ -106,6 +122,10 @@ def test_meta_info_section(self, app):
assert "3208" in html
assert "Parent Pipeline ID" in html
assert "3207" in html
+ assert "Environment Snapshot" in html
+ assert "sha256:abcdef" in html
+ assert "rccs-cloud" in html
+ assert "slurm" in html
assert "Back to Results" in html
assert "Results" in html
diff --git a/result_server/utils/environment_snapshots.py b/result_server/utils/environment_snapshots.py
new file mode 100644
index 0000000..d212dc1
--- /dev/null
+++ b/result_server/utils/environment_snapshots.py
@@ -0,0 +1,185 @@
+"""Environment snapshot storage helpers for received benchmark results."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from utils.execution_profiles import ExecutionProfileStore
+
+
+def _utc_now_iso() -> str:
+ return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def _as_text(value: Any) -> str:
+ if value is None or isinstance(value, (dict, list)):
+ return ""
+ return str(value).strip()
+
+
+def _json_dump(value: Any) -> str:
+ return json.dumps(value or {}, ensure_ascii=False, sort_keys=True)
+
+
+def extract_environment_snapshot_record(payload: dict[str, Any]) -> dict[str, Any] | None:
+ """Return normalized snapshot fields from a Result JSON payload."""
+ snapshot = payload.get("environment_snapshot")
+ if not isinstance(snapshot, dict):
+ return None
+
+ snapshot_hash = _as_text(snapshot.get("hash"))
+ if not snapshot_hash:
+ return None
+
+ summary = snapshot.get("summary")
+ summary = summary if isinstance(summary, dict) else {}
+ snapshot_payload = snapshot.get("payload")
+ snapshot_payload = snapshot_payload if isinstance(snapshot_payload, dict) else snapshot
+
+ return {
+ "snapshot_hash": snapshot_hash,
+ "schema_version": int(snapshot.get("schema_version") or snapshot_payload.get("schema_version") or 1),
+ "summary_json": _json_dump(summary),
+ "payload_json": _json_dump(snapshot_payload),
+ "result_uuid": _as_text(payload.get("_server_uuid")),
+ "json_file": "",
+ "code": _as_text(payload.get("code")),
+ "system": _as_text(payload.get("system")),
+ "exp": _as_text(payload.get("Exp")),
+ "pipeline_id": _as_text(payload.get("pipeline_id")),
+ }
+
+
+def index_environment_snapshot(
+ *,
+ db_path: str | None,
+ payload: dict[str, Any],
+ json_file: str,
+) -> bool:
+ """Upsert environment snapshot payload and link it to a received result."""
+ if not db_path:
+ return False
+
+ record = extract_environment_snapshot_record(payload)
+ if record is None:
+ return False
+
+ record["json_file"] = json_file
+ store = ExecutionProfileStore(db_path)
+ store.migrate()
+ now = _utc_now_iso()
+ with store.connect() as conn:
+ existing = conn.execute(
+ """
+ SELECT first_seen_at, result_count FROM environment_snapshots
+ WHERE snapshot_hash = ?
+ """,
+ (record["snapshot_hash"],),
+ ).fetchone()
+ first_seen_at = existing["first_seen_at"] if existing else now
+ result_count = int(existing["result_count"]) if existing else 0
+ conn.execute(
+ """
+ INSERT INTO environment_snapshots (
+ snapshot_hash, schema_version, summary_json, payload_json,
+ first_seen_at, last_seen_at, result_count
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(snapshot_hash) DO UPDATE SET
+ schema_version=excluded.schema_version,
+ summary_json=excluded.summary_json,
+ payload_json=excluded.payload_json,
+ last_seen_at=excluded.last_seen_at
+ """,
+ (
+ record["snapshot_hash"],
+ record["schema_version"],
+ record["summary_json"],
+ record["payload_json"],
+ first_seen_at,
+ now,
+ result_count,
+ ),
+ )
+
+ linked_existing = conn.execute(
+ """
+ SELECT snapshot_hash FROM environment_snapshot_results
+ WHERE result_uuid = ?
+ """,
+ (record["result_uuid"],),
+ ).fetchone() if record["result_uuid"] else None
+ if record["result_uuid"]:
+ conn.execute(
+ """
+ INSERT INTO environment_snapshot_results (
+ result_uuid, snapshot_hash, json_file, code, system, exp,
+ pipeline_id, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(result_uuid) DO UPDATE SET
+ snapshot_hash=excluded.snapshot_hash,
+ json_file=excluded.json_file,
+ code=excluded.code,
+ system=excluded.system,
+ exp=excluded.exp,
+ pipeline_id=excluded.pipeline_id,
+ updated_at=excluded.updated_at
+ """,
+ (
+ record["result_uuid"],
+ record["snapshot_hash"],
+ record["json_file"],
+ record["code"],
+ record["system"],
+ record["exp"],
+ record["pipeline_id"],
+ now,
+ now,
+ ),
+ )
+ if not linked_existing:
+ conn.execute(
+ """
+ UPDATE environment_snapshots
+ SET result_count = result_count + 1
+ WHERE snapshot_hash = ?
+ """,
+ (record["snapshot_hash"],),
+ )
+ elif linked_existing["snapshot_hash"] != record["snapshot_hash"]:
+ conn.execute(
+ """
+ UPDATE environment_snapshots
+ SET result_count = MAX(result_count - 1, 0)
+ WHERE snapshot_hash = ?
+ """,
+ (linked_existing["snapshot_hash"],),
+ )
+ conn.execute(
+ """
+ UPDATE environment_snapshots
+ SET result_count = result_count + 1
+ WHERE snapshot_hash = ?
+ """,
+ (record["snapshot_hash"],),
+ )
+ return True
+
+
+def list_environment_snapshots(db_path: str, *, limit: int = 100) -> list[dict[str, Any]]:
+ """Return snapshot rows ordered by latest observation."""
+ store = ExecutionProfileStore(db_path)
+ store.migrate()
+ with store.connect() as conn:
+ return [
+ dict(row)
+ for row in conn.execute(
+ """
+ SELECT * FROM environment_snapshots
+ ORDER BY last_seen_at DESC, snapshot_hash DESC
+ LIMIT ?
+ """,
+ (limit,),
+ ).fetchall()
+ ]
diff --git a/result_server/utils/execution_profiles.py b/result_server/utils/execution_profiles.py
index d3dc709..9a32cc2 100644
--- a/result_server/utils/execution_profiles.py
+++ b/result_server/utils/execution_profiles.py
@@ -15,7 +15,7 @@
PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
TRIGGER_TYPES = {"manual_button", "scheduled", "watch_event"}
MATCH_MODES = {"any", "all"}
-SCHEMA_VERSION = 6
+SCHEMA_VERSION = 9
@dataclass(frozen=True)
@@ -325,6 +325,9 @@ def migrate(self) -> None:
current = 7
if current < 8:
self._apply_v8(conn)
+ current = 8
+ if current < 9:
+ self._apply_v9(conn)
def _apply_v1(self, conn: sqlite3.Connection) -> None:
now = _utc_now_iso()
@@ -558,6 +561,44 @@ def _apply_v8(self, conn: sqlite3.Connection) -> None:
(8, now),
)
+ def _apply_v9(self, conn: sqlite3.Connection) -> None:
+ now = _utc_now_iso()
+ conn.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS environment_snapshots (
+ snapshot_hash TEXT PRIMARY KEY,
+ schema_version INTEGER NOT NULL DEFAULT 1,
+ summary_json TEXT NOT NULL DEFAULT '{}',
+ payload_json TEXT NOT NULL DEFAULT '{}',
+ first_seen_at TEXT NOT NULL,
+ last_seen_at TEXT NOT NULL,
+ result_count INTEGER NOT NULL DEFAULT 0
+ );
+
+ CREATE TABLE IF NOT EXISTS environment_snapshot_results (
+ result_uuid TEXT PRIMARY KEY,
+ snapshot_hash TEXT NOT NULL REFERENCES environment_snapshots(snapshot_hash)
+ ON DELETE CASCADE,
+ json_file TEXT NOT NULL DEFAULT '',
+ code TEXT NOT NULL DEFAULT '',
+ system TEXT NOT NULL DEFAULT '',
+ exp TEXT NOT NULL DEFAULT '',
+ pipeline_id TEXT NOT NULL DEFAULT '',
+ created_at TEXT NOT NULL,
+ updated_at TEXT NOT NULL
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_environment_snapshot_results_hash
+ ON environment_snapshot_results(snapshot_hash, updated_at);
+ CREATE INDEX IF NOT EXISTS idx_environment_snapshot_results_scope
+ ON environment_snapshot_results(code, system, exp);
+ """
+ )
+ conn.execute(
+ "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)",
+ (9, now),
+ )
+
def upsert_profile(self, profile: dict[str, Any], *, actor: str = "") -> None:
self.migrate()
now = _utc_now_iso()
diff --git a/result_server/utils/result_detail_view.py b/result_server/utils/result_detail_view.py
index 27e9d40..fe0c66a 100644
--- a/result_server/utils/result_detail_view.py
+++ b/result_server/utils/result_detail_view.py
@@ -12,6 +12,7 @@ def build_result_detail_context(result, quality, trigger_runs_by_pipeline=None):
"meta_rows": _build_meta_rows(result, trigger_runs_by_pipeline),
"profile_rows": _build_profile_rows(profile_data),
"quality_rows": _build_quality_rows(quality),
+ "environment_rows": _build_environment_rows(result.get("environment_snapshot")),
"vector_metrics": vector_metrics,
"scalar_rows": _build_scalar_rows(scalar_metrics),
"build_rows": _build_build_rows(build_data),
@@ -134,6 +135,41 @@ def _build_quality_rows(quality):
]
+def _build_environment_rows(environment_snapshot):
+ if not isinstance(environment_snapshot, dict):
+ return []
+
+ summary = environment_snapshot.get("summary")
+ summary = summary if isinstance(summary, dict) else {}
+ payload = environment_snapshot.get("payload")
+ payload = payload if isinstance(payload, dict) else {}
+ system = payload.get("system") if isinstance(payload.get("system"), dict) else {}
+ scheduler = payload.get("scheduler") if isinstance(payload.get("scheduler"), dict) else {}
+ runner = payload.get("runner") if isinstance(payload.get("runner"), dict) else {}
+ ci = payload.get("ci") if isinstance(payload.get("ci"), dict) else {}
+ benchkit = payload.get("benchkit") if isinstance(payload.get("benchkit"), dict) else {}
+ toolchain = payload.get("toolchain") if isinstance(payload.get("toolchain"), dict) else {}
+
+ rows = build_labeled_value_rows([
+ ("Snapshot Hash", environment_snapshot.get("hash", "N/A")),
+ ("System", summary.get("system") or system.get("name") or "N/A"),
+ (
+ "Allocation Project ID",
+ summary.get("allocation_project_id")
+ or system.get("allocation_project_id")
+ or "not specified",
+ ),
+ ("Scheduler", summary.get("scheduler") or scheduler.get("kind") or "N/A"),
+ ("Runner", summary.get("runner") or runner.get("description") or "N/A"),
+ ("CI Job", ci.get("job_name") or "N/A"),
+ ("BenchKit Commit", summary.get("benchkit_commit") or benchkit.get("commit_hash") or "N/A"),
+ ])
+ modules = toolchain.get("modules") or []
+ if modules:
+ rows.append({"label": "Modules", "list": modules[:20]})
+ return rows
+
+
def _build_scalar_rows(scalar_metrics):
if len(scalar_metrics.keys()) < 2:
return []
diff --git a/scripts/collect_environment_snapshot.sh b/scripts/collect_environment_snapshot.sh
new file mode 100755
index 0000000..0f48a77
--- /dev/null
+++ b/scripts/collect_environment_snapshot.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+set -euo pipefail
+
+out_file="${1:-results/environment_snapshot.json}"
+snapshot_stage="${BK_SNAPSHOT_STAGE:-unknown}"
+mkdir -p "$(dirname "$out_file")"
+
+json_string_array() {
+ jq -R -s -c 'split("\n") | map(select(length > 0))'
+}
+
+command_path() {
+ command -v "$1" 2>/dev/null || true
+}
+
+command_version() {
+ local cmd="$1"
+ shift
+ if command -v "$cmd" >/dev/null 2>&1; then
+ "$cmd" "$@" 2>/dev/null | head -n 1 || true
+ fi
+}
+
+module_list_json="[]"
+if command -v module >/dev/null 2>&1; then
+ module_list_json=$(module -t list 2>&1 | sed '/^No Modulefiles Currently Loaded/d' | json_string_array)
+elif [ -n "${LOADEDMODULES:-}" ]; then
+ module_list_json=$(printf '%s' "$LOADEDMODULES" | tr ':' '\n' | json_string_array)
+fi
+
+git_commit=""
+git_branch=""
+git_dirty=""
+if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+ git_commit=$(git rev-parse HEAD 2>/dev/null || true)
+ git_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)
+ if [ -n "$(git status --porcelain 2>/dev/null || true)" ]; then
+ git_dirty="true"
+ else
+ git_dirty="false"
+ fi
+fi
+
+scheduler_kind="unknown"
+if [ -n "${SLURM_JOB_ID:-}" ] || [ -n "${SLURM_JOBID:-}" ]; then
+ scheduler_kind="slurm"
+elif [ -n "${PBS_JOBID:-}" ]; then
+ scheduler_kind="pbs"
+elif [ -n "${JACAMAR_CI:-}" ] || [ -n "${JACAMAR_SCHEDULER_ACTION:-}" ]; then
+ scheduler_kind="jacamar"
+fi
+
+hostname_value=$(hostname 2>/dev/null || true)
+uname_value=$(uname -srmo 2>/dev/null || uname -a 2>/dev/null || true)
+cpu_model=$(awk -F: '/model name|Hardware|Processor/ {gsub(/^ +/, "", $2); print $2; exit}' /proc/cpuinfo 2>/dev/null || true)
+
+jq -n \
+ --arg schema_version "1" \
+ --arg stage "$snapshot_stage" \
+ --arg collected_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+ --arg system "${BK_SYSTEM:-${system:-}}" \
+ --arg allocation_project_id "${BK_ALLOCATION_PROJECT_ID:-}" \
+ --arg runner_description "${CI_RUNNER_DESCRIPTION:-}" \
+ --arg runner_id "${CI_RUNNER_ID:-}" \
+ --arg runner_tags "${CI_RUNNER_TAGS:-}" \
+ --arg hostname "$hostname_value" \
+ --arg uname "$uname_value" \
+ --arg cpu_model "$cpu_model" \
+ --arg scheduler_kind "$scheduler_kind" \
+ --arg slurm_job_id "${SLURM_JOB_ID:-${SLURM_JOBID:-}}" \
+ --arg slurm_partition "${SLURM_JOB_PARTITION:-}" \
+ --arg pbs_jobid "${PBS_JOBID:-}" \
+ --arg jacamar_scheduler_action "${JACAMAR_SCHEDULER_ACTION:-}" \
+ --arg ci_server "${CI_SERVER_URL:-}" \
+ --arg ci_project "${CI_PROJECT_PATH:-}" \
+ --arg ci_pipeline_id "${CI_PIPELINE_ID:-}" \
+ --arg ci_job_id "${CI_JOB_ID:-}" \
+ --arg ci_job_name "${CI_JOB_NAME:-}" \
+ --arg ci_commit_ref "${CI_COMMIT_REF_NAME:-}" \
+ --arg ci_commit_sha "${CI_COMMIT_SHA:-}" \
+ --arg benchkit_commit "$git_commit" \
+ --arg benchkit_branch "$git_branch" \
+ --arg benchkit_dirty "$git_dirty" \
+ --arg gcc_version "$(command_version gcc --version)" \
+ --arg mpicc_version "$(command_version mpicc --version)" \
+ --arg nvcc_version "$(command_version nvcc --version)" \
+ --arg python_version "$(command_version python3 --version)" \
+ --argjson modules "$module_list_json" \
+ '{
+ schema_version: ($schema_version | tonumber),
+ stage: $stage,
+ collected_at: $collected_at,
+ system: {
+ name: $system,
+ allocation_project_id: $allocation_project_id,
+ host: {
+ hostname: $hostname,
+ uname: $uname,
+ cpu_model: $cpu_model
+ }
+ },
+ scheduler: {
+ kind: $scheduler_kind,
+ slurm_job_id: $slurm_job_id,
+ slurm_partition: $slurm_partition,
+ pbs_jobid: $pbs_jobid,
+ jacamar_scheduler_action: $jacamar_scheduler_action
+ },
+ runner: {
+ description: $runner_description,
+ id: $runner_id,
+ tags: $runner_tags
+ },
+ ci: {
+ server_url: $ci_server,
+ project_path: $ci_project,
+ pipeline_id: $ci_pipeline_id,
+ job_id: $ci_job_id,
+ job_name: $ci_job_name,
+ commit_ref_name: $ci_commit_ref,
+ commit_sha: $ci_commit_sha
+ },
+ benchkit: {
+ branch: $benchkit_branch,
+ commit_hash: $benchkit_commit,
+ dirty: $benchkit_dirty
+ },
+ toolchain: {
+ gcc: $gcc_version,
+ mpicc: $mpicc_version,
+ nvcc: $nvcc_version,
+ python3: $python_version,
+ modules: $modules
+ }
+ }' > "$out_file"
+
+echo "Wrote environment snapshot: $out_file"
diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh
index 7c18ac6..4bb6ad3 100644
--- a/scripts/matrix_generate.sh
+++ b/scripts/matrix_generate.sh
@@ -123,6 +123,7 @@ ${build_key}_build:
tags: [\"$build_tag\"]
script:
- mkdir -p results
+ - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=build bash scripts/collect_environment_snapshot.sh results/environment_snapshot_build.json
- bash scripts/record_timestamp.sh results/build_start
- echo \"[BUILD] $program for $system\"
- bash $program_path/build.sh $system
@@ -153,6 +154,7 @@ ${job_prefix}_run:
script:
- echo \"Starting job\"
- ls -la $program_path/
+ - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh results/environment_snapshot_run.json
- bash scripts/record_timestamp.sh results/run_start
- bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads}
- bash scripts/record_timestamp.sh results/run_end
@@ -199,6 +201,7 @@ ${job_prefix}_build_run:
- echo \"Pre-created results directory on login node\"
script:
- echo \"Starting build and run\"
+ - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=build_run bash scripts/collect_environment_snapshot.sh results/environment_snapshot_build_run.json
- bash scripts/record_timestamp.sh results/build_start
- bash $program_path/build.sh $system
- bash scripts/record_timestamp.sh results/build_end
diff --git a/scripts/result.sh b/scripts/result.sh
index 32c4a7e..8711c17 100644
--- a/scripts/result.sh
+++ b/scripts/result.sh
@@ -152,6 +152,110 @@ build_source_info_block() {
# It is parsed as data and converted with jq; it is never sourced as shell.
source_info_block=$(build_source_info_block)
+sha256_text() {
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum | awk '{print $1}'
+ return 0
+ fi
+ if command -v shasum >/dev/null 2>&1; then
+ shasum -a 256 | awk '{print $1}'
+ return 0
+ fi
+ return 1
+}
+
+build_environment_snapshot_block() {
+ local snapshot_file="results/environment_snapshot.json"
+ local build_snapshot_file="results/environment_snapshot_build.json"
+ local run_snapshot_file="results/environment_snapshot_run.json"
+ local native_snapshot_file="results/environment_snapshot_build_run.json"
+ local snapshot_source
+
+ if [ -f "$snapshot_file" ]; then
+ snapshot_source=$(jq -cS . "$snapshot_file" 2>/dev/null || true)
+ elif [ -f "$build_snapshot_file" ] || [ -f "$run_snapshot_file" ] || [ -f "$native_snapshot_file" ]; then
+ local build_snapshot="{}"
+ local run_snapshot="{}"
+ local build_run_snapshot="{}"
+ if [ -f "$build_snapshot_file" ]; then
+ build_snapshot=$(jq -cS . "$build_snapshot_file" 2>/dev/null || printf '{}')
+ fi
+ if [ -f "$run_snapshot_file" ]; then
+ run_snapshot=$(jq -cS . "$run_snapshot_file" 2>/dev/null || printf '{}')
+ fi
+ if [ -f "$native_snapshot_file" ]; then
+ build_run_snapshot=$(jq -cS . "$native_snapshot_file" 2>/dev/null || printf '{}')
+ fi
+ snapshot_source=$(jq -cS -n \
+ --argjson build "$build_snapshot" \
+ --argjson run "$run_snapshot" \
+ --argjson build_run "$build_run_snapshot" \
+ '
+ ($run | if . == {} then ($build_run | if . == {} then $build else . end) else . end) as $primary |
+ {
+ schema_version: 1,
+ collected_at: ($primary.collected_at // ""),
+ system: ($primary.system // {}),
+ scheduler: ($primary.scheduler // {}),
+ runner: ($primary.runner // {}),
+ ci: ($primary.ci // {}),
+ benchkit: (
+ if ($build.benchkit // {}) != {}
+ then $build.benchkit
+ else ($primary.benchkit // {})
+ end
+ ),
+ toolchain: {
+ build: ($build.toolchain // {}),
+ run: ($run.toolchain // {}),
+ build_run: ($build_run.toolchain // {})
+ },
+ stages: {
+ build: $build,
+ run: $run,
+ build_run: $build_run
+ }
+ }
+ ' 2>/dev/null || true)
+ else
+ printf '%s' ""
+ return 0
+ fi
+
+ local canonical_json
+ canonical_json=$(printf '%s' "$snapshot_source" | jq -cS . 2>/dev/null || true)
+ if [ -z "$canonical_json" ] || [ "$canonical_json" = "null" ]; then
+ printf '%s' ""
+ return 0
+ fi
+
+ local snapshot_hash
+ snapshot_hash=$(printf '%s' "$canonical_json" | sha256_text 2>/dev/null || true)
+ if [ -z "$snapshot_hash" ]; then
+ printf '%s' ""
+ return 0
+ fi
+
+ jq -n -c \
+ --arg hash "sha256:${snapshot_hash}" \
+ --argjson payload "$canonical_json" \
+ '{
+ schema_version: ($payload.schema_version // 1),
+ hash: $hash,
+ summary: {
+ system: ($payload.system.name // ""),
+ allocation_project_id: ($payload.system.allocation_project_id // ""),
+ scheduler: ($payload.scheduler.kind // ""),
+ runner: ($payload.runner.description // ""),
+ ci_pipeline_id: ($payload.ci.pipeline_id // ""),
+ benchkit_commit: ($payload.benchkit.commit_hash // "")
+ },
+ payload: $payload
+ }'
+}
+
+environment_snapshot_block=$(build_environment_snapshot_block)
+
# Function to write a Result_JSON file for one FOM block
# Arguments: $1=index, uses global vars: code, system, fom, fom_unit, fom_version, exp, node_count, numproc_node, description, confidential, sections_json, overlaps_json
write_result_json() {
@@ -227,6 +331,12 @@ write_result_json() {
\"execution_trigger\": ${execution_trigger_json}"
fi
+ local environment_snapshot_json_block=""
+ if [ -n "$environment_snapshot_block" ]; then
+ environment_snapshot_json_block=",
+ \"environment_snapshot\": ${environment_snapshot_block}"
+ fi
+
# Attach the profiler summary that matches this FOM index. fapp exposes
# counter events, while ncu exposes the Nsight Compute option preset.
local profile_data_block=""
@@ -278,7 +388,7 @@ write_result_json() {
"nthreads": "$nthreads",
"description": "$description",
"confidential": "$confidential",
- "source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}
+ "source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}${environment_snapshot_json_block}
}
EOF
diff --git a/scripts/tests/test_process_and_send_results.sh b/scripts/tests/test_process_and_send_results.sh
index 4d343d0..d8f3962 100644
--- a/scripts/tests/test_process_and_send_results.sh
+++ b/scripts/tests/test_process_and_send_results.sh
@@ -14,6 +14,7 @@ trap 'chmod -R u+rwX "${TMP_DIR}" 2>/dev/null || true; rm -rf "${TMP_DIR}"' EXIT
mkdir -p "${TMP_DIR}/project/scripts/result_server" "${TMP_DIR}/project/results" "${TMP_DIR}/bin"
cp "${REPO_DIR}/scripts/collect_timing.sh" "${TMP_DIR}/project/scripts/collect_timing.sh"
+cp "${REPO_DIR}/scripts/collect_environment_snapshot.sh" "${TMP_DIR}/project/scripts/collect_environment_snapshot.sh"
cp "${REPO_DIR}/scripts/result.sh" "${TMP_DIR}/project/scripts/result.sh"
cp "${REPO_DIR}/scripts/result_server/send_results.sh" "${TMP_DIR}/project/scripts/result_server/send_results.sh"
cp "${REPO_DIR}/scripts/result_server/process_and_send_results.sh" "${TMP_DIR}/project/scripts/result_server/process_and_send_results.sh"
@@ -26,6 +27,29 @@ printf '%s\n' 100 > "${TMP_DIR}/project/results/build_start"
printf '%s\n' 105 > "${TMP_DIR}/project/results/build_end"
printf '%s\n' 110 > "${TMP_DIR}/project/results/run_start"
printf '%s\n' 120 > "${TMP_DIR}/project/results/run_end"
+cat > "${TMP_DIR}/project/results/environment_snapshot_run.json" <<'EOF'
+{
+ "schema_version": 1,
+ "stage": "run",
+ "collected_at": "2026-08-10T00:00:00Z",
+ "system": {
+ "name": "Fugaku",
+ "allocation_project_id": "rkp00010"
+ },
+ "scheduler": {
+ "kind": "pbs"
+ },
+ "runner": {
+ "description": "fugaku-runner"
+ },
+ "ci": {
+ "pipeline_id": "12345"
+ },
+ "benchkit": {
+ "commit_hash": "abcdef"
+ }
+}
+EOF
cat > "${TMP_DIR}/bin/curl" <<'EOF'
#!/bin/bash
@@ -58,6 +82,7 @@ test ! -f "${TMP_DIR}/project/results/result0.json"
test -f "${TMP_DIR}/project/send_results_workspace/results/result0.json"
test -f "${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json"
test -f "${TMP_DIR}/project/send_results_workspace/results/pipeline_timing.json"
+test -f "${TMP_DIR}/project/send_results_workspace/results/environment_snapshot_run.json"
jq -e '._server_uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
@@ -65,6 +90,15 @@ jq -e '.execution_trigger.id == "qws-fugaku-1400" and .execution_trigger.type ==
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
jq -e '.pipeline_id == 12345 and .parent_pipeline_id == 54321' \
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
+jq -e '
+ .environment_snapshot.hash | startswith("sha256:")
+' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
+jq -e '
+ .environment_snapshot.summary.system == "Fugaku"
+' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
+jq -e '
+ .environment_snapshot.payload.stages.run.stage == "run"
+' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
jq -e '."result0.json".uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \
"${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json" >/dev/null