diff --git a/.github/workflows/result-server-tests.yml b/.github/workflows/result-server-tests.yml index 1c8f033..7998ca1 100644 --- a/.github/workflows/result-server-tests.yml +++ b/.github/workflows/result-server-tests.yml @@ -5,7 +5,9 @@ on: paths: - "result_server/**" - "scripts/bk_functions.sh" + - "scripts/collect_environment_snapshot.sh" - "scripts/collect_timing.sh" + - "scripts/matrix_generate.sh" - "scripts/result.sh" - "scripts/result_server/**" - "scripts/site/setup_trigger_runner.sh" @@ -39,7 +41,9 @@ on: paths: - "result_server/**" - "scripts/bk_functions.sh" + - "scripts/collect_environment_snapshot.sh" - "scripts/collect_timing.sh" + - "scripts/matrix_generate.sh" - "scripts/result.sh" - "scripts/result_server/**" - "scripts/site/setup_trigger_runner.sh" diff --git a/docs/guides/portal-execution-profiles-handoff.md b/docs/guides/portal-execution-profiles-handoff.md index d5f1067..e44a33d 100644 --- a/docs/guides/portal-execution-profiles-handoff.md +++ b/docs/guides/portal-execution-profiles-handoff.md @@ -54,13 +54,21 @@ Completed: - Received benchmark and estimation JSON metadata is indexed into `result_metadata_index` at ingest time. JSON/tgz artifacts remain the raw records and existing result pages remain file-backed. +- Generated GitLab CI jobs collect lightweight environment snapshots in the + common build/run/build_run wrappers, without requiring application + `build.sh` or `run.sh` changes. Result submission combines those artifacts + into a Result JSON `environment_snapshot` reference block. The Portal indexes + the snapshot payload by hash in `environment_snapshots` and links received + results through `environment_snapshot_results`. Result detail pages show the + snapshot hash, allocation project ID, scheduler, runner, and BenchKit commit. Remaining follow-up: -1. Add environment snapshot storage after deciding which host/runtime metadata - should define an environment identity. -2. Review which result and estimate views should move from file-backed scans to +1. Review which result and estimate views should move from file-backed scans to indexed lookup once the operational view requirements are stable. +2. Expand environment snapshot collectors only when a specific site needs more + runtime detail. The v1 collector intentionally avoids full environment dumps + and secret-bearing CI variables. GitLab schedules should not be the primary governance point. The Portal should own periodic and event-triggered execution decisions, then trigger GitLab CI @@ -83,6 +91,14 @@ results count run time only, because build and run are separated. Systems that build and run in one scheduler job are recorded as `native`; `native` counts build time plus run time. +Environment snapshots are result-time evidence, not profile policy. Profiles +describe the intended scope, allocation, approval state, and trigger bindings; +snapshots describe what the CI job actually observed when it packaged the +result. The v1 snapshot identity is the SHA-256 hash of the canonical snapshot +payload assembled from `results/environment_snapshot_build.json`, +`results/environment_snapshot_run.json`, or +`results/environment_snapshot_build_run.json`. + ## GitLab Pipeline Trigger Configuration Dry-run payload rendering requires: diff --git a/result_server/routes/api.py b/result_server/routes/api.py index 207c688..f4a4d74 100644 --- a/result_server/routes/api.py +++ b/result_server/routes/api.py @@ -15,6 +15,7 @@ from utils.auth import verify_ingest_key, verify_trusted_proxy_auth from utils.audit_logging import audit_event +from utils.environment_snapshots import index_environment_snapshot from utils.rate_limit import rate_limited from utils.result_metadata_index import index_result_metadata @@ -145,6 +146,27 @@ def _index_saved_json(record_type, saved): return indexed +def _index_environment_snapshot(saved): + """Index an embedded environment snapshot when present.""" + try: + indexed = index_environment_snapshot( + db_path=current_app.config.get("EXECUTION_PROFILE_DB_PATH"), + payload=saved.get("payload", {}), + json_file=saved.get("json_file", ""), + ) + except (sqlite3.Error, OSError, ValueError) as exc: + current_app.logger.exception("environment snapshot index update failed") + audit_event( + "environment_snapshot_index_failed", + target=saved.get("json_file", ""), + result="failure", + level=logging.ERROR, + details={"error": str(exc)}, + ) + return False + return indexed + + def _saved_json_response(saved): """Return the public API response fields for a saved JSON payload.""" return { @@ -317,6 +339,7 @@ def ingest_result(): prefix="result", out_dir=current_app.config["RECEIVED_DIR"], ) + _index_environment_snapshot(saved) _index_saved_json("result", saved) audit_event( "ingest_accepted", diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 6358761..6cecc39 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -74,6 +74,10 @@

Quality

{% endif %} +{% if environment_rows %} +{{ render_titled_key_value_table("Environment Snapshot", environment_rows, "meta-table") }} +{% endif %} + {% if vector_metrics %}

Vector Metrics - Graph

diff --git a/result_server/tests/test_api_routes.py b/result_server/tests/test_api_routes.py index 73818d6..a97d563 100644 --- a/result_server/tests/test_api_routes.py +++ b/result_server/tests/test_api_routes.py @@ -140,6 +140,76 @@ def test_post_valid_json_indexes_metadata(self, tmp_dirs, tmp_path): "3152", ) + def test_post_valid_json_indexes_environment_snapshot(self, tmp_dirs, tmp_path): + """Accepted result payloads should index embedded environment snapshots.""" + received, received_padata, received_estimation_artifacts, estimated = tmp_dirs + db_path = tmp_path / "cx_portal.sqlite3" + app = build_api_route_app( + received_dir=received, + received_padata_dir=received_padata, + received_estimation_artifacts_dir=received_estimation_artifacts, + estimated_dir=estimated, + execution_profile_db_path=str(db_path), + ) + app.config["INGEST_KEYS"] = {API_KEY: "test-runner"} + + with app.test_client() as client: + resp = client.post( + "/api/ingest/result", + data=json.dumps({ + "code": "qws", + "system": "Fugaku", + "Exp": "CASE1", + "pipeline_id": 3270, + "environment_snapshot": { + "schema_version": 1, + "hash": "sha256:test", + "summary": { + "system": "Fugaku", + "allocation_project_id": "rkp00010", + "scheduler": "pbs", + }, + "payload": { + "schema_version": 1, + "system": { + "name": "Fugaku", + "allocation_project_id": "rkp00010", + }, + "scheduler": {"kind": "pbs"}, + }, + }, + }), + headers={ + "X-API-Key": API_KEY, + "Content-Type": "application/json", + }, + ) + + assert resp.status_code == 200 + body = resp.get_json() + import sqlite3 + + with sqlite3.connect(db_path) as conn: + snapshot = conn.execute( + "SELECT snapshot_hash, result_count FROM environment_snapshots" + ).fetchone() + link = conn.execute( + """ + SELECT result_uuid, snapshot_hash, json_file, code, system, exp, pipeline_id + FROM environment_snapshot_results + """ + ).fetchone() + assert snapshot == ("sha256:test", 1) + assert link == ( + body["id"], + "sha256:test", + body["json_file"], + "qws", + "Fugaku", + "CASE1", + "3270", + ) + def test_valid_key_logs_runner_id(self, client, caplog): """Accepted API requests should include the resolved runner id in logs.""" with caplog.at_level(logging.INFO): diff --git a/result_server/tests/test_environment_snapshot_ci_scripts.py b/result_server/tests/test_environment_snapshot_ci_scripts.py new file mode 100644 index 0000000..cbd3c9c --- /dev/null +++ b/result_server/tests/test_environment_snapshot_ci_scripts.py @@ -0,0 +1,27 @@ +"""Static checks for environment snapshot CI integration.""" + +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def test_matrix_generator_collects_snapshots_in_common_wrappers(): + matrix_generate = (REPO_ROOT / "scripts" / "matrix_generate.sh").read_text( + encoding="utf-8" + ) + + assert "BK_SNAPSHOT_STAGE=build bash scripts/collect_environment_snapshot.sh" in matrix_generate + assert "BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh" in matrix_generate + assert ( + "BK_SNAPSHOT_STAGE=build_run bash scripts/collect_environment_snapshot.sh" + in matrix_generate + ) + + +def test_send_results_process_does_not_collect_send_stage_snapshot(): + process_script = ( + REPO_ROOT / "scripts" / "result_server" / "process_and_send_results.sh" + ).read_text(encoding="utf-8") + + assert "collect_environment_snapshot.sh" not in process_script diff --git a/result_server/tests/test_environment_snapshots.py b/result_server/tests/test_environment_snapshots.py new file mode 100644 index 0000000..3a64565 --- /dev/null +++ b/result_server/tests/test_environment_snapshots.py @@ -0,0 +1,110 @@ +"""Tests for environment snapshot indexing.""" + +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.environment_snapshots import ( # noqa: E402 + extract_environment_snapshot_record, + index_environment_snapshot, + list_environment_snapshots, +) + + +def _payload(snapshot_hash="sha256:abc123"): + return { + "code": "qws", + "system": "Fugaku", + "Exp": "CASE1", + "_server_uuid": "11111111-2222-3333-4444-555555555555", + "pipeline_id": 3270, + "environment_snapshot": { + "schema_version": 1, + "hash": snapshot_hash, + "summary": { + "system": "Fugaku", + "allocation_project_id": "rkp00010", + "scheduler": "pbs", + "runner": "fugaku-runner", + "benchkit_commit": "abcdef", + }, + "payload": { + "schema_version": 1, + "system": { + "name": "Fugaku", + "allocation_project_id": "rkp00010", + }, + "scheduler": {"kind": "pbs"}, + }, + }, + } + + +def test_extract_environment_snapshot_record(): + record = extract_environment_snapshot_record(_payload()) + + assert record is not None + assert record["snapshot_hash"] == "sha256:abc123" + assert record["schema_version"] == 1 + assert record["result_uuid"] == "11111111-2222-3333-4444-555555555555" + assert record["pipeline_id"] == "3270" + assert json.loads(record["summary_json"])["allocation_project_id"] == "rkp00010" + + +def test_index_environment_snapshot_deduplicates_payloads(tmp_path): + db_path = tmp_path / "cx_portal.sqlite3" + + assert index_environment_snapshot( + db_path=str(db_path), + payload=_payload(), + json_file="result-a.json", + ) + payload = _payload() + payload["_server_uuid"] = "22222222-3333-4444-5555-666666666666" + assert index_environment_snapshot( + db_path=str(db_path), + payload=payload, + json_file="result-b.json", + ) + + rows = list_environment_snapshots(str(db_path)) + assert len(rows) == 1 + assert rows[0]["snapshot_hash"] == "sha256:abc123" + assert rows[0]["result_count"] == 2 + + import sqlite3 + + with sqlite3.connect(db_path) as conn: + links = conn.execute( + "SELECT result_uuid, snapshot_hash FROM environment_snapshot_results ORDER BY result_uuid" + ).fetchall() + assert links == [ + ("11111111-2222-3333-4444-555555555555", "sha256:abc123"), + ("22222222-3333-4444-5555-666666666666", "sha256:abc123"), + ] + + +def test_index_environment_snapshot_moves_existing_result_link(tmp_path): + db_path = tmp_path / "cx_portal.sqlite3" + + index_environment_snapshot( + db_path=str(db_path), + payload=_payload("sha256:old"), + json_file="result-a.json", + ) + index_environment_snapshot( + db_path=str(db_path), + payload=_payload("sha256:new"), + json_file="result-a.json", + ) + + rows = { + row["snapshot_hash"]: row["result_count"] + for row in list_environment_snapshots(str(db_path)) + } + assert rows["sha256:old"] == 0 + assert rows["sha256:new"] == 1 diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py index 25a28d1..4aefc9d 100644 --- a/result_server/tests/test_result_detail_template.py +++ b/result_server/tests/test_result_detail_template.py @@ -63,6 +63,22 @@ def app(): "events": ["pa1"], "report_kinds": ["summary_text"], }, + "environment_snapshot": { + "schema_version": 1, + "hash": "sha256:abcdef", + "summary": { + "system": "RC_GH200", + "allocation_project_id": "rccs-cloud", + "scheduler": "slurm", + "runner": "gh200-runner", + "benchkit_commit": "1234567", + }, + "payload": { + "schema_version": 1, + "ci": {"job_name": "qws_RC_GH200_run"}, + "toolchain": {"modules": ["gcc/11.5.0", "openmpi/4.1.7"]}, + }, + }, } FULL_QUALITY = { @@ -106,6 +122,10 @@ def test_meta_info_section(self, app): assert "3208" in html assert "Parent Pipeline ID" in html assert "3207" in html + assert "Environment Snapshot" in html + assert "sha256:abcdef" in html + assert "rccs-cloud" in html + assert "slurm" in html assert "Back to Results" in html assert "Results" in html diff --git a/result_server/utils/environment_snapshots.py b/result_server/utils/environment_snapshots.py new file mode 100644 index 0000000..d212dc1 --- /dev/null +++ b/result_server/utils/environment_snapshots.py @@ -0,0 +1,185 @@ +"""Environment snapshot storage helpers for received benchmark results.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +from utils.execution_profiles import ExecutionProfileStore + + +def _utc_now_iso() -> str: + return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _as_text(value: Any) -> str: + if value is None or isinstance(value, (dict, list)): + return "" + return str(value).strip() + + +def _json_dump(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, sort_keys=True) + + +def extract_environment_snapshot_record(payload: dict[str, Any]) -> dict[str, Any] | None: + """Return normalized snapshot fields from a Result JSON payload.""" + snapshot = payload.get("environment_snapshot") + if not isinstance(snapshot, dict): + return None + + snapshot_hash = _as_text(snapshot.get("hash")) + if not snapshot_hash: + return None + + summary = snapshot.get("summary") + summary = summary if isinstance(summary, dict) else {} + snapshot_payload = snapshot.get("payload") + snapshot_payload = snapshot_payload if isinstance(snapshot_payload, dict) else snapshot + + return { + "snapshot_hash": snapshot_hash, + "schema_version": int(snapshot.get("schema_version") or snapshot_payload.get("schema_version") or 1), + "summary_json": _json_dump(summary), + "payload_json": _json_dump(snapshot_payload), + "result_uuid": _as_text(payload.get("_server_uuid")), + "json_file": "", + "code": _as_text(payload.get("code")), + "system": _as_text(payload.get("system")), + "exp": _as_text(payload.get("Exp")), + "pipeline_id": _as_text(payload.get("pipeline_id")), + } + + +def index_environment_snapshot( + *, + db_path: str | None, + payload: dict[str, Any], + json_file: str, +) -> bool: + """Upsert environment snapshot payload and link it to a received result.""" + if not db_path: + return False + + record = extract_environment_snapshot_record(payload) + if record is None: + return False + + record["json_file"] = json_file + store = ExecutionProfileStore(db_path) + store.migrate() + now = _utc_now_iso() + with store.connect() as conn: + existing = conn.execute( + """ + SELECT first_seen_at, result_count FROM environment_snapshots + WHERE snapshot_hash = ? + """, + (record["snapshot_hash"],), + ).fetchone() + first_seen_at = existing["first_seen_at"] if existing else now + result_count = int(existing["result_count"]) if existing else 0 + conn.execute( + """ + INSERT INTO environment_snapshots ( + snapshot_hash, schema_version, summary_json, payload_json, + first_seen_at, last_seen_at, result_count + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(snapshot_hash) DO UPDATE SET + schema_version=excluded.schema_version, + summary_json=excluded.summary_json, + payload_json=excluded.payload_json, + last_seen_at=excluded.last_seen_at + """, + ( + record["snapshot_hash"], + record["schema_version"], + record["summary_json"], + record["payload_json"], + first_seen_at, + now, + result_count, + ), + ) + + linked_existing = conn.execute( + """ + SELECT snapshot_hash FROM environment_snapshot_results + WHERE result_uuid = ? + """, + (record["result_uuid"],), + ).fetchone() if record["result_uuid"] else None + if record["result_uuid"]: + conn.execute( + """ + INSERT INTO environment_snapshot_results ( + result_uuid, snapshot_hash, json_file, code, system, exp, + pipeline_id, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(result_uuid) DO UPDATE SET + snapshot_hash=excluded.snapshot_hash, + json_file=excluded.json_file, + code=excluded.code, + system=excluded.system, + exp=excluded.exp, + pipeline_id=excluded.pipeline_id, + updated_at=excluded.updated_at + """, + ( + record["result_uuid"], + record["snapshot_hash"], + record["json_file"], + record["code"], + record["system"], + record["exp"], + record["pipeline_id"], + now, + now, + ), + ) + if not linked_existing: + conn.execute( + """ + UPDATE environment_snapshots + SET result_count = result_count + 1 + WHERE snapshot_hash = ? + """, + (record["snapshot_hash"],), + ) + elif linked_existing["snapshot_hash"] != record["snapshot_hash"]: + conn.execute( + """ + UPDATE environment_snapshots + SET result_count = MAX(result_count - 1, 0) + WHERE snapshot_hash = ? + """, + (linked_existing["snapshot_hash"],), + ) + conn.execute( + """ + UPDATE environment_snapshots + SET result_count = result_count + 1 + WHERE snapshot_hash = ? + """, + (record["snapshot_hash"],), + ) + return True + + +def list_environment_snapshots(db_path: str, *, limit: int = 100) -> list[dict[str, Any]]: + """Return snapshot rows ordered by latest observation.""" + store = ExecutionProfileStore(db_path) + store.migrate() + with store.connect() as conn: + return [ + dict(row) + for row in conn.execute( + """ + SELECT * FROM environment_snapshots + ORDER BY last_seen_at DESC, snapshot_hash DESC + LIMIT ? + """, + (limit,), + ).fetchall() + ] diff --git a/result_server/utils/execution_profiles.py b/result_server/utils/execution_profiles.py index d3dc709..9a32cc2 100644 --- a/result_server/utils/execution_profiles.py +++ b/result_server/utils/execution_profiles.py @@ -15,7 +15,7 @@ PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") TRIGGER_TYPES = {"manual_button", "scheduled", "watch_event"} MATCH_MODES = {"any", "all"} -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 9 @dataclass(frozen=True) @@ -325,6 +325,9 @@ def migrate(self) -> None: current = 7 if current < 8: self._apply_v8(conn) + current = 8 + if current < 9: + self._apply_v9(conn) def _apply_v1(self, conn: sqlite3.Connection) -> None: now = _utc_now_iso() @@ -558,6 +561,44 @@ def _apply_v8(self, conn: sqlite3.Connection) -> None: (8, now), ) + def _apply_v9(self, conn: sqlite3.Connection) -> None: + now = _utc_now_iso() + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS environment_snapshots ( + snapshot_hash TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL DEFAULT 1, + summary_json TEXT NOT NULL DEFAULT '{}', + payload_json TEXT NOT NULL DEFAULT '{}', + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + result_count INTEGER NOT NULL DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS environment_snapshot_results ( + result_uuid TEXT PRIMARY KEY, + snapshot_hash TEXT NOT NULL REFERENCES environment_snapshots(snapshot_hash) + ON DELETE CASCADE, + json_file TEXT NOT NULL DEFAULT '', + code TEXT NOT NULL DEFAULT '', + system TEXT NOT NULL DEFAULT '', + exp TEXT NOT NULL DEFAULT '', + pipeline_id TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_environment_snapshot_results_hash + ON environment_snapshot_results(snapshot_hash, updated_at); + CREATE INDEX IF NOT EXISTS idx_environment_snapshot_results_scope + ON environment_snapshot_results(code, system, exp); + """ + ) + conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", + (9, now), + ) + def upsert_profile(self, profile: dict[str, Any], *, actor: str = "") -> None: self.migrate() now = _utc_now_iso() diff --git a/result_server/utils/result_detail_view.py b/result_server/utils/result_detail_view.py index 27e9d40..fe0c66a 100644 --- a/result_server/utils/result_detail_view.py +++ b/result_server/utils/result_detail_view.py @@ -12,6 +12,7 @@ def build_result_detail_context(result, quality, trigger_runs_by_pipeline=None): "meta_rows": _build_meta_rows(result, trigger_runs_by_pipeline), "profile_rows": _build_profile_rows(profile_data), "quality_rows": _build_quality_rows(quality), + "environment_rows": _build_environment_rows(result.get("environment_snapshot")), "vector_metrics": vector_metrics, "scalar_rows": _build_scalar_rows(scalar_metrics), "build_rows": _build_build_rows(build_data), @@ -134,6 +135,41 @@ def _build_quality_rows(quality): ] +def _build_environment_rows(environment_snapshot): + if not isinstance(environment_snapshot, dict): + return [] + + summary = environment_snapshot.get("summary") + summary = summary if isinstance(summary, dict) else {} + payload = environment_snapshot.get("payload") + payload = payload if isinstance(payload, dict) else {} + system = payload.get("system") if isinstance(payload.get("system"), dict) else {} + scheduler = payload.get("scheduler") if isinstance(payload.get("scheduler"), dict) else {} + runner = payload.get("runner") if isinstance(payload.get("runner"), dict) else {} + ci = payload.get("ci") if isinstance(payload.get("ci"), dict) else {} + benchkit = payload.get("benchkit") if isinstance(payload.get("benchkit"), dict) else {} + toolchain = payload.get("toolchain") if isinstance(payload.get("toolchain"), dict) else {} + + rows = build_labeled_value_rows([ + ("Snapshot Hash", environment_snapshot.get("hash", "N/A")), + ("System", summary.get("system") or system.get("name") or "N/A"), + ( + "Allocation Project ID", + summary.get("allocation_project_id") + or system.get("allocation_project_id") + or "not specified", + ), + ("Scheduler", summary.get("scheduler") or scheduler.get("kind") or "N/A"), + ("Runner", summary.get("runner") or runner.get("description") or "N/A"), + ("CI Job", ci.get("job_name") or "N/A"), + ("BenchKit Commit", summary.get("benchkit_commit") or benchkit.get("commit_hash") or "N/A"), + ]) + modules = toolchain.get("modules") or [] + if modules: + rows.append({"label": "Modules", "list": modules[:20]}) + return rows + + def _build_scalar_rows(scalar_metrics): if len(scalar_metrics.keys()) < 2: return [] diff --git a/scripts/collect_environment_snapshot.sh b/scripts/collect_environment_snapshot.sh new file mode 100755 index 0000000..0f48a77 --- /dev/null +++ b/scripts/collect_environment_snapshot.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -euo pipefail + +out_file="${1:-results/environment_snapshot.json}" +snapshot_stage="${BK_SNAPSHOT_STAGE:-unknown}" +mkdir -p "$(dirname "$out_file")" + +json_string_array() { + jq -R -s -c 'split("\n") | map(select(length > 0))' +} + +command_path() { + command -v "$1" 2>/dev/null || true +} + +command_version() { + local cmd="$1" + shift + if command -v "$cmd" >/dev/null 2>&1; then + "$cmd" "$@" 2>/dev/null | head -n 1 || true + fi +} + +module_list_json="[]" +if command -v module >/dev/null 2>&1; then + module_list_json=$(module -t list 2>&1 | sed '/^No Modulefiles Currently Loaded/d' | json_string_array) +elif [ -n "${LOADEDMODULES:-}" ]; then + module_list_json=$(printf '%s' "$LOADEDMODULES" | tr ':' '\n' | json_string_array) +fi + +git_commit="" +git_branch="" +git_dirty="" +if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git_commit=$(git rev-parse HEAD 2>/dev/null || true) + git_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true) + if [ -n "$(git status --porcelain 2>/dev/null || true)" ]; then + git_dirty="true" + else + git_dirty="false" + fi +fi + +scheduler_kind="unknown" +if [ -n "${SLURM_JOB_ID:-}" ] || [ -n "${SLURM_JOBID:-}" ]; then + scheduler_kind="slurm" +elif [ -n "${PBS_JOBID:-}" ]; then + scheduler_kind="pbs" +elif [ -n "${JACAMAR_CI:-}" ] || [ -n "${JACAMAR_SCHEDULER_ACTION:-}" ]; then + scheduler_kind="jacamar" +fi + +hostname_value=$(hostname 2>/dev/null || true) +uname_value=$(uname -srmo 2>/dev/null || uname -a 2>/dev/null || true) +cpu_model=$(awk -F: '/model name|Hardware|Processor/ {gsub(/^ +/, "", $2); print $2; exit}' /proc/cpuinfo 2>/dev/null || true) + +jq -n \ + --arg schema_version "1" \ + --arg stage "$snapshot_stage" \ + --arg collected_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg system "${BK_SYSTEM:-${system:-}}" \ + --arg allocation_project_id "${BK_ALLOCATION_PROJECT_ID:-}" \ + --arg runner_description "${CI_RUNNER_DESCRIPTION:-}" \ + --arg runner_id "${CI_RUNNER_ID:-}" \ + --arg runner_tags "${CI_RUNNER_TAGS:-}" \ + --arg hostname "$hostname_value" \ + --arg uname "$uname_value" \ + --arg cpu_model "$cpu_model" \ + --arg scheduler_kind "$scheduler_kind" \ + --arg slurm_job_id "${SLURM_JOB_ID:-${SLURM_JOBID:-}}" \ + --arg slurm_partition "${SLURM_JOB_PARTITION:-}" \ + --arg pbs_jobid "${PBS_JOBID:-}" \ + --arg jacamar_scheduler_action "${JACAMAR_SCHEDULER_ACTION:-}" \ + --arg ci_server "${CI_SERVER_URL:-}" \ + --arg ci_project "${CI_PROJECT_PATH:-}" \ + --arg ci_pipeline_id "${CI_PIPELINE_ID:-}" \ + --arg ci_job_id "${CI_JOB_ID:-}" \ + --arg ci_job_name "${CI_JOB_NAME:-}" \ + --arg ci_commit_ref "${CI_COMMIT_REF_NAME:-}" \ + --arg ci_commit_sha "${CI_COMMIT_SHA:-}" \ + --arg benchkit_commit "$git_commit" \ + --arg benchkit_branch "$git_branch" \ + --arg benchkit_dirty "$git_dirty" \ + --arg gcc_version "$(command_version gcc --version)" \ + --arg mpicc_version "$(command_version mpicc --version)" \ + --arg nvcc_version "$(command_version nvcc --version)" \ + --arg python_version "$(command_version python3 --version)" \ + --argjson modules "$module_list_json" \ + '{ + schema_version: ($schema_version | tonumber), + stage: $stage, + collected_at: $collected_at, + system: { + name: $system, + allocation_project_id: $allocation_project_id, + host: { + hostname: $hostname, + uname: $uname, + cpu_model: $cpu_model + } + }, + scheduler: { + kind: $scheduler_kind, + slurm_job_id: $slurm_job_id, + slurm_partition: $slurm_partition, + pbs_jobid: $pbs_jobid, + jacamar_scheduler_action: $jacamar_scheduler_action + }, + runner: { + description: $runner_description, + id: $runner_id, + tags: $runner_tags + }, + ci: { + server_url: $ci_server, + project_path: $ci_project, + pipeline_id: $ci_pipeline_id, + job_id: $ci_job_id, + job_name: $ci_job_name, + commit_ref_name: $ci_commit_ref, + commit_sha: $ci_commit_sha + }, + benchkit: { + branch: $benchkit_branch, + commit_hash: $benchkit_commit, + dirty: $benchkit_dirty + }, + toolchain: { + gcc: $gcc_version, + mpicc: $mpicc_version, + nvcc: $nvcc_version, + python3: $python_version, + modules: $modules + } + }' > "$out_file" + +echo "Wrote environment snapshot: $out_file" diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh index 7c18ac6..4bb6ad3 100644 --- a/scripts/matrix_generate.sh +++ b/scripts/matrix_generate.sh @@ -123,6 +123,7 @@ ${build_key}_build: tags: [\"$build_tag\"] script: - mkdir -p results + - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=build bash scripts/collect_environment_snapshot.sh results/environment_snapshot_build.json - bash scripts/record_timestamp.sh results/build_start - echo \"[BUILD] $program for $system\" - bash $program_path/build.sh $system @@ -153,6 +154,7 @@ ${job_prefix}_run: script: - echo \"Starting job\" - ls -la $program_path/ + - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh results/environment_snapshot_run.json - bash scripts/record_timestamp.sh results/run_start - bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads} - bash scripts/record_timestamp.sh results/run_end @@ -199,6 +201,7 @@ ${job_prefix}_build_run: - echo \"Pre-created results directory on login node\" script: - echo \"Starting build and run\" + - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=build_run bash scripts/collect_environment_snapshot.sh results/environment_snapshot_build_run.json - bash scripts/record_timestamp.sh results/build_start - bash $program_path/build.sh $system - bash scripts/record_timestamp.sh results/build_end diff --git a/scripts/result.sh b/scripts/result.sh index 32c4a7e..8711c17 100644 --- a/scripts/result.sh +++ b/scripts/result.sh @@ -152,6 +152,110 @@ build_source_info_block() { # It is parsed as data and converted with jq; it is never sourced as shell. source_info_block=$(build_source_info_block) +sha256_text() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + return 0 + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 | awk '{print $1}' + return 0 + fi + return 1 +} + +build_environment_snapshot_block() { + local snapshot_file="results/environment_snapshot.json" + local build_snapshot_file="results/environment_snapshot_build.json" + local run_snapshot_file="results/environment_snapshot_run.json" + local native_snapshot_file="results/environment_snapshot_build_run.json" + local snapshot_source + + if [ -f "$snapshot_file" ]; then + snapshot_source=$(jq -cS . "$snapshot_file" 2>/dev/null || true) + elif [ -f "$build_snapshot_file" ] || [ -f "$run_snapshot_file" ] || [ -f "$native_snapshot_file" ]; then + local build_snapshot="{}" + local run_snapshot="{}" + local build_run_snapshot="{}" + if [ -f "$build_snapshot_file" ]; then + build_snapshot=$(jq -cS . "$build_snapshot_file" 2>/dev/null || printf '{}') + fi + if [ -f "$run_snapshot_file" ]; then + run_snapshot=$(jq -cS . "$run_snapshot_file" 2>/dev/null || printf '{}') + fi + if [ -f "$native_snapshot_file" ]; then + build_run_snapshot=$(jq -cS . "$native_snapshot_file" 2>/dev/null || printf '{}') + fi + snapshot_source=$(jq -cS -n \ + --argjson build "$build_snapshot" \ + --argjson run "$run_snapshot" \ + --argjson build_run "$build_run_snapshot" \ + ' + ($run | if . == {} then ($build_run | if . == {} then $build else . end) else . end) as $primary | + { + schema_version: 1, + collected_at: ($primary.collected_at // ""), + system: ($primary.system // {}), + scheduler: ($primary.scheduler // {}), + runner: ($primary.runner // {}), + ci: ($primary.ci // {}), + benchkit: ( + if ($build.benchkit // {}) != {} + then $build.benchkit + else ($primary.benchkit // {}) + end + ), + toolchain: { + build: ($build.toolchain // {}), + run: ($run.toolchain // {}), + build_run: ($build_run.toolchain // {}) + }, + stages: { + build: $build, + run: $run, + build_run: $build_run + } + } + ' 2>/dev/null || true) + else + printf '%s' "" + return 0 + fi + + local canonical_json + canonical_json=$(printf '%s' "$snapshot_source" | jq -cS . 2>/dev/null || true) + if [ -z "$canonical_json" ] || [ "$canonical_json" = "null" ]; then + printf '%s' "" + return 0 + fi + + local snapshot_hash + snapshot_hash=$(printf '%s' "$canonical_json" | sha256_text 2>/dev/null || true) + if [ -z "$snapshot_hash" ]; then + printf '%s' "" + return 0 + fi + + jq -n -c \ + --arg hash "sha256:${snapshot_hash}" \ + --argjson payload "$canonical_json" \ + '{ + schema_version: ($payload.schema_version // 1), + hash: $hash, + summary: { + system: ($payload.system.name // ""), + allocation_project_id: ($payload.system.allocation_project_id // ""), + scheduler: ($payload.scheduler.kind // ""), + runner: ($payload.runner.description // ""), + ci_pipeline_id: ($payload.ci.pipeline_id // ""), + benchkit_commit: ($payload.benchkit.commit_hash // "") + }, + payload: $payload + }' +} + +environment_snapshot_block=$(build_environment_snapshot_block) + # Function to write a Result_JSON file for one FOM block # Arguments: $1=index, uses global vars: code, system, fom, fom_unit, fom_version, exp, node_count, numproc_node, description, confidential, sections_json, overlaps_json write_result_json() { @@ -227,6 +331,12 @@ write_result_json() { \"execution_trigger\": ${execution_trigger_json}" fi + local environment_snapshot_json_block="" + if [ -n "$environment_snapshot_block" ]; then + environment_snapshot_json_block=", + \"environment_snapshot\": ${environment_snapshot_block}" + fi + # Attach the profiler summary that matches this FOM index. fapp exposes # counter events, while ncu exposes the Nsight Compute option preset. local profile_data_block="" @@ -278,7 +388,7 @@ write_result_json() { "nthreads": "$nthreads", "description": "$description", "confidential": "$confidential", - "source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block} + "source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}${environment_snapshot_json_block} } EOF diff --git a/scripts/tests/test_process_and_send_results.sh b/scripts/tests/test_process_and_send_results.sh index 4d343d0..d8f3962 100644 --- a/scripts/tests/test_process_and_send_results.sh +++ b/scripts/tests/test_process_and_send_results.sh @@ -14,6 +14,7 @@ trap 'chmod -R u+rwX "${TMP_DIR}" 2>/dev/null || true; rm -rf "${TMP_DIR}"' EXIT mkdir -p "${TMP_DIR}/project/scripts/result_server" "${TMP_DIR}/project/results" "${TMP_DIR}/bin" cp "${REPO_DIR}/scripts/collect_timing.sh" "${TMP_DIR}/project/scripts/collect_timing.sh" +cp "${REPO_DIR}/scripts/collect_environment_snapshot.sh" "${TMP_DIR}/project/scripts/collect_environment_snapshot.sh" cp "${REPO_DIR}/scripts/result.sh" "${TMP_DIR}/project/scripts/result.sh" cp "${REPO_DIR}/scripts/result_server/send_results.sh" "${TMP_DIR}/project/scripts/result_server/send_results.sh" cp "${REPO_DIR}/scripts/result_server/process_and_send_results.sh" "${TMP_DIR}/project/scripts/result_server/process_and_send_results.sh" @@ -26,6 +27,29 @@ printf '%s\n' 100 > "${TMP_DIR}/project/results/build_start" printf '%s\n' 105 > "${TMP_DIR}/project/results/build_end" printf '%s\n' 110 > "${TMP_DIR}/project/results/run_start" printf '%s\n' 120 > "${TMP_DIR}/project/results/run_end" +cat > "${TMP_DIR}/project/results/environment_snapshot_run.json" <<'EOF' +{ + "schema_version": 1, + "stage": "run", + "collected_at": "2026-08-10T00:00:00Z", + "system": { + "name": "Fugaku", + "allocation_project_id": "rkp00010" + }, + "scheduler": { + "kind": "pbs" + }, + "runner": { + "description": "fugaku-runner" + }, + "ci": { + "pipeline_id": "12345" + }, + "benchkit": { + "commit_hash": "abcdef" + } +} +EOF cat > "${TMP_DIR}/bin/curl" <<'EOF' #!/bin/bash @@ -58,6 +82,7 @@ test ! -f "${TMP_DIR}/project/results/result0.json" test -f "${TMP_DIR}/project/send_results_workspace/results/result0.json" test -f "${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json" test -f "${TMP_DIR}/project/send_results_workspace/results/pipeline_timing.json" +test -f "${TMP_DIR}/project/send_results_workspace/results/environment_snapshot_run.json" jq -e '._server_uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \ "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null @@ -65,6 +90,15 @@ jq -e '.execution_trigger.id == "qws-fugaku-1400" and .execution_trigger.type == "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null jq -e '.pipeline_id == 12345 and .parent_pipeline_id == 54321' \ "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null +jq -e ' + .environment_snapshot.hash | startswith("sha256:") +' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null +jq -e ' + .environment_snapshot.summary.system == "Fugaku" +' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null +jq -e ' + .environment_snapshot.payload.stages.run.stage == "run" +' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null jq -e '."result0.json".uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \ "${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json" >/dev/null