Add environment snapshot indexing

.github/workflows/result-server-tests.yml

@@ -5,7 +5,9 @@ on:
     paths:
       - "result_server/**"
       - "scripts/bk_functions.sh"
+      - "scripts/collect_environment_snapshot.sh"
       - "scripts/collect_timing.sh"
+      - "scripts/matrix_generate.sh"
       - "scripts/result.sh"
       - "scripts/result_server/**"
       - "scripts/site/setup_trigger_runner.sh"
@@ -39,7 +41,9 @@ on:
     paths:
       - "result_server/**"
       - "scripts/bk_functions.sh"
+      - "scripts/collect_environment_snapshot.sh"
       - "scripts/collect_timing.sh"
+      - "scripts/matrix_generate.sh"
       - "scripts/result.sh"
       - "scripts/result_server/**"
       - "scripts/site/setup_trigger_runner.sh"

docs/guides/portal-execution-profiles-handoff.md

@@ -54,13 +54,21 @@ Completed:
 - Received benchmark and estimation JSON metadata is indexed into
   `result_metadata_index` at ingest time. JSON/tgz artifacts remain the raw
   records and existing result pages remain file-backed.
+- Generated GitLab CI jobs collect lightweight environment snapshots in the
+  common build/run/build_run wrappers, without requiring application
+  `build.sh` or `run.sh` changes. Result submission combines those artifacts
+  into a Result JSON `environment_snapshot` reference block. The Portal indexes
+  the snapshot payload by hash in `environment_snapshots` and links received
+  results through `environment_snapshot_results`. Result detail pages show the
+  snapshot hash, allocation project ID, scheduler, runner, and BenchKit commit.
 
 Remaining follow-up:
 
-1. Add environment snapshot storage after deciding which host/runtime metadata
-   should define an environment identity.
-2. Review which result and estimate views should move from file-backed scans to
+1. Review which result and estimate views should move from file-backed scans to
    indexed lookup once the operational view requirements are stable.
+2. Expand environment snapshot collectors only when a specific site needs more
+   runtime detail. The v1 collector intentionally avoids full environment dumps
+   and secret-bearing CI variables.
 
 GitLab schedules should not be the primary governance point. The Portal should
 own periodic and event-triggered execution decisions, then trigger GitLab CI
@@ -83,6 +91,14 @@ results count run time only, because build and run are separated. Systems that
 build and run in one scheduler job are recorded as `native`; `native` counts
 build time plus run time.
 
+Environment snapshots are result-time evidence, not profile policy. Profiles
+describe the intended scope, allocation, approval state, and trigger bindings;
+snapshots describe what the CI job actually observed when it packaged the
+result. The v1 snapshot identity is the SHA-256 hash of the canonical snapshot
+payload assembled from `results/environment_snapshot_build.json`,
+`results/environment_snapshot_run.json`, or
+`results/environment_snapshot_build_run.json`.
+
 ## GitLab Pipeline Trigger Configuration
 
 Dry-run payload rendering requires:

result_server/routes/api.py

@@ -15,6 +15,7 @@
 
 from utils.auth import verify_ingest_key, verify_trusted_proxy_auth
 from utils.audit_logging import audit_event
+from utils.environment_snapshots import index_environment_snapshot
 from utils.rate_limit import rate_limited
 from utils.result_metadata_index import index_result_metadata
 
@@ -145,6 +146,27 @@ def _index_saved_json(record_type, saved):
     return indexed
 
 
+def _index_environment_snapshot(saved):
+    """Index an embedded environment snapshot when present."""
+    try:
+        indexed = index_environment_snapshot(
+            db_path=current_app.config.get("EXECUTION_PROFILE_DB_PATH"),
+            payload=saved.get("payload", {}),
+            json_file=saved.get("json_file", ""),
+        )
+    except (sqlite3.Error, OSError, ValueError) as exc:
+        current_app.logger.exception("environment snapshot index update failed")
+        audit_event(
+            "environment_snapshot_index_failed",
+            target=saved.get("json_file", ""),
+            result="failure",
+            level=logging.ERROR,
+            details={"error": str(exc)},
+        )
+        return False
+    return indexed
+
+
 def _saved_json_response(saved):
     """Return the public API response fields for a saved JSON payload."""
     return {
@@ -317,6 +339,7 @@ def ingest_result():
         prefix="result",
         out_dir=current_app.config["RECEIVED_DIR"],
     )
+    _index_environment_snapshot(saved)
     _index_saved_json("result", saved)
     audit_event(
         "ingest_accepted",

result_server/templates/result_detail.html

@@ -74,6 +74,10 @@ <h2>Quality</h2>
 </div>
 {% endif %}
 
+{% if environment_rows %}
+{{ render_titled_key_value_table("Environment Snapshot", environment_rows, "meta-table") }}
+{% endif %}
+
 {% if vector_metrics %}
 <div class="section">
     <h2>Vector Metrics - Graph</h2>

result_server/tests/test_api_routes.py

@@ -140,6 +140,76 @@ def test_post_valid_json_indexes_metadata(self, tmp_dirs, tmp_path):
             "3152",
         )
 
+    def test_post_valid_json_indexes_environment_snapshot(self, tmp_dirs, tmp_path):
+        """Accepted result payloads should index embedded environment snapshots."""
+        received, received_padata, received_estimation_artifacts, estimated = tmp_dirs
+        db_path = tmp_path / "cx_portal.sqlite3"
+        app = build_api_route_app(
+            received_dir=received,
+            received_padata_dir=received_padata,
+            received_estimation_artifacts_dir=received_estimation_artifacts,
+            estimated_dir=estimated,
+            execution_profile_db_path=str(db_path),
+        )
+        app.config["INGEST_KEYS"] = {API_KEY: "test-runner"}
+
+        with app.test_client() as client:
+            resp = client.post(
+                "/api/ingest/result",
+                data=json.dumps({
+                    "code": "qws",
+                    "system": "Fugaku",
+                    "Exp": "CASE1",
+                    "pipeline_id": 3270,
+                    "environment_snapshot": {
+                        "schema_version": 1,
+                        "hash": "sha256:test",
+                        "summary": {
+                            "system": "Fugaku",
+                            "allocation_project_id": "rkp00010",
+                            "scheduler": "pbs",
+                        },
+                        "payload": {
+                            "schema_version": 1,
+                            "system": {
+                                "name": "Fugaku",
+                                "allocation_project_id": "rkp00010",
+                            },
+                            "scheduler": {"kind": "pbs"},
+                        },
+                    },
+                }),
+                headers={
+                    "X-API-Key": API_KEY,
+                    "Content-Type": "application/json",
+                },
+            )
+
+        assert resp.status_code == 200
+        body = resp.get_json()
+        import sqlite3
+
+        with sqlite3.connect(db_path) as conn:
+            snapshot = conn.execute(
+                "SELECT snapshot_hash, result_count FROM environment_snapshots"
+            ).fetchone()
+            link = conn.execute(
+                """
+                SELECT result_uuid, snapshot_hash, json_file, code, system, exp, pipeline_id
+                FROM environment_snapshot_results
+                """
+            ).fetchone()
+        assert snapshot == ("sha256:test", 1)
+        assert link == (
+            body["id"],
+            "sha256:test",
+            body["json_file"],
+            "qws",
+            "Fugaku",
+            "CASE1",
+            "3270",
+        )
+
     def test_valid_key_logs_runner_id(self, client, caplog):
         """Accepted API requests should include the resolved runner id in logs."""
         with caplog.at_level(logging.INFO):

result_server/tests/test_environment_snapshot_ci_scripts.py

@@ -0,0 +1,27 @@
+"""Static checks for environment snapshot CI integration."""
+
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_matrix_generator_collects_snapshots_in_common_wrappers():
+    matrix_generate = (REPO_ROOT / "scripts" / "matrix_generate.sh").read_text(
+        encoding="utf-8"
+    )
+
+    assert "BK_SNAPSHOT_STAGE=build bash scripts/collect_environment_snapshot.sh" in matrix_generate
+    assert "BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh" in matrix_generate
+    assert (
+        "BK_SNAPSHOT_STAGE=build_run bash scripts/collect_environment_snapshot.sh"
+        in matrix_generate
+    )
+
+
+def test_send_results_process_does_not_collect_send_stage_snapshot():
+    process_script = (
+        REPO_ROOT / "scripts" / "result_server" / "process_and_send_results.sh"
+    ).read_text(encoding="utf-8")
+
+    assert "collect_environment_snapshot.sh" not in process_script

result_server/tests/test_environment_snapshots.py

@@ -0,0 +1,110 @@
+"""Tests for environment snapshot indexing."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from utils.environment_snapshots import (  # noqa: E402
+    extract_environment_snapshot_record,
+    index_environment_snapshot,
+    list_environment_snapshots,
+)
+
+
+def _payload(snapshot_hash="sha256:abc123"):
+    return {
+        "code": "qws",
+        "system": "Fugaku",
+        "Exp": "CASE1",
+        "_server_uuid": "11111111-2222-3333-4444-555555555555",
+        "pipeline_id": 3270,
+        "environment_snapshot": {
+            "schema_version": 1,
+            "hash": snapshot_hash,
+            "summary": {
+                "system": "Fugaku",
+                "allocation_project_id": "rkp00010",
+                "scheduler": "pbs",
+                "runner": "fugaku-runner",
+                "benchkit_commit": "abcdef",
+            },
+            "payload": {
+                "schema_version": 1,
+                "system": {
+                    "name": "Fugaku",
+                    "allocation_project_id": "rkp00010",
+                },
+                "scheduler": {"kind": "pbs"},
+            },
+        },
+    }
+
+
+def test_extract_environment_snapshot_record():
+    record = extract_environment_snapshot_record(_payload())
+
+    assert record is not None
+    assert record["snapshot_hash"] == "sha256:abc123"
+    assert record["schema_version"] == 1
+    assert record["result_uuid"] == "11111111-2222-3333-4444-555555555555"
+    assert record["pipeline_id"] == "3270"
+    assert json.loads(record["summary_json"])["allocation_project_id"] == "rkp00010"
+
+
+def test_index_environment_snapshot_deduplicates_payloads(tmp_path):
+    db_path = tmp_path / "cx_portal.sqlite3"
+
+    assert index_environment_snapshot(
+        db_path=str(db_path),
+        payload=_payload(),
+        json_file="result-a.json",
+    )
+    payload = _payload()
+    payload["_server_uuid"] = "22222222-3333-4444-5555-666666666666"
+    assert index_environment_snapshot(
+        db_path=str(db_path),
+        payload=payload,
+        json_file="result-b.json",
+    )
+
+    rows = list_environment_snapshots(str(db_path))
+    assert len(rows) == 1
+    assert rows[0]["snapshot_hash"] == "sha256:abc123"
+    assert rows[0]["result_count"] == 2
+
+    import sqlite3
+
+    with sqlite3.connect(db_path) as conn:
+        links = conn.execute(
+            "SELECT result_uuid, snapshot_hash FROM environment_snapshot_results ORDER BY result_uuid"
+        ).fetchall()
+    assert links == [
+        ("11111111-2222-3333-4444-555555555555", "sha256:abc123"),
+        ("22222222-3333-4444-5555-666666666666", "sha256:abc123"),
+    ]
+
+
+def test_index_environment_snapshot_moves_existing_result_link(tmp_path):
+    db_path = tmp_path / "cx_portal.sqlite3"
+
+    index_environment_snapshot(
+        db_path=str(db_path),
+        payload=_payload("sha256:old"),
+        json_file="result-a.json",
+    )
+    index_environment_snapshot(
+        db_path=str(db_path),
+        payload=_payload("sha256:new"),
+        json_file="result-a.json",
+    )
+
+    rows = {
+        row["snapshot_hash"]: row["result_count"]
+        for row in list_environment_snapshots(str(db_path))
+    }
+    assert rows["sha256:old"] == 0
+    assert rows["sha256:new"] == 1

result_server/tests/test_result_detail_template.py

@@ -63,6 +63,22 @@ def app():
         "events": ["pa1"],
         "report_kinds": ["summary_text"],
     },
+    "environment_snapshot": {
+        "schema_version": 1,
+        "hash": "sha256:abcdef",
+        "summary": {
+            "system": "RC_GH200",
+            "allocation_project_id": "rccs-cloud",
+            "scheduler": "slurm",
+            "runner": "gh200-runner",
+            "benchkit_commit": "1234567",
+        },
+        "payload": {
+            "schema_version": 1,
+            "ci": {"job_name": "qws_RC_GH200_run"},
+            "toolchain": {"modules": ["gcc/11.5.0", "openmpi/4.1.7"]},
+        },
+    },
 }
 
 FULL_QUALITY = {
@@ -106,6 +122,10 @@ def test_meta_info_section(self, app):
         assert "3208" in html
         assert "Parent Pipeline ID" in html
         assert "3207" in html
+        assert "Environment Snapshot" in html
+        assert "sha256:abcdef" in html
+        assert "rccs-cloud" in html
+        assert "slurm" in html
         assert "Back to Results" in html
         assert "Results" in html