Release: reuse main's exact-commit CI evidence at the tag, fail closed; both Runner jobs, release contents and the pinned integration inputs kept (#175)

.github/ci-evidence.json

@@ -0,0 +1,173 @@
+{
+  "$comment": [
+    "The release evidence gate's configuration, read at the release commit by scripts/dev/release_evidence.py",
+    "(docs/releasing.md, 'CI at the tag'). tools/test_ci_evidence.py holds it to ci.yml, release.yml and dev.json.",
+    "inputs: the module pins; the script that pins the Runtime, earlier Runtime, old Runner and Gateway commits",
+    "the integration tests run against; the scripts that fail when a named real-companion test does not run and",
+    "pass; the toolchain check and the workflow naming the release toolchain; the runner whose summary is read;",
+    "the Release contents scripts and the packer's configuration."
+  ],
+  "schema": 2,
+  "repository": "Judgment-Pack/judgment-pack-runner",
+  "branch": "main",
+  "max_age_hours": 48,
+  "inputs": [
+    "go.mod",
+    "go.sum",
+    "scripts/checks/build-pinned.sh",
+    "scripts/checks/manual-gate.sh",
+    "scripts/checks/chain-tests.sh",
+    "scripts/checks/unit-tests.sh",
+    "scripts/checks/release-toolchain.sh",
+    "scripts/dev/check.sh",
+    ".github/workflows/release.yml",
+    "scripts/checks/release-programs.sh",
+    "tools/third_party_notices.py",
+    "tools/release_archives.py",
+    "tools/test_release_archives.py",
+    ".goreleaser.yml"
+  ],
+  "fresh_at_release": [
+    "tag syntax, tag names the commit, commit on main",
+    "release notes",
+    "production environment reviewer",
+    "third-party notices with the release toolchain, packaging, checksums",
+    "contents of every archive",
+    "native archive smoke tests (linux/amd64, linux/arm64, darwin/arm64)",
+    "tag still names the commit, no existing release, attestation, draft",
+    "draft identity, notes and asset digests, publication, published asset digests"
+  ],
+  "profiles": {
+    "linux": {
+      "$comment": [
+        "Both Linux legs, compared with the release job's own runner: the same hosted image and tools, and the",
+        "Go toolchain release.yml links the archives with, set up there the same way."
+      ],
+      "platform": {
+        "RUNNER_OS": "Linux",
+        "RUNNER_ARCH": "X64"
+      },
+      "compare": "release-runner",
+      "tools": {
+        "go": [
+          "go",
+          "version"
+        ],
+        "python": [
+          "python3",
+          "--version"
+        ],
+        "git": [
+          "git",
+          "--version"
+        ],
+        "bash": [
+          "bash",
+          "--version"
+        ]
+      },
+      "pins": {
+        "go": "go version go1.26.5 linux/amd64"
+      }
+    },
+    "macos": {
+      "$comment": [
+        "The macOS leg, which the release job cannot resolve on its own runner: the platform, the image family",
+        "(ImageOS) and the exact Go toolchain are pinned; the image version within the family is recorded and not",
+        "compared. When macos-latest moves to another family, capture on main refuses (runner-platform) until",
+        "ImageOS moves here (docs/releasing.md, 'Moving a pin')."
+      ],
+      "platform": {
+        "RUNNER_OS": "macOS",
+        "RUNNER_ARCH": "ARM64",
+        "ImageOS": "macos26"
+      },
+      "compare": "pins",
+      "tools": {
+        "go": [
+          "go",
+          "version"
+        ]
+      },
+      "pins": {
+        "go": "go version go1.26.5 darwin/arm64"
+      }
+    }
+  },
+  "workflows": {
+    "ci": {
+      "$comment": [
+        "Every job of ci.yml is required on a push to main; none is skipped there. The four release steps are",
+        "left out of the Runner legs by ci.yml and run in Release contents, so each leg lists what it runs."
+      ],
+      "path": ".github/workflows/ci.yml",
+      "name": "CI",
+      "events": [
+        "push"
+      ],
+      "jobs": {
+        "runner-ubuntu-latest": {
+          "name": "Runner (ubuntu-latest)",
+          "job_id": "check",
+          "labels": [
+            "ubuntu-latest"
+          ],
+          "profile": "linux",
+          "checks": [
+            "local-paths",
+            "agent-docs",
+            "diff-check",
+            "go-fmt",
+            "go-vet",
+            "build-pinned",
+            "manual-gate",
+            "unit-tests",
+            "chain-tests",
+            "openapi",
+            "build",
+            "ci-evidence"
+          ]
+        },
+        "runner-macos-latest": {
+          "name": "Runner (macos-latest)",
+          "job_id": "check",
+          "labels": [
+            "macos-latest"
+          ],
+          "profile": "macos",
+          "checks": [
+            "local-paths",
+            "agent-docs",
+            "diff-check",
+            "go-fmt",
+            "go-vet",
+            "build-pinned",
+            "manual-gate",
+            "unit-tests",
+            "chain-tests",
+            "openapi",
+            "build",
+            "ci-evidence"
+          ]
+        },
+        "release-contents": {
+          "name": "Release contents",
+          "job_id": "release-contents",
+          "labels": [
+            "ubuntu-latest"
+          ],
+          "profile": "linux",
+          "checks": [
+            "release-toolchain",
+            "third-party-notices",
+            "release-archives",
+            "release-programs"
+          ]
+        }
+      }
+    }
+  },
+  "tag": {
+    "prerelease": true
+  }
+}

.github/workflows/ci.yml

@@ -3,16 +3,19 @@ name: CI
 # The check list is dev.json, run by scripts/dev/check.sh (the development kit's vendored runner);
 # the longer steps are scripts under scripts/checks/ with the comments they had here. Two jobs, both
 # required by branch protection under their names: `Runner (<os>)` runs every step but the release
-# ones on Linux and macOS, and `Release contents` runs the release steps with the exact toolchain
-# release.yml names. On a pull request the `Runner` jobs run only the steps jpdev.json maps the
-# changed paths to (`check.sh --changed`); on a push to main, and when release.yml calls this
-# workflow at a tag, they run the whole list. `Release contents` runs its four steps every time.
+# ones on Linux and macOS, and `Release contents` runs the release steps. Both set up the exact
+# toolchain release.yml names. On a pull request the `Runner` jobs run only the steps jpdev.json
+# maps the changed paths to (`check.sh --changed`); on a push to main they run the whole list.
+# `Release contents` runs its four steps every time. A push to main also records each job's
+# evidence (scripts/dev/release_evidence.py capture), which the release takes at that commit
+# instead of running this workflow again (docs/releasing.md, "CI at the tag").
 
 on:
   pull_request:
   push:
     branches: [main]
-  # The release workflow calls this one at a tag (release.yml).
+  # Kept; the release no longer calls this workflow. A refusal at the tag is answered by re-running
+  # main's run at the commit (docs/releasing.md), which keeps the evidence a push-to-main run's.
   workflow_call:
 
 permissions:
@@ -35,26 +38,42 @@ jobs:
       # First, so that a manifest that does not parse fails before the long steps start.
       - name: The manifest parses and its first step runs
         run: scripts/dev/check.sh --manifest dev.json --only local-paths
+      # The exact toolchain release.yml links the archives with, so that the integration tests a
+      # release takes as its proof ran under it (.github/ci-evidence.json pins it).
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.26.x'
+          go-version: '1.26.5'
           cache: true
       # The pinned Runtimes, the ownership-era Runner and the Gateway are checked out and built by
       # the build-pinned step (scripts/checks/build-pinned.sh).
       # A pull request runs the steps jpdev.json maps its changed paths to, and the whole list when
       # a path is unmapped or the mapping, dev.json or the runner changed. A push to main runs the
       # whole list, and only that run is what a release is held to. The release steps are skipped
-      # here either way: `Release contents` runs them.
+      # here either way: `Release contents` runs them. The summary is kept for the evidence.
       - name: Checks
         env:
           EVENT: ${{ github.event_name }}
           BASE_REF: ${{ github.base_ref }}
         run: |
+          set -o pipefail
           if [ "$EVENT" = pull_request ]; then
-            scripts/dev/check.sh --all --changed "origin/$BASE_REF" --skip release-toolchain,third-party-notices,release-archives,release-programs
+            scripts/dev/check.sh --all --changed "origin/$BASE_REF" --skip release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log"
           else
-            scripts/dev/check.sh --all --skip release-toolchain,third-party-notices,release-archives,release-programs
+            scripts/dev/check.sh --all --skip release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log"
           fi
+      # The commit, its input identities, this runner's image and tools, and every check's result:
+      # the evidence the release takes (.github/ci-evidence.json). Only a push to main records it.
+      - name: Record exact CI inputs and coverage
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: python3 scripts/dev/release_evidence.py capture --workflow ci --job runner-${{ matrix.os }} --check-log "${{ runner.temp }}/checks.log" --output "${{ runner.temp }}/evidence.json"
+      - name: Upload immutable CI evidence
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ci-evidence-ci-runner-${{ matrix.os }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/evidence.json
+          if-no-files-found: error
+          retention-days: 7
 
   # What a repository release carries (docs/releasing.md), checked on every
   # commit so a tag is never the first to find a gap. The toolchain is the exact
@@ -70,4 +89,17 @@ jobs:
           go-version: '1.26.5'
           cache: true
       - name: The release steps of dev.json
-        run: scripts/dev/check.sh --all --only release-toolchain,third-party-notices,release-archives,release-programs
+        run: |
+          set -o pipefail
+          scripts/dev/check.sh --all --only release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log"
+      - name: Record exact CI inputs and coverage
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: python3 scripts/dev/release_evidence.py capture --workflow ci --job release-contents --check-log "${{ runner.temp }}/checks.log" --output "${{ runner.temp }}/evidence.json"
+      - name: Upload immutable CI evidence
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ci-evidence-ci-release-contents-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/evidence.json
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/release.yml

@@ -7,7 +7,8 @@ name: Release
 # which is Runner's own record and is unaffected.
 #
 # A person decides there is a release and writes its notes
-# (docs/releases/<tag>.md). The workflow holds the tagged commit to CI, builds
+# (docs/releases/<tag>.md). The workflow holds the tagged commit to the CI
+# evidence main produced for it (docs/releasing.md, "CI at the tag"), builds
 # from it, runs three of the four archives on their own platforms, and
 # publishes no release until a maintainer approves the `production`
 # environment.
@@ -101,15 +102,48 @@ jobs:
             || { echo "The production environment could not be read; create it with a required reviewer (docs/releasing.md)" >&2; exit 1; }
           [ "${reviewers}" -ge 1 ] || { echo "The production environment requires no reviewer" >&2; exit 1; }
 
-  # The tagged commit's own ci.yml, called, so release.yml carries no second
-  # copy of the checks or of the Runtime and Gateway pins. A check main has
-  # gained since that commit is not run.
+  # The CI evidence main already produced at this exact commit, or a refusal
+  # (docs/releasing.md, "CI at the tag"): both Runner jobs and Release contents,
+  # every check passed, against the pins the commit holds, with the toolchain
+  # below. ci.yml is not called again, and a refusal stops the release: the
+  # fresh route is the operator's. Every other job's checks stay as they were.
   verify:
     name: CI at the tag
     needs: admit
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
     permissions:
       contents: read
-    uses: ./.github/workflows/ci.yml
+      # Reading the CI runs, their jobs and their evidence artifacts.
+      actions: read
+    steps:
+      - name: Check out the commit the run was started for
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      # The Linux jobs' evidence must name this runner's image and tools, and
+      # the Go toolchain the package job links with (.github/ci-evidence.json).
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: 1.26.5
+          cache: false
+
+      - name: Reuse main's exact-commit CI evidence, or refuse
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 scripts/dev/release_evidence.py verify --output "${{ runner.temp }}/release-evidence.json"
+
+      # The decision record, reuse or refusal, is also in the log of the step above.
+      - name: Keep the decision record
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-evidence-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/release-evidence.json
+          if-no-files-found: error
 
   package:
     name: Package

AGENTS.md

@@ -81,6 +81,11 @@ summary block, and anything you stopped on or chose not to do.
   `openapi.json`, `vectors/`, `docs/MAPPING-V2.md`, `docs/design/*.json` and test data across
   packages, so those paths select the test steps. A new step, a new input of a step or a test that
   reads a new file is mapped in `jpdev.json`; when unsure, map it broader.
+- A release takes main's CI evidence at the tagged commit instead of running CI again
+  (`scripts/dev/release_evidence.py`, configured by `.github/ci-evidence.json`; docs/releasing.md,
+  "CI at the tag"), so a change to a CI job's name, a matrix leg, the check list or the Go
+  toolchain, or a new step script that pins a companion or names must-run tests, changes that file
+  in the same pull request; `tools/test_ci_evidence.py` fails until it does.
 - Runner is a single-owner local Jobs pilot; do not add shared-user permissions, external actions
   or agent loops. Do not put real credentials, customer evidence or private operational records in
   examples or fixtures.

CLAUDE.md

@@ -81,6 +81,11 @@ summary block, and anything you stopped on or chose not to do.
   `openapi.json`, `vectors/`, `docs/MAPPING-V2.md`, `docs/design/*.json` and test data across
   packages, so those paths select the test steps. A new step, a new input of a step or a test that
   reads a new file is mapped in `jpdev.json`; when unsure, map it broader.
+- A release takes main's CI evidence at the tagged commit instead of running CI again
+  (`scripts/dev/release_evidence.py`, configured by `.github/ci-evidence.json`; docs/releasing.md,
+  "CI at the tag"), so a change to a CI job's name, a matrix leg, the check list or the Go
+  toolchain, or a new step script that pins a companion or names must-run tests, changes that file
+  in the same pull request; `tools/test_ci_evidence.py` fails until it does.
 - Runner is a single-owner local Jobs pilot; do not add shared-user permissions, external actions
   or agent loops. Do not put real credentials, customer evidence or private operational records in
   examples or fixtures.

dev.json

@@ -13,6 +13,7 @@
     {"name": "chain-tests", "run": "scripts/checks/chain-tests.sh", "toolchain": "go", "summary": "^(ok|FAIL) ", "timeout": 3000},
     {"name": "openapi", "run": "python3 -m json.tool openapi.json > /dev/null", "toolchain": "python"},
     {"name": "build", "run": "go build -trimpath -o /dev/null ./cmd/jpack-runner && go build -trimpath -o /dev/null ./cmd/jpack-source-worker", "toolchain": "go", "timeout": 3000},
+    {"name": "ci-evidence", "run": "python3 tools/test_ci_evidence.py", "toolchain": "python", "summary": "^(Ran |OK|FAILED)"},
     {"name": "release-toolchain", "run": "scripts/checks/release-toolchain.sh", "toolchain": "go", "timeout": 600},
     {"name": "third-party-notices", "run": "python3 tools/third_party_notices.py --check", "toolchain": "go", "timeout": 3000},
     {"name": "release-archives", "run": "python3 tools/test_release_archives.py", "toolchain": "go", "summary": "^(Ran |OK|FAILED)", "timeout": 3000},