From 9d28a1a2a71b0f536e4a2be64f46c6be71538705 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:18:25 -0400 Subject: [PATCH 1/5] Release: reuse main's exact-commit CI evidence at the tag, fail closed (#175) The release's "CI at the tag" job runs the development kit's release evidence gate (scripts/dev/release_evidence.py, vendored from v0.10.0) against .github/ci-evidence.json instead of calling ci.yml again. Main CI records each job's evidence on a push to main: both Runner legs and Release contents, every check passed, the input identities, the image and the toolchain. The Runner jobs now set up the exact Go toolchain release.yml names, so the integration tests a release takes ran under it. tools/test_ci_evidence.py holds the configuration to the workflows and the manifest, and the vendored gate to the kit's. Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .github/ci-evidence.json | 107 ++++ .github/workflows/ci.yml | 52 +- .github/workflows/release.yml | 44 +- AGENTS.md | 5 + CLAUDE.md | 5 + dev.json | 1 + jpdev.json | 10 +- scripts/checks/release-toolchain.sh | 7 +- scripts/dev/release_evidence.py | 736 ++++++++++++++++++++++++++++ tools/test_ci_evidence.py | 286 +++++++++++ 10 files changed, 1233 insertions(+), 20 deletions(-) create mode 100644 .github/ci-evidence.json create mode 100644 scripts/dev/release_evidence.py create mode 100644 tools/test_ci_evidence.py diff --git a/.github/ci-evidence.json b/.github/ci-evidence.json new file mode 100644 index 0000000..9617b5d --- /dev/null +++ b/.github/ci-evidence.json @@ -0,0 +1,107 @@ +{ + "$comment": [ + "The release evidence gate's configuration, read at the release commit by scripts/dev/release_evidence.py", + "(docs/releasing.md, 'CI at the tag'). tools/test_ci_evidence.py holds it to ci.yml, release.yml and dev.json.", + "inputs: the module pins; the script that pins the Runtime, earlier Runtime, old Runner and Gateway commits", + "the integration tests run against; the scripts that fail when a named real-companion test does not run and", + "pass; the toolchain check and the workflow naming the release toolchain; the runner whose summary is read." + ], + "schema": 2, + "repository": "Judgment-Pack/judgment-pack-runner", + "branch": "main", + "max_age_hours": 48, + "inputs": [ + "go.mod", + "go.sum", + "scripts/checks/build-pinned.sh", + "scripts/checks/manual-gate.sh", + "scripts/checks/chain-tests.sh", + "scripts/checks/unit-tests.sh", + "scripts/checks/release-toolchain.sh", + "scripts/dev/check.sh", + ".github/workflows/release.yml" + ], + "fresh_at_release": [ + "tag syntax, tag names the commit, commit on main", + "release notes", + "production environment reviewer", + "third-party notices with the release toolchain, packaging, checksums", + "contents of every archive", + "native archive smoke tests (linux/amd64, linux/arm64, darwin/arm64)", + "tag still names the commit, no existing release, attestation, draft", + "draft identity, notes and asset digests, publication, published asset digests" + ], + "profiles": { + "linux": { + "$comment": [ + "Both Linux legs, compared with the release job's own runner: the same hosted image and tools, and the", + "Go toolchain release.yml links the archives with, set up there the same way." + ], + "platform": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64" + }, + "compare": "release-runner", + "tools": { + "go": ["go", "version"], + "python": ["python3", "--version"], + "git": ["git", "--version"], + "bash": ["bash", "--version"] + }, + "pins": { + "go": "go version go1.26.5 linux/amd64" + } + }, + "macos": { + "$comment": [ + "The macOS leg, which the release job cannot resolve on its own runner: the platform and the exact Go", + "toolchain are pinned; the image version is recorded and not compared." + ], + "platform": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "ARM64" + }, + "compare": "pins", + "tools": { + "go": ["go", "version"] + }, + "pins": { + "go": "go version go1.26.5 darwin/arm64" + } + } + }, + "workflows": { + "ci": { + "$comment": [ + "Every job of ci.yml is required on a push to main; none is skipped there. The four release steps are", + "left out of the Runner legs by ci.yml and run in Release contents, so each leg lists what it runs." + ], + "path": ".github/workflows/ci.yml", + "name": "CI", + "events": ["push"], + "jobs": { + "runner-ubuntu-latest": { + "name": "Runner (ubuntu-latest)", + "job_id": "check", + "labels": ["ubuntu-latest"], + "profile": "linux", + "checks": ["local-paths", "agent-docs", "diff-check", "go-fmt", "go-vet", "build-pinned", "manual-gate", "unit-tests", "chain-tests", "openapi", "build", "ci-evidence"] + }, + "runner-macos-latest": { + "name": "Runner (macos-latest)", + "job_id": "check", + "labels": ["macos-latest"], + "profile": "macos", + "checks": ["local-paths", "agent-docs", "diff-check", "go-fmt", "go-vet", "build-pinned", "manual-gate", "unit-tests", "chain-tests", "openapi", "build", "ci-evidence"] + }, + "release-contents": { + "name": "Release contents", + "job_id": "release-contents", + "labels": ["ubuntu-latest"], + "profile": "linux", + "checks": ["release-toolchain", "third-party-notices", "release-archives", "release-programs"] + } + } + } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50653ab..ed907c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,16 +3,19 @@ name: CI # The check list is dev.json, run by scripts/dev/check.sh (the development kit's vendored runner); # the longer steps are scripts under scripts/checks/ with the comments they had here. Two jobs, both # required by branch protection under their names: `Runner ()` runs every step but the release -# ones on Linux and macOS, and `Release contents` runs the release steps with the exact toolchain -# release.yml names. On a pull request the `Runner` jobs run only the steps jpdev.json maps the -# changed paths to (`check.sh --changed`); on a push to main, and when release.yml calls this -# workflow at a tag, they run the whole list. `Release contents` runs its four steps every time. +# ones on Linux and macOS, and `Release contents` runs the release steps. Both set up the exact +# toolchain release.yml names. On a pull request the `Runner` jobs run only the steps jpdev.json +# maps the changed paths to (`check.sh --changed`); on a push to main they run the whole list. +# `Release contents` runs its four steps every time. A push to main also records each job's +# evidence (scripts/dev/release_evidence.py capture), which the release takes at that commit +# instead of running this workflow again (docs/releasing.md, "CI at the tag"). on: pull_request: push: branches: [main] - # The release workflow calls this one at a tag (release.yml). + # Kept; the release no longer calls this workflow. A refusal at the tag is answered by re-running + # main's run at the commit (docs/releasing.md), which keeps the evidence a push-to-main run's. workflow_call: permissions: @@ -35,26 +38,42 @@ jobs: # First, so that a manifest that does not parse fails before the long steps start. - name: The manifest parses and its first step runs run: scripts/dev/check.sh --manifest dev.json --only local-paths + # The exact toolchain release.yml links the archives with, so that the integration tests a + # release takes as its proof ran under it (.github/ci-evidence.json pins it). - uses: actions/setup-go@v5 with: - go-version: '1.26.x' + go-version: '1.26.5' cache: true # The pinned Runtimes, the ownership-era Runner and the Gateway are checked out and built by # the build-pinned step (scripts/checks/build-pinned.sh). # A pull request runs the steps jpdev.json maps its changed paths to, and the whole list when # a path is unmapped or the mapping, dev.json or the runner changed. A push to main runs the # whole list, and only that run is what a release is held to. The release steps are skipped - # here either way: `Release contents` runs them. + # here either way: `Release contents` runs them. The summary is kept for the evidence. - name: Checks env: EVENT: ${{ github.event_name }} BASE_REF: ${{ github.base_ref }} run: | + set -o pipefail if [ "$EVENT" = pull_request ]; then - scripts/dev/check.sh --all --changed "origin/$BASE_REF" --skip release-toolchain,third-party-notices,release-archives,release-programs + scripts/dev/check.sh --all --changed "origin/$BASE_REF" --skip release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log" else - scripts/dev/check.sh --all --skip release-toolchain,third-party-notices,release-archives,release-programs + scripts/dev/check.sh --all --skip release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log" fi + # The commit, its input identities, this runner's image and tools, and every check's result: + # the evidence the release takes (.github/ci-evidence.json). Only a push to main records it. + - name: Record exact CI inputs and coverage + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: python3 scripts/dev/release_evidence.py capture --workflow ci --job runner-${{ matrix.os }} --check-log "${{ runner.temp }}/checks.log" --output "${{ runner.temp }}/evidence.json" + - name: Upload immutable CI evidence + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-evidence-ci-runner-${{ matrix.os }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence.json + if-no-files-found: error + retention-days: 7 # What a repository release carries (docs/releasing.md), checked on every # commit so a tag is never the first to find a gap. The toolchain is the exact @@ -70,4 +89,17 @@ jobs: go-version: '1.26.5' cache: true - name: The release steps of dev.json - run: scripts/dev/check.sh --all --only release-toolchain,third-party-notices,release-archives,release-programs + run: | + set -o pipefail + scripts/dev/check.sh --all --only release-toolchain,third-party-notices,release-archives,release-programs | tee "${{ runner.temp }}/checks.log" + - name: Record exact CI inputs and coverage + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: python3 scripts/dev/release_evidence.py capture --workflow ci --job release-contents --check-log "${{ runner.temp }}/checks.log" --output "${{ runner.temp }}/evidence.json" + - name: Upload immutable CI evidence + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-evidence-ci-release-contents-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence.json + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dea400c..6cc5570 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,8 @@ name: Release # which is Runner's own record and is unaffected. # # A person decides there is a release and writes its notes -# (docs/releases/.md). The workflow holds the tagged commit to CI, builds +# (docs/releases/.md). The workflow holds the tagged commit to the CI +# evidence main produced for it (docs/releasing.md, "CI at the tag"), builds # from it, runs three of the four archives on their own platforms, and # publishes no release until a maintainer approves the `production` # environment. @@ -101,15 +102,48 @@ jobs: || { echo "The production environment could not be read; create it with a required reviewer (docs/releasing.md)" >&2; exit 1; } [ "${reviewers}" -ge 1 ] || { echo "The production environment requires no reviewer" >&2; exit 1; } - # The tagged commit's own ci.yml, called, so release.yml carries no second - # copy of the checks or of the Runtime and Gateway pins. A check main has - # gained since that commit is not run. + # The CI evidence main already produced at this exact commit, or a refusal + # (docs/releasing.md, "CI at the tag"): both Runner jobs and Release contents, + # every check passed, against the pins the commit holds, with the toolchain + # below. ci.yml is not called again, and a refusal stops the release: the + # fresh route is the operator's. Every other job's checks stay as they were. verify: name: CI at the tag needs: admit + runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read - uses: ./.github/workflows/ci.yml + # Reading the CI runs, their jobs and their evidence artifacts. + actions: read + steps: + - name: Check out the commit the run was started for + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + + # The Linux jobs' evidence must name this runner's image and tools, and + # the Go toolchain the package job links with (.github/ci-evidence.json). + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: 1.26.5 + cache: false + + - name: Reuse main's exact-commit CI evidence, or refuse + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/dev/release_evidence.py verify --output "${{ runner.temp }}/release-evidence.json" + + # The decision record, reuse or refusal, is also in the log of the step above. + - name: Keep the decision record + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-evidence-${{ github.run_attempt }} + path: ${{ runner.temp }}/release-evidence.json + if-no-files-found: error package: name: Package diff --git a/AGENTS.md b/AGENTS.md index 467b71e..e4ac027 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,11 @@ summary block, and anything you stopped on or chose not to do. `openapi.json`, `vectors/`, `docs/MAPPING-V2.md`, `docs/design/*.json` and test data across packages, so those paths select the test steps. A new step, a new input of a step or a test that reads a new file is mapped in `jpdev.json`; when unsure, map it broader. +- A release takes main's CI evidence at the tagged commit instead of running CI again + (`scripts/dev/release_evidence.py`, configured by `.github/ci-evidence.json`; docs/releasing.md, + "CI at the tag"), so a change to a CI job's name, a matrix leg, the check list or the Go + toolchain, or a new step script that pins a companion or names must-run tests, changes that file + in the same pull request; `tools/test_ci_evidence.py` fails until it does. - Runner is a single-owner local Jobs pilot; do not add shared-user permissions, external actions or agent loops. Do not put real credentials, customer evidence or private operational records in examples or fixtures. diff --git a/CLAUDE.md b/CLAUDE.md index 467b71e..e4ac027 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,11 @@ summary block, and anything you stopped on or chose not to do. `openapi.json`, `vectors/`, `docs/MAPPING-V2.md`, `docs/design/*.json` and test data across packages, so those paths select the test steps. A new step, a new input of a step or a test that reads a new file is mapped in `jpdev.json`; when unsure, map it broader. +- A release takes main's CI evidence at the tagged commit instead of running CI again + (`scripts/dev/release_evidence.py`, configured by `.github/ci-evidence.json`; docs/releasing.md, + "CI at the tag"), so a change to a CI job's name, a matrix leg, the check list or the Go + toolchain, or a new step script that pins a companion or names must-run tests, changes that file + in the same pull request; `tools/test_ci_evidence.py` fails until it does. - Runner is a single-owner local Jobs pilot; do not add shared-user permissions, external actions or agent loops. Do not put real credentials, customer evidence or private operational records in examples or fixtures. diff --git a/dev.json b/dev.json index 56ba9aa..e2543ab 100644 --- a/dev.json +++ b/dev.json @@ -13,6 +13,7 @@ {"name": "chain-tests", "run": "scripts/checks/chain-tests.sh", "toolchain": "go", "summary": "^(ok|FAIL) ", "timeout": 3000}, {"name": "openapi", "run": "python3 -m json.tool openapi.json > /dev/null", "toolchain": "python"}, {"name": "build", "run": "go build -trimpath -o /dev/null ./cmd/jpack-runner && go build -trimpath -o /dev/null ./cmd/jpack-source-worker", "toolchain": "go", "timeout": 3000}, + {"name": "ci-evidence", "run": "python3 tools/test_ci_evidence.py", "toolchain": "python", "summary": "^(Ran |OK|FAILED)"}, {"name": "release-toolchain", "run": "scripts/checks/release-toolchain.sh", "toolchain": "go", "timeout": 600}, {"name": "third-party-notices", "run": "python3 tools/third_party_notices.py --check", "toolchain": "go", "timeout": 3000}, {"name": "release-archives", "run": "python3 tools/test_release_archives.py", "toolchain": "go", "summary": "^(Ran |OK|FAILED)", "timeout": 3000}, diff --git a/jpdev.json b/jpdev.json index d472e5f..3878962 100644 --- a/jpdev.json +++ b/jpdev.json @@ -7,7 +7,7 @@ }, { "paths": ["go.mod", "go.sum"], - "checks": ["local-paths", "agent-docs", "diff-check", "release-toolchain", "release-archives"] + "checks": ["local-paths", "agent-docs", "diff-check", "release-toolchain", "release-archives", "ci-evidence"] }, { "paths": ["cmd/**", "internal/**", "tools/preparation-vectors/**"], @@ -27,7 +27,7 @@ }, { "paths": ["scripts/checks/build-pinned.sh", "scripts/checks/manual-gate.sh", "scripts/checks/unit-tests.sh", "scripts/checks/chain-tests.sh"], - "checks": ["local-paths", "agent-docs", "diff-check", "build-pinned", "manual-gate", "unit-tests", "chain-tests"] + "checks": ["local-paths", "agent-docs", "diff-check", "build-pinned", "manual-gate", "unit-tests", "chain-tests", "ci-evidence"] }, { "paths": ["THIRD_PARTY_NOTICES", "tools/third_party_notices.py"], @@ -43,7 +43,11 @@ }, { "paths": ["scripts/checks/release-toolchain.sh"], - "checks": ["local-paths", "agent-docs", "diff-check", "release-toolchain"] + "checks": ["local-paths", "agent-docs", "diff-check", "release-toolchain", "ci-evidence"] + }, + { + "paths": [".github/ci-evidence.json", "scripts/dev/release_evidence.py", "tools/test_ci_evidence.py"], + "checks": ["local-paths", "agent-docs", "diff-check", "ci-evidence"] }, { "paths": ["*.md", "docs/**", "LICENSE", "deploy/**", "scripts/checks/local-paths.sh", "scripts/sync-agent-docs.sh", "scripts/dev/handover_check.sh"], diff --git a/scripts/checks/release-toolchain.sh b/scripts/checks/release-toolchain.sh index a7efa72..0b3545a 100755 --- a/scripts/checks/release-toolchain.sh +++ b/scripts/checks/release-toolchain.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash # CI step: Check the toolchain is the one the release workflow names +# release.yml names it in each job that sets Go up (the package job, and the CI at the tag job whose +# runner the CI evidence is compared with): every one names the same version, and this job runs it. set -e -named="$(sed -n "s/^ *go-version: *['\"]\{0,1\}\([0-9][0-9.]*\)['\"]\{0,1\} *\$/\1/p" .github/workflows/release.yml)" -[ -n "$named" ] && [ "go$named" = "$(go env GOVERSION)" ] || { echo "release.yml names Go ${named:-nothing}; this job runs $(go env GOVERSION)"; exit 1; } +named="$(sed -n "s/^ *go-version: *['\"]\{0,1\}\([0-9][0-9.]*\)['\"]\{0,1\} *\$/\1/p" .github/workflows/release.yml | sort -u)" +[ -n "$named" ] && [ "$(printf '%s\n' "$named" | wc -l)" -eq 1 ] || { echo "release.yml names Go $(printf '%s' "${named:-nothing}" | tr '\n' ' '); its jobs must name one version"; exit 1; } +[ "go$named" = "$(go env GOVERSION)" ] || { echo "release.yml names Go ${named}; this job runs $(go env GOVERSION)"; exit 1; } diff --git a/scripts/dev/release_evidence.py b/scripts/dev/release_evidence.py new file mode 100644 index 0000000..6648d0c --- /dev/null +++ b/scripts/dev/release_evidence.py @@ -0,0 +1,736 @@ +#!/usr/bin/env python3 +# from judgment-pack-devkit v0.10.0 (lib/release_evidence.py) +"""release_evidence.py capture|verify: reuse the CI evidence main produced at the release commit. + +capture Runs in a CI job on the trusted branch, after the job's checks. Writes evidence.json for + the job's artifact: the commit, its input identities, the runner and tool identity and + the check results. +verify Runs in the release workflow. Reads this repository's Actions API for the configured + workflow's runs at the exact release commit, selects the newest applicable attempt, + verifies it, and writes the decision record, reuse or refuse, to standard output and to a + file. Missing, partial, stale, unreadable or different evidence refuses: nothing here + turns unavailable evidence into success. + +The configuration is .github/ci-evidence.json, schema 2 (docs/ci-evidence.md). There is no input +for a local evidence file, another API host, a green flag or an age override. +""" +import argparse +import datetime as dt +import hashlib +import io +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +from urllib.parse import quote +import zipfile +import zlib + +SCHEMA = 2 +MAX_AGE_HOURS = 48 +EVENTS = ('push', 'workflow_dispatch') +# Runs of these events are neither evidence nor part of the newer-attempt scan. +OUTSIDE_EVENT_PREFIX = 'pull_request' +OUTSIDE_EVENTS = ('merge_group',) +COMPARE = ('release-runner', 'pins') +# runner_group_id and runner_group_name of a job on a GitHub-hosted runner. +HOSTED = (0, 'GitHub Actions') +RUNNER_KEYS = ('ImageOS', 'ImageVersion', 'RUNNER_OS', 'RUNNER_ARCH') +EVIDENCE_LIMIT = 256 * 1024 +# evidence.json nests four levels; deeper JSON is refused before it is parsed, whatever the +# interpreter's recursion limit. +EVIDENCE_DEPTH = 16 +ARTIFACT_LIMIT = 1024 * 1024 +KEY = re.compile(r'[A-Za-z0-9_-]{1,64}') +REPOSITORY = re.compile(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+') +PATH = re.compile(r'[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*') +WORKFLOW_PATH = re.compile(r'\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml') +TAG = re.compile(r'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)') +SHA = re.compile(r'[0-9a-f]{40}') +OBJECT_ID = re.compile(r'[0-9a-f]{40}|[0-9a-f]{64}') + + +class Refusal(Exception): + def __init__(self, code, detail=''): + super().__init__(code) + self.code = code + self.detail = detail + + +def need(condition, code, detail=''): + if not condition: + raise Refusal(code, detail) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def command(argv, code, limit=8 * 1024 * 1024): + # Spooled under TMPDIR: remote output is never accumulated unbounded in memory, and stderr, + # which may echo a request, is never logged. + with tempfile.TemporaryFile() as output: + try: + result = subprocess.run(argv, stdin=subprocess.DEVNULL, stdout=output, + stderr=subprocess.DEVNULL, timeout=45, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + raise Refusal(code, argv[0] + ' did not run') from error + need(result.returncode == 0, code, '%s exited %d' % (argv[0], result.returncode)) + need(output.tell() <= limit, 'response-size', argv[0]) + output.seek(0) + return output.read() + + +def git(*args, code='git-unavailable'): + return command(['git', *args], code) + + +class API: + """This repository's REST API through `gh api`, the token the workflow grants.""" + + def __init__(self, repository): + self.base = 'repos/' + repository + + def get(self, suffix, binary=False): + raw = command(['gh', 'api', '--hostname', 'github.com', '-H', + 'Accept: application/vnd.github+json', '-H', + 'X-GitHub-Api-Version: 2026-03-10', + self.base + ('/' + suffix if suffix else '')], + 'api-unavailable') + return raw if binary else json.loads(raw) + + def pages(self, suffix, key): + """Every row of a listing: stable total, full pages, unique ids, under 1,000 rows.""" + rows, count = [], None + for page in range(1, 11): + separator = '&' if '?' in suffix else '?' + value = self.get(suffix + separator + 'per_page=100&page=' + str(page)) + total, part = value.get('total_count'), value.get(key) + # Filtered workflow-run searches stop at 1,000 results. + need(type(total) is int and 0 <= total < 1000, 'pagination-cap', suffix) + need(isinstance(part, list) and len(part) <= 100, 'pagination-shape', suffix) + need(count is None or count == total, 'pagination-changed', suffix) + count = total + rows.extend(part) + need(len(rows) <= count, 'pagination-count', suffix) + if len(rows) == count: + need(len({row['id'] for row in rows}) == count, 'pagination-duplicate', suffix) + return rows + need(len(part) == 100, 'pagination-partial', suffix) + raise Refusal('pagination-cap', suffix) + + +def timestamp(value): + need(isinstance(value, str), 'timestamp') + moment = dt.datetime.fromisoformat(value.replace('Z', '+00:00')) + need(moment.tzinfo is not None, 'timestamp') + return moment + + +def positive(value): + return type(value) is int and value > 0 + + +# --- configuration ------------------------------------------------------------------------------- + +def check_config(config): + """Refuse anything but a complete schema 2 configuration. "$comment" is allowed, and ignored, + in every object whose members are fixed names.""" + def fail(where, why): + raise Refusal('config-invalid', where + ': ' + why) + + def fields(value, where, required, optional=()): + if not isinstance(value, dict): + fail(where, 'an object is required') + unknown = sorted(set(value) - set(required) - set(optional) - {'$comment'}) + missing = sorted(set(required) - set(value)) + if unknown or missing: + fail(where, 'unknown %s, missing %s' % (unknown, missing)) + note = value.get('$comment', '') + if not (isinstance(note, str) or + (isinstance(note, list) and all(isinstance(line, str) for line in note))): + fail(where + '.$comment', 'a string or a list of strings is required') + + def text(value, where, pattern=None): + if not isinstance(value, str) or not value or (pattern and not pattern.fullmatch(value)): + fail(where, 'a string%s is required' % (' matching ' + pattern.pattern if pattern else '')) + + def texts(value, where, pattern=None, empty=True): + if not isinstance(value, list) or (not value and not empty): + fail(where, 'a list%s is required' % ('' if empty else ' with at least one entry')) + for index, item in enumerate(value): + text(item, '%s[%d]' % (where, index), pattern) + if len(set(value)) != len(value): + fail(where, 'an entry is repeated') + + def named(value, where, empty=False): + if not isinstance(value, dict) or (not value and not empty): + fail(where, 'an object%s is required' % ('' if empty else ' with at least one member')) + for name in value: + text(name, where + ' member name', KEY) + + def path(value, where, pattern=PATH): + text(value, where, pattern) + if any(part in ('.', '..') for part in value.split('/')): + fail(where, 'a path without . or .. segments is required') + + fields(config, 'config', ['schema', 'repository', 'branch', 'max_age_hours', 'inputs', + 'profiles', 'workflows'], ['fresh_at_release']) + if config['schema'] != SCHEMA or type(config['schema']) is not int: + fail('config.schema', 'must be %d' % SCHEMA) + text(config['repository'], 'config.repository', REPOSITORY) + path(config['branch'], 'config.branch') + hours = config['max_age_hours'] + if type(hours) is not int or not 1 <= hours <= MAX_AGE_HOURS: + fail('config.max_age_hours', 'an integer from 1 to %d is required' % MAX_AGE_HOURS) + texts(config['inputs'], 'config.inputs') + for index, item in enumerate(config['inputs']): + path(item, 'config.inputs[%d]' % index) + texts(config.get('fresh_at_release', []), 'config.fresh_at_release') + + named(config['profiles'], 'config.profiles') + for name, profile in config['profiles'].items(): + where = 'config.profiles.' + name + fields(profile, where, ['platform', 'compare', 'tools'], ['pins']) + fields(profile['platform'], where + '.platform', ['RUNNER_OS', 'RUNNER_ARCH'], ['ImageOS']) + for key, value in profile['platform'].items(): + if key != '$comment': + text(value, '%s.platform.%s' % (where, key)) + if profile['compare'] not in COMPARE: + fail(where + '.compare', 'one of %s is required' % (COMPARE,)) + named(profile['tools'], where + '.tools', empty=True) + for tool, argv in profile['tools'].items(): + texts(argv, '%s.tools.%s' % (where, tool), empty=False) + pins = profile.get('pins', {}) + named(pins, where + '.pins', empty=True) + for tool, pin in pins.items(): + text(pin, '%s.pins.%s' % (where, tool)) + if not set(pins) <= set(profile['tools']): + fail(where + '.pins', 'every pin names a tool of the profile') + if profile['compare'] == 'pins' and (not pins or set(pins) != set(profile['tools'])): + fail(where + '.pins', 'compare "pins" needs a pin for every tool, and a tool') + + named(config['workflows'], 'config.workflows') + for key, workflow in config['workflows'].items(): + where = 'config.workflows.' + key + fields(workflow, where, ['path', 'name', 'events', 'jobs'], ['skipped_jobs']) + path(workflow['path'], where + '.path', WORKFLOW_PATH) + text(workflow['name'], where + '.name') + texts(workflow['events'], where + '.events', empty=False) + if not set(workflow['events']) <= set(EVENTS): + fail(where + '.events', 'only %s are trusted-branch events' % (EVENTS,)) + texts(workflow.get('skipped_jobs', []), where + '.skipped_jobs') + named(workflow['jobs'], where + '.jobs') + names = list(workflow.get('skipped_jobs', [])) + for job_key, spec in workflow['jobs'].items(): + at = '%s.jobs.%s' % (where, job_key) + fields(spec, at, ['name', 'job_id', 'labels', 'profile', 'checks'], + ['optional_skips', 'setup']) + text(spec['name'], at + '.name') + text(spec['job_id'], at + '.job_id', KEY) + texts(spec['labels'], at + '.labels', empty=False) + if spec['profile'] not in config['profiles']: + fail(at + '.profile', 'names no profile') + if spec['checks'] != 'all': + texts(spec['checks'], at + '.checks') + texts(spec.get('optional_skips', []), at + '.optional_skips') + if spec['checks'] != 'all' and not set(spec.get('optional_skips', [])) <= set(spec['checks']): + fail(at + '.optional_skips', 'every optional skip names a check of the job') + if type(spec.get('setup', False)) is not bool: + fail(at + '.setup', 'true or false is required') + names.append(spec['name']) + if len(names) != len(set(names)): + fail(where, 'a job name is repeated') + + +def helper_path(root): + try: + return Path(__file__).resolve().relative_to(root.resolve()).as_posix() + except ValueError as error: + raise Refusal('helper-source', 'the helper does not run from the checkout') from error + + +def source_context(config_path): + """The release commit and every input identity, read from the checkout of that commit.""" + sha = os.environ.get('GITHUB_SHA', '') + need(SHA.fullmatch(sha), 'source-sha') + need(PATH.fullmatch(config_path) and not {'.', '..'} & set(config_path.split('/')), 'config-source') + root = Path(git('rev-parse', '--show-toplevel').decode().strip()) + need(root.resolve() == Path.cwd().resolve(), 'checkout-root') + need(git('rev-parse', 'HEAD').decode().strip() == sha, 'checkout-sha') + need(git('status', '--porcelain', '--untracked-files=no') == b'', 'checkout-changed') + committed = git('show', sha + ':' + config_path, code='config-source') + need(Path(config_path).read_bytes() == committed, 'config-source') + config = json.loads(committed) + check_config(config) + need(config['repository'] == os.environ.get('GITHUB_REPOSITORY'), 'context-repository') + repository_id = int(os.environ.get('GITHUB_REPOSITORY_ID', '0')) + need(repository_id > 0, 'context-repository-id') + helper = helper_path(root) + need(Path(helper).read_bytes() == git('show', sha + ':' + helper, code='helper-source'), + 'helper-source') + # The manifest, the configuration, the helper and each workflow file are inputs whatever the + # configuration lists; each is identified by its git object id at the commit. + paths = sorted(set(config['inputs']) | {config_path, 'dev.json', helper} | + {workflow['path'] for workflow in config['workflows'].values()}) + ids = git('rev-parse', *[sha + ':' + path for path in paths], code='input-missing').decode().split() + need(len(ids) == len(paths) and all(OBJECT_ID.fullmatch(oid) for oid in ids), 'input-missing') + return config, {'schema': SCHEMA, 'repository': config['repository'], + 'repository_id': repository_id, 'source': sha, 'inputs': dict(zip(paths, ids))} + + +# --- the evidence a CI job records -------------------------------------------------------------- + +def runtime(profile): + """This runner's image, platform and tool identity, for a profile.""" + runner = {key: os.environ.get(key, '') for key in RUNNER_KEYS} + need(all(runner.values()), 'runner-identity') + for key, value in profile['platform'].items(): + if key != '$comment': + need(runner[key] == value, 'runner-platform', '%s is %s, not %s' % (key, runner[key], value)) + if runner['RUNNER_OS'] == 'Linux': + runner['os_release'] = digest(Path('/etc/os-release').read_bytes()) + tools = {name: command(argv, 'tool-identity', 65536).decode().strip() + for name, argv in profile['tools'].items()} + need(all(tools.values()), 'tool-identity') + return {'runner': runner, 'tools': tools} + + +def required_checks(spec): + checks = spec['checks'] + if checks == 'all': + manifest = json.loads(Path('dev.json').read_text(encoding='utf-8')) + checks = [row['name'] for row in manifest['checks']] + if 'parity' in manifest: + checks.append('parity') + result = list(checks) + if spec.get('setup'): + manifest = json.loads(Path('dev.json').read_text(encoding='utf-8')) + result += ['setup:' + row['name'] for row in manifest['setup']] + need(len(result) == len(set(result)), 'check-config') + return result + + +def coverage(spec, logs): + result = {} + for kind, path in logs: + text = Path(path).read_text(encoding='utf-8') + need('infrastructure failure:' not in text, 'check-infrastructure') + for status, name in re.findall(r'^(PASS|SKIP|FAIL|NOT RUN)\s+(\S+)(?:\s|$)', text, re.M): + key = ('setup:' if kind == 'setup' else '') + name + need(key not in result, 'check-duplicate', key) + result[key] = status + validate_coverage(spec, result) + return result + + +def validate_coverage(spec, result): + need(isinstance(result, dict), 'check-coverage') + required = set(required_checks(spec)) + need(set(result) == required, 'check-coverage', 'missing %s, unexpected %s' % ( + sorted(required - set(result)), sorted(set(result) - required))) + for name, status in sorted(result.items()): + need(status == 'PASS' or (status == 'SKIP' and name in spec.get('optional_skips', [])), + 'check-result', '%s %s' % (status, name)) + + +def evidence(source, workflow_key, workflow, job_key, run_id, attempt, event, branch, runtime_identity, + check_results): + """The one shape of a job's evidence: what capture writes and what verify expects.""" + return dict(source, workflow=workflow_key, workflow_path=workflow['path'], + workflow_name=workflow['name'], job=job_key, run_id=run_id, attempt=attempt, + event=event, ref='refs/heads/' + branch, runtime=runtime_identity, + coverage=check_results) + + +def capture(config, source, workflow_key, job_key, logs): + workflow = config['workflows'].get(workflow_key) + need(workflow is not None, 'capture-job', 'no workflow %s' % workflow_key) + spec = workflow['jobs'].get(job_key) + need(spec is not None, 'capture-job', 'no job %s' % job_key) + ref = 'refs/heads/' + config['branch'] + need(os.environ.get('GITHUB_REF') == ref, 'capture-ref') + need(os.environ.get('GITHUB_EVENT_NAME') in workflow['events'], 'capture-event') + need(os.environ.get('GITHUB_JOB') == spec['job_id'], 'capture-job') + # The workflow file that is running is the configured one, at the commit being recorded. + need(os.environ.get('GITHUB_WORKFLOW_REF') == '%s/%s@%s' % (source['repository'], + workflow['path'], ref) and + os.environ.get('GITHUB_WORKFLOW_SHA') == source['source'], 'capture-workflow') + run_id, attempt = int(os.environ['GITHUB_RUN_ID']), int(os.environ['GITHUB_RUN_ATTEMPT']) + need(positive(run_id) and positive(attempt), 'capture-run') + return evidence(source, workflow_key, workflow, job_key, run_id, attempt, + os.environ['GITHUB_EVENT_NAME'], config['branch'], + runtime(config['profiles'][spec['profile']]), coverage(spec, logs)) + + +# --- the release's verification ----------------------------------------------------------------- + +def run_identity(run, source, workflow, workflow_id, branch): + repository = (source['repository'], source['repository_id']) + need((run['repository']['full_name'], run['repository']['id']) == repository and + (run['head_repository']['full_name'], run['head_repository']['id']) == repository, + 'run-repository', 'run %s' % run.get('id')) + need(run['head_sha'] == source['source'], 'run-sha', 'run %s is at %s' % (run.get('id'), run['head_sha'])) + need(run['workflow_id'] == workflow_id and run['path'] == workflow['path'], 'run-workflow', + 'run %s' % run.get('id')) + need(run['head_branch'] == branch and run['event'] in workflow['events'] and + run['pull_requests'] == [], 'run-trigger', 'run %s' % run.get('id')) + need(positive(run['id']) and positive(run['run_attempt']), 'run-identity') + + +def attempt_order(run): + return (timestamp(run['run_started_at']), timestamp(run['created_at']), run['id']) + + +def select(api, source, workflow, workflow_id, branch, trail=None): + """The newest applicable attempt, never an older green one. + + Applicable: a run of this workflow at the exact commit, on the trusted branch, by a trusted + event; only an applicable run is evidence. Every run of this repository on the trusted branch + at the commit, whatever its trigger, takes part in the scan: none may be unfinished, and none + whose latest attempt started after the selected one's may have failed. The selected run is the + applicable one whose latest attempt started last, whatever its result.""" + runs = api.pages('actions/workflows/%d/runs?head_sha=%s' % (workflow_id, source['source']), + 'workflow_runs') + applicable = [run for run in runs + if run.get('head_branch') == branch and run.get('event') in workflow['events']] + if trail is not None: + trail['runs_at_commit'] = len(runs) + trail['applicable'] = len(applicable) + newest = sorted(runs, key=lambda run: str(run.get('run_started_at')), reverse=True) + trail['candidates'] = [{key: run.get(key) for key in ('id', 'run_attempt', 'event', 'head_branch', + 'status', 'conclusion', 'run_started_at')} + for run in newest[:20]] + need(applicable, 'run-missing', 'no %s run of %s at %s' % ('/'.join(workflow['events']), + workflow['path'], source['source'])) + for run in applicable: + run_identity(run, source, workflow, workflow_id, branch) + # A schedule, workflow_run or repository_dispatch run on the trusted branch is scanned. A pull + # request or merge queue run, another branch's run or another head repository's run tests + # other code or can be started from outside: never evidence, and never a block either. + scanned = [run for run in runs if run.get('head_branch') == branch and + not str(run.get('event')).startswith(OUTSIDE_EVENT_PREFIX) and + run.get('event') not in OUTSIDE_EVENTS and + (run.get('head_repository') or {}).get('id') == source['repository_id']] + unfinished = [run['id'] for run in scanned if run['status'] != 'completed'] + need(not unfinished, 'run-unfinished', 'runs %s have not completed' % unfinished) + chosen = max(applicable, key=attempt_order) + newer = ['%s (%s, %s)' % (run['id'], run['event'], run['conclusion']) for run in scanned + if attempt_order(run) > attempt_order(chosen) and run['conclusion'] != 'success'] + need(not newer, 'run-newer-failed', 'runs %s started after run %s and did not succeed' % ( + newer, chosen['id'])) + return chosen + + +def selection(run): + return (run['id'], run['run_attempt'], run['status'], run['conclusion']) + + +def tag_object(api, name, sha): + """The annotated tag's object id, after checking that the tag peels to the release commit.""" + need(isinstance(name, str) and TAG.fullmatch(name), 'tag-syntax') + ref = api.get('git/ref/tags/' + quote(name, safe='')) + need(ref['ref'] == 'refs/tags/' + name and ref['object']['type'] == 'tag', 'tag-annotated') + original = ref['object']['sha'] + obj = ref['object'] + for _ in range(5): + if obj['type'] != 'tag': + break + obj = api.get('git/tags/' + obj['sha'])['object'] + need(obj['type'] == 'commit' and obj['sha'] == sha, 'tag-commit', + '%s names %s %s' % (name, obj['type'], obj['sha'])) + return original + + +def check_jobs(jobs, run, workflow, branch): + names = [job['name'] for job in jobs] + expected = [spec['name'] for spec in workflow['jobs'].values()] + workflow.get('skipped_jobs', []) + need(len(names) == len(set(names)), 'job-coverage', 'a job name is repeated') + need(set(names) == set(expected), 'job-coverage', 'missing %s, unexpected %s' % ( + sorted(set(expected) - set(names)), sorted(set(names) - set(expected)))) + by_name = {job['name']: job for job in jobs} + for job in jobs: + need(job['run_id'] == run['id'] and job['run_attempt'] == run['run_attempt'] and + job['head_sha'] == run['head_sha'] and job['head_branch'] == branch and + job['workflow_name'] == workflow['name'], 'job-identity', job['name']) + need(job['status'] == 'completed', 'job-result', '%s is %s' % (job['name'], job['status'])) + for spec in workflow['jobs'].values(): + job = by_name[spec['name']] + need(job['conclusion'] != 'skipped', 'job-skipped', spec['name']) + need(job['conclusion'] == 'success', 'job-result', '%s: %s' % (spec['name'], job['conclusion'])) + need(job['labels'] == spec['labels'] and + (job['runner_group_id'], job['runner_group_name']) == HOSTED, 'job-runner', spec['name']) + for name in workflow.get('skipped_jobs', []): + need(by_name[name]['conclusion'] == 'skipped', 'job-not-skipped', + '%s: %s' % (name, by_name[name]['conclusion'])) + return by_name + + +def nesting_within(data, limit): + """Whether no array or object in the JSON text nests deeper than limit (strings skipped).""" + depth, in_string, escaped = 0, False, False + for byte in data: + if in_string: + if escaped: + escaped = False + elif byte == 0x5c: + escaped = True + elif byte == 0x22: + in_string = False + elif byte == 0x22: + in_string = True + elif byte in (0x5b, 0x7b): + depth += 1 + if depth > limit: + return False + elif byte in (0x5d, 0x7d): + depth -= 1 + return True + + +def read_artifact(api, artifact, run, job): + need(artifact['expired'] is False, 'artifact-expired', artifact['name']) + size = artifact['size_in_bytes'] + need(type(size) is int and 0 < size <= ARTIFACT_LIMIT, 'artifact-size', artifact['name']) + origin = artifact['workflow_run'] + need(origin['id'] == run['id'] and origin['head_sha'] == run['head_sha'] and + origin['head_branch'] == run['head_branch'] and + origin['repository_id'] == run['repository']['id'] and + origin['head_repository_id'] == run['repository']['id'], 'artifact-run', artifact['name']) + need(timestamp(job['started_at']) <= timestamp(artifact['created_at']) <= + timestamp(job['completed_at']), 'artifact-interval', artifact['name']) + need(positive(artifact['id']), 'artifact-run', artifact['name']) + raw = api.get('actions/artifacts/%d/zip' % artifact['id'], binary=True) + need(len(raw) == size and artifact['digest'] == 'sha256:' + digest(raw), 'artifact-digest', + artifact['name']) + # A malformed or unreadable archive or document is one refusal, unavailable-or-malformed, and + # is told apart from a well-formed archive of another shape (artifact-entries) before that + # check. The archive is read, never extracted, and no declared size is trusted. + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entries = archive.infolist() + need(all(entry.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for entry in entries), + 'unavailable-or-malformed', artifact['name'] + ': a compression other than stored or deflated') + need(sum(entry.file_size for entry in entries) <= EVIDENCE_LIMIT, 'unavailable-or-malformed', + '%s: declared over %d bytes' % (artifact['name'], EVIDENCE_LIMIT)) + need(len(entries) == 1 and entries[0].filename == 'evidence.json' and not entries[0].is_dir(), + 'artifact-entries', artifact['name']) + with archive.open(entries[0]) as member: + data = member.read(EVIDENCE_LIMIT + 1) + need(len(data) <= EVIDENCE_LIMIT, 'unavailable-or-malformed', + '%s: over %d bytes when read' % (artifact['name'], EVIDENCE_LIMIT)) + need(nesting_within(data, EVIDENCE_DEPTH), 'unavailable-or-malformed', + '%s: nesting deeper than %d' % (artifact['name'], EVIDENCE_DEPTH)) + payload = json.loads(data) + need(isinstance(payload, dict), 'unavailable-or-malformed', + artifact['name'] + ': evidence.json is not an object') + return payload + + +def check_runtime(profile, recorded, here, job_key): + """Toolchain identity: the pinned values, and for "release-runner" what this runner resolves.""" + need(isinstance(recorded, dict) and set(recorded) == {'runner', 'tools'} and + isinstance(recorded['runner'], dict) and isinstance(recorded['tools'], dict), + 'runtime-identity', job_key + ': no runtime record') + runner, tools = recorded['runner'], recorded['tools'] + for key, value in profile['platform'].items(): + if key != '$comment': + need(runner.get(key) == value, 'runtime-identity', + '%s: %s is %s, not %s' % (job_key, key, runner.get(key), value)) + need(set(tools) == set(profile['tools']), 'runtime-identity', job_key + ': tool set') + for tool, pin in sorted(profile.get('pins', {}).items()): + need(tools[tool] == pin, 'runtime-identity', '%s: %s is %r, pinned %r' % ( + job_key, tool, tools[tool], pin)) + if profile['compare'] == 'release-runner': + changed = sorted(key for key in set(runner) | set(here['runner']) + if runner.get(key) != here['runner'].get(key)) + changed += sorted(tool for tool in tools if tools[tool] != here['tools'].get(tool)) + need(recorded == here, 'runtime-identity', '%s: differs from this runner in %s' % ( + job_key, changed)) + + +def verify_workflow(api, config, source, key, now, here, trail): + workflow, branch = config['workflows'][key], config['branch'] + resolved = api.get('actions/workflows/' + quote(Path(workflow['path']).name, safe='')) + need(resolved['path'] == workflow['path'] and resolved['name'] == workflow['name'] and + positive(resolved['id']) and resolved['state'] == 'active', 'workflow-identity', + '%s is %s (%s)' % (workflow['path'], resolved.get('name'), resolved.get('state'))) + trail.update(workflow_id=resolved['id'], path=workflow['path'], name=workflow['name'], + events=workflow['events']) + chosen = select(api, source, workflow, resolved['id'], branch, trail) + trail.update(run_id=chosen['id'], attempt=chosen['run_attempt'], event=chosen['event'], + status=chosen['status'], conclusion=chosen['conclusion'], + run_started_at=chosen['run_started_at']) + run = api.get('actions/runs/%d' % chosen['id']) + run_identity(run, source, workflow, resolved['id'], branch) + need((run['id'], run['run_attempt']) == (chosen['id'], chosen['run_attempt']), 'run-race') + need(run['status'] == 'completed' and run['conclusion'] == 'success', 'run-result', + 'run %d attempt %d: %s %s' % (run['id'], run['run_attempt'], run['status'], run['conclusion'])) + age = (now - timestamp(run['run_started_at'])).total_seconds() + need(0 <= age <= config['max_age_hours'] * 3600, 'run-age', + 'run %d started %.1f hours ago; the limit is %d' % (run['id'], age / 3600, config['max_age_hours'])) + trail['age_hours'] = round(age / 3600, 2) + # Reusable workflows the run resolved, each with the commit it ran (an audit record only). + trail['referenced_workflows'] = run.get('referenced_workflows', []) + + jobs = api.pages('actions/runs/%d/attempts/%d/jobs' % (run['id'], run['run_attempt']), 'jobs') + by_name = check_jobs(jobs, run, workflow, branch) + artifacts = api.pages('actions/runs/%d/artifacts' % run['id'], 'artifacts') + trail['jobs'] = {} + for job_key, spec in workflow['jobs'].items(): + job = by_name[spec['name']] + name = 'ci-evidence-%s-%s-%d' % (key, job_key, run['run_attempt']) + matching = [artifact for artifact in artifacts if artifact['name'] == name] + need(len(matching) == 1, 'artifact-coverage', '%d artifacts named %s' % (len(matching), name)) + payload = read_artifact(api, matching[0], run, job) + expected = evidence(source, key, workflow, job_key, run['id'], run['run_attempt'], run['event'], + branch, payload.get('runtime'), payload.get('coverage')) + for field in ('schema', 'repository', 'repository_id', 'source', 'workflow', 'workflow_path', + 'workflow_name', 'job', 'run_id', 'attempt', 'event', 'ref'): + need(payload.get(field) == expected[field], 'evidence-identity', + '%s: %s is %r' % (job_key, field, payload.get(field))) + recorded_inputs = payload.get('inputs') + need(isinstance(recorded_inputs, dict) and recorded_inputs == source['inputs'], + 'input-identity', '%s: %s' % (job_key, sorted( + path for path in set(source['inputs']) | set(recorded_inputs or {}) + if (recorded_inputs or {}).get(path) != source['inputs'].get(path)))) + profile = config['profiles'][spec['profile']] + if profile['compare'] == 'release-runner' and spec['profile'] not in here: + here[spec['profile']] = runtime(profile) + check_runtime(profile, payload.get('runtime'), here.get(spec['profile']), job_key) + validate_coverage(spec, payload.get('coverage')) + need(payload == expected, 'evidence-identity', job_key + ': unexpected members') + trail['jobs'][job_key] = { + 'name': spec['name'], 'job_id': job['id'], 'result': job['conclusion'], + 'labels': job['labels'], 'started_at': job['started_at'], + 'completed_at': job['completed_at'], + 'artifact': {'id': matching[0]['id'], 'name': name, 'digest': matching[0]['digest']}, + 'runtime': payload['runtime'], 'coverage': payload['coverage']} + trail['skipped_jobs'] = {name: by_name[name]['conclusion'] + for name in workflow.get('skipped_jobs', [])} + + # The attempt read is still the run's latest, and the run still the newest applicable one. + fields = ('id', 'run_attempt', 'status', 'conclusion', 'run_started_at', 'updated_at') + after = api.get('actions/runs/%d' % run['id']) + need({field: after.get(field) for field in fields} == {field: run.get(field) for field in fields}, + 'attempt-changed', 'run %d' % run['id']) + latest = select(api, source, workflow, resolved['id'], branch) + need(selection(latest) == selection(run), 'latest-changed', 'run %d attempt %d' % ( + latest['id'], latest['run_attempt'])) + return resolved['id'], run + + +def verify(api, config, source, release_tag, now, record): + branch = config['branch'] + repository = api.get('') + need((repository['full_name'], repository['id']) == (source['repository'], source['repository_id']), + 'repository-identity', '%s (%s)' % (repository.get('full_name'), repository.get('id'))) + need(repository['default_branch'] == branch, 'branch-default', + 'the default branch is %s, the configuration trusts %s' % (repository['default_branch'], branch)) + first_tag = tag_object(api, release_tag, source['source']) + record['tag_object'] = first_tag + here, chosen = {}, {} + for key in config['workflows']: + record['workflows'][key] = {} + chosen[key] = verify_workflow(api, config, source, key, now, here, record['workflows'][key]) + # Collection takes time: every workflow's selection, then the tag, once more at the end. + for key, (workflow_id, run) in chosen.items(): + final = select(api, source, config['workflows'][key], workflow_id, branch) + need(selection(final) == selection(run), 'final-latest-changed', key) + need(tag_object(api, release_tag, source['source']) == first_tag, 'tag-changed') + + +# Malformed or missing API data or files: a refusal like any other, never a success. +MALFORMED = (KeyError, IndexError, ValueError, TypeError, AttributeError, OSError, EOFError, + NotImplementedError, RecursionError, zipfile.BadZipFile, zlib.error) + + +def refusal_of(error): + if isinstance(error, Refusal): + return {'code': error.code, 'detail': error.detail} + detail = type(error).__name__ + if isinstance(error, KeyError) and error.args: + detail += ' ' + repr(error.args[0])[:80] + return {'code': 'unavailable-or-malformed', 'detail': detail} + + +def decide(api, config, source, release_tag, now, record): + """The decision is reuse only when every check of verify passed.""" + try: + verify(api, config, source, release_tag, now, record) + except (Refusal,) + MALFORMED as error: + record['decision'], record['refusal'] = 'refuse', refusal_of(error) + else: + record['decision'], record['refusal'] = 'reuse', None + return record + + +def run_verify(args): + now = dt.datetime.now(dt.timezone.utc) + output = Path(args.output or 'release-evidence.json') + record = {'schema': SCHEMA, 'decision': 'refuse', 'refusal': None, + 'tag': os.environ.get('RELEASE_TAG', ''), 'checked_at': now.isoformat(), + 'workflows': {}} + try: + config, source = source_context(args.config) + except (Refusal,) + MALFORMED as error: + record['refusal'] = refusal_of(error) + else: + record.update(repository=source['repository'], repository_id=source['repository_id'], + source=source['source'], inputs=source['inputs'], branch=config['branch'], + max_age_hours=config['max_age_hours'], + fresh_at_release=config.get('fresh_at_release', [])) + decide(API(source['repository']), config, source, record['tag'], now, record) + text = json.dumps(record, indent=2, sort_keys=True) + print(text) + try: + output.write_text(text + '\n', encoding='utf-8') + except OSError: + print('release-evidence: refuse: the decision record could not be written to %s' % output, + file=sys.stderr) + return 1 + if record['decision'] != 'reuse': + refusal = record['refusal'] + print('release-evidence: refuse: %s%s (decision record: %s)' % ( + refusal['code'], ': ' + refusal['detail'] if refusal['detail'] else '', output), + file=sys.stderr) + return 1 + print('release-evidence: reuse: %s (decision record: %s)' % (', '.join( + '%s run %s attempt %s' % (key, value['run_id'], value['attempt']) + for key, value in record['workflows'].items()), output), file=sys.stderr) + return 0 + + +def run_capture(args): + try: + config, source = source_context(args.config) + result = capture(config, source, args.workflow, args.job, + [('check', path) for path in args.check_log] + + [('setup', path) for path in args.setup_log]) + Path(args.output or 'evidence.json').write_text(json.dumps(result, sort_keys=True) + '\n', + encoding='utf-8') + return 0 + except (Refusal,) + MALFORMED as error: + refusal = refusal_of(error) + print('release-evidence: capture refused: %s%s' % ( + refusal['code'], ': ' + refusal['detail'] if refusal['detail'] else ''), file=sys.stderr) + return 1 + + +def main(argv=None): + parser = argparse.ArgumentParser(prog='release_evidence.py', description=__doc__.splitlines()[0]) + parser.add_argument('mode', choices=['capture', 'verify']) + parser.add_argument('--config', default='.github/ci-evidence.json') + parser.add_argument('--workflow', help='capture: the workflow key of the configuration') + parser.add_argument('--job', help='capture: the job key (one per matrix leg)') + parser.add_argument('--check-log', action='append', default=[]) + parser.add_argument('--setup-log', action='append', default=[]) + parser.add_argument('--output', help='capture: evidence.json; verify: release-evidence.json') + args = parser.parse_args(argv) + return run_capture(args) if args.mode == 'capture' else run_verify(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/test_ci_evidence.py b/tools/test_ci_evidence.py new file mode 100644 index 0000000..2ed2d45 --- /dev/null +++ b/tools/test_ci_evidence.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""The release evidence gate's configuration agrees with the workflows and the manifest. + +Usage: python3 tools/test_ci_evidence.py + +.github/ci-evidence.json names the CI jobs and matrix legs a release takes its evidence from, the +checks each of them runs, the files whose identity the evidence carries and the toolchain each ran +with. The gate refuses a release whose evidence disagrees with that (docs/releasing.md, "CI at the +tag"); these tests fail the pull request that makes the two drift apart instead. They also hold +scripts/dev/release_evidence.py to the development kit's lib/release_evidence.py of the release its +line 2 names. + +The workflows are read as text, in the shapes this repository writes them. This is not a YAML +reader: a workflow rewritten in another shape fails here until the reading follows it. +""" +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys +import unittest + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parent.parent +CONFIG = '.github/ci-evidence.json' +GATE = 'scripts/dev/release_evidence.py' +RELEASE = '.github/workflows/release.yml' + +# The SHA-256 of the kit's lib/release_evidence.py at each release this repository has vendored it +# from. A re-vendor row adds the release it moves to. +KIT_GATE = { + 'v0.10.0': 'd44af3df07f64036013ba9967a54cf799cec678068f0937395d6a2eef370e0e8', +} +RUNNER_OS = {'ubuntu': 'Linux', 'macos': 'macOS'} +GOOS = {'Linux': 'linux', 'macOS': 'darwin'} +GOARCH = {'X64': 'amd64', 'ARM64': 'arm64'} +ON_MAIN_PUSH = "github.ref == 'refs/heads/main' && github.event_name == 'push'" +CHECK_LOG = '| tee "${{ runner.temp }}/checks.log"' + + +def read(path): + return (ROOT / path).read_text(encoding='utf-8') + + +def config(): + return json.loads(read(CONFIG)) + + +def manifest_checks(): + return [row['name'] for row in json.loads(read('dev.json'))['checks']] + + +def jobs(text): + """Each job of a workflow: its key under jobs: and the lines of its block.""" + lines = text.split('\n') + result, key = {}, None + for line in lines[lines.index('jobs:') + 1:]: + found = re.fullmatch(r' ([A-Za-z0-9_-]+):', line) + if found: + key = found.group(1) + result[key] = [] + elif key is not None: + result[key].append(line) + return {key: '\n'.join(block) for key, block in result.items()} + + +def one(values, what): + if len(values) != 1: + raise AssertionError('expected one %s, found %r' % (what, values)) + return values[0] + + +def steps(block): + """The steps of a job, each as the text from its leading dash to the next step.""" + return ['- ' + part for part in re.split(r'^ - ', block, flags=re.M)[1:]] + + +def legs(key, block): + """The display name, labels and matrix value of each leg of a job.""" + name = one(re.findall(r'^ name: (.+)$', block, re.M), key + ' name') + runs_on = one(re.findall(r'^ runs-on: (.+)$', block, re.M), key + ' runs-on') + matrix = re.findall(r'^ os: \[(.*)\]$', block, re.M) + values = [value.strip() for value in one(matrix, key + ' matrix').split(',')] if matrix else [None] + result = [] + for value in values: + def expand(text, value=value): + return text if value is None else text.replace('${{ matrix.os }}', value) + result.append({'job_id': key, 'name': expand(name), 'labels': [expand(runs_on)], 'expand': expand}) + return result + + +def ci_legs(): + workflow = config()['workflows']['ci'] + return {leg['name']: leg for key, block in jobs(read(workflow['path'])).items() + for leg in legs(key, block)} + + +def push_selection(block): + """The checks a push to main runs in a job: its check.sh call that is not --changed.""" + calls = [call for call in re.findall(r'scripts/dev/check\.sh --all (.*)$', block, re.M) + if '--changed' not in call] + words = one(calls, 'check.sh --all call on a push').split() + everything = manifest_checks() + if words[0] == '--only': + named = words[1].split(',') + return [name for name in everything if name in named] + if words[0] == '--skip': + named = words[1].split(',') + return [name for name in everything if name not in named] + raise AssertionError('check.sh --all is followed by %r' % words[0]) + + +def go_versions(text): + return re.findall(r"^ *go-version: *['\"]?([0-9][0-9.x]*)['\"]? *$", text, re.M) + + +class Jobs(unittest.TestCase): + def test_every_job_and_leg_of_ci_is_required_by_its_display_name(self): + workflow = config()['workflows']['ci'] + self.assertTrue((ROOT / workflow['path']).is_file(), workflow['path']) + text = read(workflow['path']) + self.assertEqual(text.split('\n')[0], 'name: ' + workflow['name']) + # On a push to main nothing is skipped, and a skip would have to be named here on purpose. + self.assertNotIn('skipped_jobs', workflow) + found = ci_legs() + self.assertEqual(sorted(spec['name'] for spec in workflow['jobs'].values()), sorted(found)) + for job_key, spec in workflow['jobs'].items(): + with self.subTest(job=job_key): + leg = found[spec['name']] + self.assertEqual(spec['job_id'], leg['job_id']) + self.assertEqual(spec['labels'], leg['labels']) + + def test_only_a_push_to_the_trusted_branch_is_evidence(self): + settings = config() + workflow = settings['workflows']['ci'] + self.assertEqual(workflow['events'], ['push']) + self.assertIn('\n push:\n branches: [%s]\n' % settings['branch'], read(workflow['path'])) + + def test_each_leg_records_its_evidence_under_the_name_the_gate_reads(self): + workflow = config()['workflows']['ci'] + found = ci_legs() + blocks = jobs(read(workflow['path'])) + for job_key, spec in workflow['jobs'].items(): + with self.subTest(job=job_key): + leg = found[spec['name']] + listed = steps(blocks[leg['job_id']]) + captures = [index for index, step in enumerate(listed) + if 'scripts/dev/release_evidence.py capture' in step] + capture = listed[one(captures, 'capture step')] + self.assertIn('\n if: %s\n' % ON_MAIN_PUSH, capture) + self.assertIn('run: python3 scripts/dev/release_evidence.py capture --workflow ci ' + '--job %s --check-log "${{ runner.temp }}/checks.log" ' + '--output "${{ runner.temp }}/evidence.json"\n' + % job_key, leg['expand'](capture)) + upload = listed[captures[0] + 1] + self.assertIn('\n if: %s\n' % ON_MAIN_PUSH, upload) + self.assertRegex(upload, r'\n uses: actions/upload-artifact@[0-9a-f]{40} ') + self.assertIn('\n name: ci-evidence-ci-%s-${{ github.run_attempt }}\n' + ' path: ${{ runner.temp }}/evidence.json\n' + ' if-no-files-found: error\n' % job_key, leg['expand'](upload)) + # The checks' summary is what the capture reads, and a failed check fails its step. + checks = [step for step in listed[:captures[0]] if 'scripts/dev/check.sh --all' in step] + self.assertTrue(checks) + for step in checks: + self.assertRegex(step, r'run: \|\n +set -o pipefail\n') + for call in re.findall(r'scripts/dev/check\.sh --all .*$', step, re.M): + self.assertTrue(call.endswith(CHECK_LOG), call) + + +class Checks(unittest.TestCase): + def test_each_leg_lists_the_checks_its_push_run_reports(self): + workflow = config()['workflows']['ci'] + found = ci_legs() + blocks = jobs(read(workflow['path'])) + covered = set() + for job_key, spec in workflow['jobs'].items(): + with self.subTest(job=job_key): + self.assertNotIn('optional_skips', spec) + self.assertEqual(spec['checks'], push_selection(blocks[found[spec['name']]['job_id']])) + covered.update(spec['checks']) + self.assertEqual(sorted(covered), sorted(manifest_checks())) + + def test_the_pins_and_the_must_run_tests_are_in_every_runner_leg(self): + """A step script that fetches a pinned companion, or fails when a named test did not run and + pass, is an input the evidence carries, and its check is in every Runner leg.""" + settings = config() + manifest = json.loads(read('dev.json'))['checks'] + named = [] + for row in manifest: + script = re.fullmatch(r'(scripts/checks/[A-Za-z0-9_-]+\.sh)', row['run']) + if script and re.search(r'^fetch_pin |did not run and pass', read(script.group(1)), re.M): + named.append((row['name'], script.group(1))) + self.assertTrue(any(name == 'build-pinned' for name, _ in named)) + self.assertTrue(any(name == 'chain-tests' for name, _ in named)) + runner = [spec for spec in settings['workflows']['ci']['jobs'].values() if spec['job_id'] == 'check'] + self.assertEqual(len(runner), 2) + for name, script in named: + with self.subTest(check=name): + self.assertIn(script, settings['inputs']) + for spec in runner: + self.assertIn(name, spec['checks']) + + def test_every_input_is_a_tracked_file(self): + inputs = config()['inputs'] + self.assertTrue(inputs) + for path in inputs: + with self.subTest(path=path): + self.assertTrue((ROOT / path).is_file(), path) + tracked = subprocess.run(['git', 'ls-files', '--error-unmatch', '--', path], cwd=ROOT, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + self.assertEqual(tracked.returncode, 0, path + ' is not tracked') + + +class Toolchain(unittest.TestCase): + def test_every_leg_ran_the_release_toolchain(self): + settings = config() + release = sorted(set(go_versions(read(RELEASE)))) + self.assertEqual(len(release), 1, release) + version = release[0] + blocks = jobs(read(settings['workflows']['ci']['path'])) + for key, block in blocks.items(): + with self.subTest(job=key): + self.assertEqual(go_versions(block), [version]) + found = ci_legs() + for job_key, spec in settings['workflows']['ci']['jobs'].items(): + with self.subTest(leg=job_key): + profile = settings['profiles'][spec['profile']] + platform = profile['platform'] + label = one(found[spec['name']]['labels'], 'label') + self.assertEqual(RUNNER_OS[label.split('-')[0]], platform['RUNNER_OS']) + self.assertEqual(profile['pins']['go'], 'go version go%s %s/%s' % ( + version, GOOS[platform['RUNNER_OS']], GOARCH[platform['RUNNER_ARCH']])) + + def test_a_leg_compared_with_the_release_runner_runs_where_the_release_job_does(self): + settings = config() + verify = jobs(read(RELEASE))['verify'] + self.assertIn('\n runs-on: ubuntu-latest\n', verify) + for job_key, spec in settings['workflows']['ci']['jobs'].items(): + if settings['profiles'][spec['profile']]['compare'] == 'release-runner': + with self.subTest(leg=job_key): + self.assertEqual(spec['labels'], ['ubuntu-latest']) + + +class Release(unittest.TestCase): + def test_the_release_takes_the_evidence_and_does_not_call_ci_again(self): + text = read(RELEASE) + self.assertNotIn('.github/workflows/ci.yml', text) + found = jobs(text) + verify = found['verify'] + self.assertTrue(verify.startswith(' name: CI at the tag\n needs: admit\n'), verify[:80]) + self.assertIn('\n contents: read\n', verify) + self.assertIn('\n actions: read\n', verify) + listed = steps(verify) + self.assertEqual(len(listed), 4) + self.assertIn('ref: ${{ github.sha }}', listed[0]) + self.assertIn('uses: actions/setup-go@', listed[1]) + self.assertIn('run: python3 scripts/dev/release_evidence.py verify ' + '--output "${{ runner.temp }}/release-evidence.json"\n', listed[2]) + # The decision record is kept on a refusal too: the upload runs unless the run was cancelled. + self.assertIn('- name: Keep the decision record\n if: ${{ !cancelled() }}\n' + ' uses: actions/upload-artifact@', listed[3]) + self.assertIn('\n name: release-evidence-${{ github.run_attempt }}\n' + ' path: ${{ runner.temp }}/release-evidence.json\n' + ' if-no-files-found: error\n', listed[3] + '\n') + self.assertIn('\n - verify\n', found['package']) + + +class Vendored(unittest.TestCase): + def test_the_gate_is_the_kits_of_the_release_line_2_names(self): + lines = (ROOT / GATE).read_bytes().split(b'\n') + found = re.fullmatch(r'# from judgment-pack-devkit (v[0-9]+\.[0-9]+\.[0-9]+) \(lib/release_evidence\.py\)', + lines[1].decode('utf-8')) + self.assertIsNotNone(found, lines[1]) + release = found.group(1) + # Re-vendored with the other scripts, from one kit release. + runner = read('scripts/dev/check.sh').split('\n')[1] + self.assertEqual(runner, '# from judgment-pack-devkit %s (templates/check.sh)' % release) + self.assertIn(release, KIT_GATE, 'no digest of the kit release %s' % release) + self.assertEqual(hashlib.sha256(b'\n'.join(lines[:1] + lines[2:])).hexdigest(), KIT_GATE[release], + '%s is not the kit\'s lib/release_evidence.py of %s after line 2' % (GATE, release)) + + +if __name__ == '__main__': + unittest.main() From 14c30f45913c52b872ec43b388fbc2045ed388b7 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:21:56 -0400 Subject: [PATCH 2/5] docs/releasing.md: CI at the tag takes main's evidence, the fresh route (#175) What the gate holds this repository's release to, the inventory of the checks before and after, the measured durations of the last two releases' CI at the tag, the fresh route after a refusal, and the tag forms the gate takes. Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- docs/releasing.md | 118 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 10 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index aa5587d..ffe63e3 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -25,7 +25,7 @@ under `deploy/google-cloud/`. | | Linux | macOS | | --- | --- | --- | -| Formatting, vet, race tests and real-Runtime integration (CI, run again at the tag) | `amd64` | `arm64` | +| Formatting, vet, race tests and real-Runtime integration (main CI at the tagged commit, its evidence taken at the tag: [CI at the tag](#ci-at-the-tag)) | `amd64` | `arm64` | | The released archive, run: all three programs present; `jpack-runner` and `jpack-source-worker` name the release; `jpack-runner` refuses to start without a boot line | `amd64` and `arm64` | `arm64` | | The released archive, read and not run: it holds files and nothing else, each under its plain name and once; its documents are the commit's, byte for byte; each program is one program, built from the package of its name, for the archive's platform, at the lowest level of its architecture, the bytes the packer built, readable and executable by everyone; no file is there that a release does not hold; the archive reads to its end | `amd64` and `arm64` | `amd64` and `arm64` | @@ -92,7 +92,8 @@ commit from source. `THIRD_PARTY_NOTICES` includes the toolchain's own licence, so it is written and checked with the toolchain that links the release. A snapshot build names itself a snapshot: `jpack-runner version` prints `jpack-runner v-SNAPSHOT-`. -3. Merge the pull request. +3. Merge the pull request, and let main CI at the merge commit complete: its run is the + evidence the release takes, for 48 hours from its start ([CI at the tag](#ci-at-the-tag)). ## Tag @@ -108,10 +109,12 @@ If no signing key is configured, stop and settle the signing policy. Do not repl signed tag with an unsigned one. Signing is the maintainer's practice; the workflow does not check it. A tag is never moved or reused; a fix is a new version. -A tag is `vX.Y.Z` or `vX.Y.Z-`, as SemVer 2.0.0 writes them. Build metadata -(`+...`) is refused. A tag with a hyphen (`v0.1.0-rc.1`) is a prerelease: it is published -as one and is never marked latest. Make the first release a release candidate, since no -run of the workflow precedes it. +`Admit the tag` takes `vX.Y.Z` or `vX.Y.Z-`, as SemVer 2.0.0 writes them, and +refuses build metadata (`+...`). `CI at the tag` then takes `vX.Y.Z` only, as an annotated +tag (`git tag -s` makes one): the evidence gate refuses a prerelease tag (`tag-syntax`) and +a lightweight tag (`tag-annotated`). The workflow still publishes a tag with a hyphen +(`v0.1.0-rc.1`) as a prerelease, never marked latest, but no such tag passes the gate +until the kit's gate takes prereleases. ## What the workflow does @@ -121,9 +124,12 @@ name. 1. **Admits the tag.** It is a version as above, it names the commit the run was started for, that commit is on `main`, its notes exist, and the `production` environment has a reviewer rule. -2. **Runs CI at that commit.** The release calls the commit's own - [`ci.yml`](../.github/workflows/ci.yml), against the Runtime and Gateway revisions it - pins. A check `main` has gained since that commit is not run. +2. **Takes main's CI evidence at that commit, or refuses** (`CI at the tag`). Both `Runner` + jobs and `Release contents` of the push-to-main run of + [`ci.yml`](../.github/workflows/ci.yml) at the exact commit, every check passed, against + the Runtime, Gateway and old-Runner commits the commit pins, with the release toolchain. + `ci.yml` is not run again; a refusal stops the release + ([CI at the tag](#ci-at-the-tag)). 3. **Packages without publishing.** One toolchain, named exactly in the workflow; no cgo; no recorded paths; `amd64` at `v1` and `arm64` at `v8.0`. The tag is written into the programs. All four archives are then read by `tools/release_archives.py`, as the @@ -180,7 +186,8 @@ What to do with a draft depends on which job failed: | The job that failed | The draft | What to do | | --- | --- | --- | -| Any job before `Attest and draft release` | none was made | re-run the failed jobs | +| `CI at the tag` | none was made | read the refusal (the step's last line, or the run's `release-evidence-` artifact) and take the [fresh route](#the-fresh-route) once. A second refusal: fix the cause on `main` and release a new version | +| Any other job before `Attest and draft release` | none was made | re-run the failed jobs | | `Attest and draft release` | may exist, and may lack assets | read it, delete it by hand, then re-run the failed jobs. The job refuses to run while a release under the tag exists, draft or published, and refuses when it cannot find out | | `Publish release`, before publication | needs looking at: it may be as it was made, changed, incomplete or gone | read why the job failed, and look at the Releases page. **Keep** a draft that is as it was made, and re-run the failed jobs: that job only publishes the release id the run drafted. Do not publish a draft that was changed or is incomplete. Where the draft cannot be published, release a new version | | `Publish release`, at `Require the published release to hold what this run packaged` | the message says the published release does not hold what this run packaged, and prints both digest sets (published, holding files this run did not package, or not published after the call) | someone who can write to the repository changed the release in the window above; the digest sets shown are the evidence. Treat it as an incident: tell the maintainers, point no one at the release, and release a new version from a new tag. Do not re-run | @@ -189,6 +196,97 @@ What to do with a draft depends on which job failed: Delete a draft with `gh release delete --repo Judgment-Pack/judgment-pack-runner`, which leaves the tag. A published release is never deleted to make room for another. +## CI at the tag + +The release does not run CI again at a commit main has already passed. `CI at the tag` runs +the development kit's release evidence gate, vendored as +[`scripts/dev/release_evidence.py`](../scripts/dev/release_evidence.py) (the kit release on its +line 2), with this repository's configuration, +[`.github/ci-evidence.json`](../.github/ci-evidence.json). The gate's contract, its refusal +codes and its decision record are the kit's `docs/ci-evidence.md`; what follows is what it +holds this repository's release to. + +Main CI records the evidence. On a push to `main`, each `ci.yml` job, after its checks, writes +the commit, the git object ids of its inputs, its runner's image and tools, and each check's +result as read from the `check.sh` summary, and uploads it as the artifact +`ci-evidence-ci--`. Pull request runs record nothing. + +The release reuses that evidence only when all of this holds, and refuses otherwise: + +- **One run.** The `CI` workflow (`.github/workflows/ci.yml`, by its id), run by a push to + `main` in this repository at the tagged commit itself, completed and successful, its newest + attempt; no newer failed, cancelled or running attempt of a `main` run at that commit, and + none unfinished. Pull request runs, other branches' runs and runs from forks neither count + nor block. The attempt started at most 48 hours before the release reads it. +- **Every job.** `Runner (ubuntu-latest)`, `Runner (macos-latest)` and `Release contents`, + each `success` on a GitHub-hosted runner with its one label, from the selected attempt; no job + may be skipped and no other job may be there. +- **Every check, passed.** The Runner legs: `local-paths`, `agent-docs`, `diff-check`, + `go-fmt`, `go-vet`, `build-pinned`, `manual-gate`, `unit-tests`, `chain-tests`, `openapi`, + `build`, `ci-evidence`. `Release contents`: `release-toolchain`, `third-party-notices`, + `release-archives`, `release-programs`. Each `PASS`; no skip is permitted. +- **The real companions ran.** `build-pinned` builds the Runtime, earlier Runtime, old Runner + and Gateway at the commits `scripts/checks/build-pinned.sh` pins. `manual-gate`, + `chain-tests` and `unit-tests` fail when one of the tests they name did not run and pass, so a + `PASS` of each, on both Runner legs, is the proof that those tests ran against the pinned + builds and did not skip. The evidence carries the object ids of those four scripts, of + `go.mod` and `go.sum`, of `scripts/checks/release-toolchain.sh`, `scripts/dev/check.sh`, both + workflows, `dev.json`, the configuration and the gate, and the release derives the same ids + from its own checkout. +- **The release toolchain.** All three jobs set up Go 1.26.5, the toolchain `package` links the + archives with. The two Linux jobs' image, OS release, Go, Python, git and bash must equal what + this job's runner resolves with the same Go set up; the macOS job must have run Go 1.26.5 on + macOS ARM64 (its image version is recorded in the decision record and not compared). +- **The tag.** The tag is annotated, names the commit, and is the same tag object when the + gate starts and after it has read everything. + +The decision, `reuse` or `refuse` with its code, is the step's last line; the whole decision +record (runs, attempt, jobs, artifacts, inputs, toolchains, check results) is in the step's +log and kept as the run's `release-evidence-` artifact. Everything after +`CI at the tag` is checked fresh, as before: the tag against the commit, the notes, the +`production` reviewer rule, the notices again with the toolchain that links the release, +packaging, checksums, the contents of every archive, the native archive runs, the attestation, +the draft and the published assets. + +| | Main CI (push to `main`) | Release before (v0.24.0, v0.25.0) | Release now | +| --- | --- | --- | --- | +| The `Runner` checks, Linux and macOS, with the pinned companions | run | run again (`ci.yml` called), with Go 1.26.x | taken from main's evidence | +| `Release contents` | run | run again | taken from main's evidence | +| Go in the `Runner` jobs | 1.26.5 (it was 1.26.x, which resolved to 1.26.9 in run 40395171085) | 1.26.x | 1.26.5, compared | +| Tag, notes, reviewer rule, packaging, notices, archives, native runs, attestation, draft, publication | not run | run | run, unchanged | + +Measured (job start to completion, from the API): v0.24.0 (run 39287501642) ran `CI at the +tag` as `Release contents` 92 s, `Runner (ubuntu-latest)` 581 s and `Runner (macos-latest)` +620 s, 651 s from the end of `Admit the tag` to the start of `Package`; v0.25.0 (run +40371047612) 74 s, 589 s and 488 s, 594 s. The gate's own time is measured at the next +release. + +### The fresh route + +A refusal stops that release run; nothing runs CI again on its own. Read the refusal, then, +once: + +1. Re-run **all** jobs of the main CI run at the commit: + `gh run rerun --repo Judgment-Pack/judgment-pack-runner` (the run id is in the + decision record). A failed-jobs-only re-run leaves the other jobs' evidence in an earlier + attempt and is refused. This also works after `main` has moved on, and is the route for + `run-age` (evidence older than 48 hours) and `runtime-identity` (the hosted image changed + since main's run). +2. When that run has completed, re-run the release run's failed jobs: + `gh run rerun --failed --repo Judgment-Pack/judgment-pack-runner`. +3. If the gate refuses again, or the evidence cannot be made at that commit (a failure that is + the code's, inputs that stay incompatible), fix the cause on `main` and release a new + version. + +`run-unfinished` usually means the tag was pushed while main CI at the commit was still +running: wait for it to complete, then step 2. A prerelease or lightweight tag cannot pass; +see [Tag](#tag). + +The gate is first exercised live on the first push to `main` after it lands (the evidence +capture) and at the next release (the gate itself). The tests that ship with the kit run it +over handwritten API answers; `tools/test_ci_evidence.py` holds the configuration to `ci.yml`, +`release.yml` and `dev.json`, and the vendored gate to the kit's. + ## Verifying a download ```sh From eef5b29456af38486456a3ba34694c02951e5fb3 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:43:16 -0400 Subject: [PATCH 3/5] Fix round for the #182 review: macOS image family pinned, must-run scripts named (#175) - The macOS profile pins ImageOS macos26: a re-pointed macos-latest refuses at capture (runner-platform) and at the release (runtime-identity) instead of being taken as the same leg. - tools/test_ci_evidence.py names manual-gate and unit-tests beside build-pinned and chain-tests, so a script that loses its must-run assertions fails it, and holds every pins-compared profile to an ImageOS. - The Release contents scripts and .goreleaser.yml are inputs too, and jpdev.json maps them to ci-evidence. - docs/releasing.md names the Linux-only must-run tests, says how to move the Go and macOS pins, and gives the rollout case of runtime-identity its own answer in the fresh route. Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .github/ci-evidence.json | 19 ++++++++++++++----- docs/releasing.md | 31 ++++++++++++++++++++++++++++--- jpdev.json | 6 +++--- tools/test_ci_evidence.py | 8 ++++++-- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/.github/ci-evidence.json b/.github/ci-evidence.json index 9617b5d..f4e1f41 100644 --- a/.github/ci-evidence.json +++ b/.github/ci-evidence.json @@ -4,7 +4,8 @@ "(docs/releasing.md, 'CI at the tag'). tools/test_ci_evidence.py holds it to ci.yml, release.yml and dev.json.", "inputs: the module pins; the script that pins the Runtime, earlier Runtime, old Runner and Gateway commits", "the integration tests run against; the scripts that fail when a named real-companion test does not run and", - "pass; the toolchain check and the workflow naming the release toolchain; the runner whose summary is read." + "pass; the toolchain check and the workflow naming the release toolchain; the runner whose summary is read;", + "the Release contents scripts and the packer's configuration." ], "schema": 2, "repository": "Judgment-Pack/judgment-pack-runner", @@ -19,7 +20,12 @@ "scripts/checks/unit-tests.sh", "scripts/checks/release-toolchain.sh", "scripts/dev/check.sh", - ".github/workflows/release.yml" + ".github/workflows/release.yml", + "scripts/checks/release-programs.sh", + "tools/third_party_notices.py", + "tools/release_archives.py", + "tools/test_release_archives.py", + ".goreleaser.yml" ], "fresh_at_release": [ "tag syntax, tag names the commit, commit on main", @@ -54,12 +60,15 @@ }, "macos": { "$comment": [ - "The macOS leg, which the release job cannot resolve on its own runner: the platform and the exact Go", - "toolchain are pinned; the image version is recorded and not compared." + "The macOS leg, which the release job cannot resolve on its own runner: the platform, the image family", + "(ImageOS) and the exact Go toolchain are pinned; the image version within the family is recorded and not", + "compared. When macos-latest moves to another family, capture on main refuses (runner-platform) until", + "ImageOS moves here (docs/releasing.md, 'Moving a pin')." ], "platform": { "RUNNER_OS": "macOS", - "RUNNER_ARCH": "ARM64" + "RUNNER_ARCH": "ARM64", + "ImageOS": "macos26" }, "compare": "pins", "tools": { diff --git a/docs/releasing.md b/docs/releasing.md index ffe63e3..0fcc0ce 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -229,14 +229,22 @@ The release reuses that evidence only when all of this holds, and refuses otherw and Gateway at the commits `scripts/checks/build-pinned.sh` pins. `manual-gate`, `chain-tests` and `unit-tests` fail when one of the tests they name did not run and pass, so a `PASS` of each, on both Runner legs, is the proof that those tests ran against the pinned - builds and did not skip. The evidence carries the object ids of those four scripts, of - `go.mod` and `go.sum`, of `scripts/checks/release-toolchain.sh`, `scripts/dev/check.sh`, both + builds and did not skip. Two of them, with two subtests, are required on Linux only, because + they observe the Runtime through Linux `/proc`, and may skip on macOS by design: + `TestManualGovernedActualRuntimeHasNoAuthorityFD` (`manual-gate`) and + `TestExecutorOwnedRealRuntimeSurvivesOwnerDeath` with `/close` and `/kill` (`chain-tests`). + The evidence carries the object ids of those four scripts, of `go.mod` and `go.sum`, of + `scripts/checks/release-toolchain.sh`, `scripts/dev/check.sh`, the `Release contents` scripts + (`scripts/checks/release-programs.sh`, `tools/third_party_notices.py`, + `tools/release_archives.py`, `tools/test_release_archives.py`) and `.goreleaser.yml`, both workflows, `dev.json`, the configuration and the gate, and the release derives the same ids from its own checkout. - **The release toolchain.** All three jobs set up Go 1.26.5, the toolchain `package` links the archives with. The two Linux jobs' image, OS release, Go, Python, git and bash must equal what this job's runner resolves with the same Go set up; the macOS job must have run Go 1.26.5 on - macOS ARM64 (its image version is recorded in the decision record and not compared). + macOS ARM64 with the image family `macos26` (`ImageOS`). A newer image of that family is + accepted for the 48 hours the evidence lasts: its version is recorded in the decision record + and not compared. - **The tag.** The tag is annotated, names the commit, and is the same tag object when the gate starts and after it has read everything. @@ -276,6 +284,9 @@ once: `gh run rerun --failed --repo Judgment-Pack/judgment-pack-runner`. 3. If the gate refuses again, or the evidence cannot be made at that commit (a failure that is the code's, inputs that stay incompatible), fix the cause on `main` and release a new + version. One exception: `runtime-identity` naming the Linux image version while GitHub is + rolling a new hosted image out (main's jobs and this job landed on different versions) is + not the code's. Wait for the rollout to finish and take the route again; it needs no new version. `run-unfinished` usually means the tag was pushed while main CI at the commit was still @@ -287,6 +298,20 @@ capture) and at the next release (the gate itself). The tests that ship with the over handwritten API answers; `tools/test_ci_evidence.py` holds the configuration to `ci.yml`, `release.yml` and `dev.json`, and the vendored gate to the kit's. +### Moving a pin + +- **The Go toolchain.** Every `go-version` in `ci.yml` (`Runner` and `Release contents`) and + `release.yml` (`Package` and `CI at the tag`), and the two `go` pins in + `.github/ci-evidence.json`, move together in one pull request; `tools/test_ci_evidence.py` + and `release-toolchain` fail until they name one version. CI stays on that exact patch, not + on the newest patch the hosted runners hold. +- **The macOS image family.** When GitHub points `macos-latest` at another macOS, the first + push to `main` after it fails at `Runner (macos-latest)`'s capture step with + `runner-platform` ("ImageOS is , not macos26"), and a release refuses with + `runtime-identity`. Pull request CI stays green: it records no evidence. Move + `profiles.macos.platform.ImageOS` to the value the refusal names, in one pull request; its + run on `main` records the new evidence. + ## Verifying a download ```sh diff --git a/jpdev.json b/jpdev.json index 3878962..d4c4cf1 100644 --- a/jpdev.json +++ b/jpdev.json @@ -31,15 +31,15 @@ }, { "paths": ["THIRD_PARTY_NOTICES", "tools/third_party_notices.py"], - "checks": ["local-paths", "agent-docs", "diff-check", "third-party-notices"] + "checks": ["local-paths", "agent-docs", "diff-check", "third-party-notices", "ci-evidence"] }, { "paths": ["tools/release_archives.py", "tools/test_release_archives.py"], - "checks": ["local-paths", "agent-docs", "diff-check", "release-archives"] + "checks": ["local-paths", "agent-docs", "diff-check", "release-archives", "ci-evidence"] }, { "paths": [".goreleaser.yml", "scripts/checks/release-programs.sh"], - "checks": ["local-paths", "agent-docs", "diff-check", "release-programs"] + "checks": ["local-paths", "agent-docs", "diff-check", "release-programs", "ci-evidence"] }, { "paths": ["scripts/checks/release-toolchain.sh"], diff --git a/tools/test_ci_evidence.py b/tools/test_ci_evidence.py index 2ed2d45..0699886 100644 --- a/tools/test_ci_evidence.py +++ b/tools/test_ci_evidence.py @@ -191,8 +191,9 @@ def test_the_pins_and_the_must_run_tests_are_in_every_runner_leg(self): script = re.fullmatch(r'(scripts/checks/[A-Za-z0-9_-]+\.sh)', row['run']) if script and re.search(r'^fetch_pin |did not run and pass', read(script.group(1)), re.M): named.append((row['name'], script.group(1))) - self.assertTrue(any(name == 'build-pinned' for name, _ in named)) - self.assertTrue(any(name == 'chain-tests' for name, _ in named)) + # Named too: a script that lost its must-run assertions would drop out of the derivation. + self.assertTrue({'build-pinned', 'manual-gate', 'unit-tests', 'chain-tests'} <= {name for name, _ in named}, + named) runner = [spec for spec in settings['workflows']['ci']['jobs'].values() if spec['job_id'] == 'check'] self.assertEqual(len(runner), 2) for name, script in named: @@ -232,6 +233,9 @@ def test_every_leg_ran_the_release_toolchain(self): self.assertEqual(RUNNER_OS[label.split('-')[0]], platform['RUNNER_OS']) self.assertEqual(profile['pins']['go'], 'go version go%s %s/%s' % ( version, GOOS[platform['RUNNER_OS']], GOARCH[platform['RUNNER_ARCH']])) + # A leg the release job cannot compare with its own runner is held to its image family. + if profile['compare'] == 'pins': + self.assertRegex(platform.get('ImageOS', ''), r'^[a-z]+[0-9]+$') def test_a_leg_compared_with_the_release_runner_runs_where_the_release_job_does(self): settings = config() From fcd15ed06b740d9bedc03615f2c13208f6764e41 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:06:02 -0400 Subject: [PATCH 4/5] Re-vendor the release evidence gate at devkit v0.11.0 scripts/dev/release_evidence.py from judgment-pack-devkit v0.11.0 (the tag grammar with SemVer prereleases, dispatch validation runs); tag.prerelease is enabled because this repository publishes prerelease tags. Co-Authored-By: Claude Fable 5.1 Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> --- .github/ci-evidence.json | 81 ++++++++++++++++++++++---- scripts/dev/release_evidence.py | 100 +++++++++++++++++++++++++++----- 2 files changed, 153 insertions(+), 28 deletions(-) diff --git a/.github/ci-evidence.json b/.github/ci-evidence.json index f4e1f41..be30755 100644 --- a/.github/ci-evidence.json +++ b/.github/ci-evidence.json @@ -49,10 +49,22 @@ }, "compare": "release-runner", "tools": { - "go": ["go", "version"], - "python": ["python3", "--version"], - "git": ["git", "--version"], - "bash": ["bash", "--version"] + "go": [ + "go", + "version" + ], + "python": [ + "python3", + "--version" + ], + "git": [ + "git", + "--version" + ], + "bash": [ + "bash", + "--version" + ] }, "pins": { "go": "go version go1.26.5 linux/amd64" @@ -72,7 +84,10 @@ }, "compare": "pins", "tools": { - "go": ["go", "version"] + "go": [ + "go", + "version" + ] }, "pins": { "go": "go version go1.26.5 darwin/arm64" @@ -87,30 +102,72 @@ ], "path": ".github/workflows/ci.yml", "name": "CI", - "events": ["push"], + "events": [ + "push" + ], "jobs": { "runner-ubuntu-latest": { "name": "Runner (ubuntu-latest)", "job_id": "check", - "labels": ["ubuntu-latest"], + "labels": [ + "ubuntu-latest" + ], "profile": "linux", - "checks": ["local-paths", "agent-docs", "diff-check", "go-fmt", "go-vet", "build-pinned", "manual-gate", "unit-tests", "chain-tests", "openapi", "build", "ci-evidence"] + "checks": [ + "local-paths", + "agent-docs", + "diff-check", + "go-fmt", + "go-vet", + "build-pinned", + "manual-gate", + "unit-tests", + "chain-tests", + "openapi", + "build", + "ci-evidence" + ] }, "runner-macos-latest": { "name": "Runner (macos-latest)", "job_id": "check", - "labels": ["macos-latest"], + "labels": [ + "macos-latest" + ], "profile": "macos", - "checks": ["local-paths", "agent-docs", "diff-check", "go-fmt", "go-vet", "build-pinned", "manual-gate", "unit-tests", "chain-tests", "openapi", "build", "ci-evidence"] + "checks": [ + "local-paths", + "agent-docs", + "diff-check", + "go-fmt", + "go-vet", + "build-pinned", + "manual-gate", + "unit-tests", + "chain-tests", + "openapi", + "build", + "ci-evidence" + ] }, "release-contents": { "name": "Release contents", "job_id": "release-contents", - "labels": ["ubuntu-latest"], + "labels": [ + "ubuntu-latest" + ], "profile": "linux", - "checks": ["release-toolchain", "third-party-notices", "release-archives", "release-programs"] + "checks": [ + "release-toolchain", + "third-party-notices", + "release-archives", + "release-programs" + ] } } } + }, + "tag": { + "prerelease": true } } diff --git a/scripts/dev/release_evidence.py b/scripts/dev/release_evidence.py index 6648d0c..2c07091 100644 --- a/scripts/dev/release_evidence.py +++ b/scripts/dev/release_evidence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# from judgment-pack-devkit v0.10.0 (lib/release_evidence.py) +# from judgment-pack-devkit v0.11.0 (lib/release_evidence.py) """release_evidence.py capture|verify: reuse the CI evidence main produced at the release commit. capture Runs in a CI job on the trusted branch, after the job's checks. Writes evidence.json for @@ -49,6 +49,15 @@ PATH = re.compile(r'[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*') WORKFLOW_PATH = re.compile(r'\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml') TAG = re.compile(r'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)') +# The same, with an optional SemVer prerelease: "-" and dot-separated identifiers of ASCII letters, +# digits and hyphens, none empty, a numeric one without leading zeroes. Build metadata never. +PRERELEASE_IDENTIFIER = r'(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)' +TAG_PRERELEASE = re.compile(TAG.pattern + r'(-%s(\.%s)*)?' % (PRERELEASE_IDENTIFIER, PRERELEASE_IDENTIFIER)) +# What a workflow_dispatch run of the release workflow validates: nothing (it does not use the +# gate), or the trusted branch's tip, as a validation-only decision. +DISPATCH = ('none', 'branch-tip') +# The decision each mode reaches when every check passes: only a publication reuses. +PASSED = {'publication': 'reuse', 'validation-only': 'validated'} SHA = re.compile(r'[0-9a-f]{40}') OBJECT_ID = re.compile(r'[0-9a-f]{40}|[0-9a-f]{64}') @@ -178,7 +187,7 @@ def path(value, where, pattern=PATH): fail(where, 'a path without . or .. segments is required') fields(config, 'config', ['schema', 'repository', 'branch', 'max_age_hours', 'inputs', - 'profiles', 'workflows'], ['fresh_at_release']) + 'profiles', 'workflows'], ['fresh_at_release', 'tag', 'dispatch']) if config['schema'] != SCHEMA or type(config['schema']) is not int: fail('config.schema', 'must be %d' % SCHEMA) text(config['repository'], 'config.repository', REPOSITORY) @@ -190,6 +199,14 @@ def path(value, where, pattern=PATH): for index, item in enumerate(config['inputs']): path(item, 'config.inputs[%d]' % index) texts(config.get('fresh_at_release', []), 'config.fresh_at_release') + tag = config.get('tag', {'prerelease': False}) + fields(tag, 'config.tag', ['prerelease']) + if type(tag['prerelease']) is not bool: + fail('config.tag.prerelease', 'true or false is required') + dispatch = config.get('dispatch', {'validates': 'none'}) + fields(dispatch, 'config.dispatch', ['validates']) + if dispatch['validates'] not in DISPATCH: + fail('config.dispatch.validates', 'one of %s is required' % (DISPATCH,)) named(config['profiles'], 'config.profiles') for name, profile in config['profiles'].items(): @@ -246,6 +263,25 @@ def path(value, where, pattern=PATH): fail(where, 'a job name is repeated') +def tag_grammar(config): + """vMAJOR.MINOR.PATCH, and a SemVer prerelease only where the configuration allows one.""" + return TAG_PRERELEASE if config.get('tag', {}).get('prerelease') is True else TAG + + +def release_mode(config): + """What this run decides. A workflow_dispatch run is a validation of the trusted branch's tip, + never a publication, where dispatch.validates is "branch-tip", and is refused otherwise; every + other run is a publication, held to its tag.""" + if os.environ.get('GITHUB_EVENT_NAME') != 'workflow_dispatch': + return 'publication' + need(config.get('dispatch', {}).get('validates') == 'branch-tip', 'dispatch-not-gated', + 'a workflow_dispatch run does not use the gate (dispatch.validates is "none")') + need(os.environ.get('GITHUB_REF') == 'refs/heads/' + config['branch'], 'dispatch-ref', + 'a validation run is dispatched on %s, not %s' % ('refs/heads/' + config['branch'], + os.environ.get('GITHUB_REF'))) + return 'validation-only' + + def helper_path(root): try: return Path(__file__).resolve().relative_to(root.resolve()).as_posix() @@ -429,9 +465,9 @@ def selection(run): return (run['id'], run['run_attempt'], run['status'], run['conclusion']) -def tag_object(api, name, sha): +def tag_object(api, name, sha, grammar=TAG): """The annotated tag's object id, after checking that the tag peels to the release commit.""" - need(isinstance(name, str) and TAG.fullmatch(name), 'tag-syntax') + need(isinstance(name, str) and grammar.fullmatch(name), 'tag-syntax') ref = api.get('git/ref/tags/' + quote(name, safe='')) need(ref['ref'] == 'refs/tags/' + name and ref['object']['type'] == 'tag', 'tag-annotated') original = ref['object']['sha'] @@ -623,14 +659,18 @@ def verify_workflow(api, config, source, key, now, here, trail): def verify(api, config, source, release_tag, now, record): - branch = config['branch'] + """A publication is held to its tag; a validation-only run reads no tag: its commit is the + trusted branch's tip the dispatch ran for, and its decision is never a publication's.""" + branch, publication = config['branch'], record['mode'] == 'publication' + grammar = tag_grammar(config) repository = api.get('') need((repository['full_name'], repository['id']) == (source['repository'], source['repository_id']), 'repository-identity', '%s (%s)' % (repository.get('full_name'), repository.get('id'))) need(repository['default_branch'] == branch, 'branch-default', 'the default branch is %s, the configuration trusts %s' % (repository['default_branch'], branch)) - first_tag = tag_object(api, release_tag, source['source']) - record['tag_object'] = first_tag + if publication: + first_tag = tag_object(api, release_tag, source['source'], grammar) + record['tag_object'] = first_tag here, chosen = {}, {} for key in config['workflows']: record['workflows'][key] = {} @@ -639,7 +679,8 @@ def verify(api, config, source, release_tag, now, record): for key, (workflow_id, run) in chosen.items(): final = select(api, source, config['workflows'][key], workflow_id, branch) need(selection(final) == selection(run), 'final-latest-changed', key) - need(tag_object(api, release_tag, source['source']) == first_tag, 'tag-changed') + if publication: + need(tag_object(api, release_tag, source['source'], grammar) == first_tag, 'tag-changed') # Malformed or missing API data or files: a refusal like any other, never a success. @@ -657,40 +698,67 @@ def refusal_of(error): def decide(api, config, source, release_tag, now, record): - """The decision is reuse only when every check of verify passed.""" + """The decision is reuse only for a publication whose every check of verify passed; a + validation-only run that passes is validated, which no publication takes.""" try: + need(record.get('mode') in PASSED, 'mode-unknown', str(record.get('mode'))) + passed = PASSED[record['mode']] verify(api, config, source, release_tag, now, record) except (Refusal,) + MALFORMED as error: record['decision'], record['refusal'] = 'refuse', refusal_of(error) else: - record['decision'], record['refusal'] = 'reuse', None + record['decision'], record['refusal'] = passed, None return record +def write_outputs(record): + """The decision and mode as step outputs, for the jobs after the gate: a publishing job + proceeds on decision "reuse" alone. Both values are from fixed sets.""" + name = os.environ.get('GITHUB_OUTPUT') + if name: + with open(name, 'a', encoding='utf-8') as output: + output.write('decision=%s\nmode=%s\n' % (record['decision'], record['mode'] or '')) + + def run_verify(args): now = dt.datetime.now(dt.timezone.utc) output = Path(args.output or 'release-evidence.json') - record = {'schema': SCHEMA, 'decision': 'refuse', 'refusal': None, + record = {'schema': SCHEMA, 'decision': 'refuse', 'refusal': None, 'mode': None, + 'event': os.environ.get('GITHUB_EVENT_NAME', ''), 'tag': os.environ.get('RELEASE_TAG', ''), 'checked_at': now.isoformat(), 'workflows': {}} try: config, source = source_context(args.config) - except (Refusal,) + MALFORMED as error: - record['refusal'] = refusal_of(error) - else: record.update(repository=source['repository'], repository_id=source['repository_id'], source=source['source'], inputs=source['inputs'], branch=config['branch'], max_age_hours=config['max_age_hours'], - fresh_at_release=config.get('fresh_at_release', [])) + fresh_at_release=config.get('fresh_at_release', []), + tag_prerelease=tag_grammar(config) is TAG_PRERELEASE, + dispatch=config.get('dispatch', {}).get('validates', 'none')) + record['mode'] = release_mode(config) + except (Refusal,) + MALFORMED as error: + record['refusal'] = refusal_of(error) + else: + if record['mode'] != 'publication': + # No tag is read or stood for: the commit is the tip the dispatch ran for. + record['tag'] = None decide(API(source['repository']), config, source, record['tag'], now, record) text = json.dumps(record, indent=2, sort_keys=True) print(text) try: output.write_text(text + '\n', encoding='utf-8') + write_outputs(record) except OSError: - print('release-evidence: refuse: the decision record could not be written to %s' % output, + print('release-evidence: refuse: the decision record or the step outputs could not be written', file=sys.stderr) return 1 + if record['decision'] == 'validated': + print('release-evidence: validated, validation-only and never a publication decision: %s at %s ' + '(decision record: %s)' % (', '.join( + '%s run %s attempt %s' % (key, value['run_id'], value['attempt']) + for key, value in record['workflows'].items()), record['source'], output), + file=sys.stderr) + return 0 if record['decision'] != 'reuse': refusal = record['refusal'] print('release-evidence: refuse: %s%s (decision record: %s)' % ( From 4b901b371ce80e6bd5700bede069331ce05e6c26 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:10:07 -0400 Subject: [PATCH 5/5] Re-vendor the remaining devkit scripts at v0.11.0 and pin the kit release in the copy test check.sh, handover_check.sh and changed_checks.py from judgment-pack-devkit v0.11.0 (line 2 only); the vendored-copy test records the v0.11.0 gate file. Co-Authored-By: Claude Fable 5.1 Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> --- docs/releasing.md | 11 +++++------ scripts/dev/changed_checks.py | 2 +- scripts/dev/check.sh | 2 +- scripts/dev/handover_check.sh | 2 +- tools/test_ci_evidence.py | 1 + 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index 0fcc0ce..6bca729 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -110,11 +110,10 @@ signed tag with an unsigned one. Signing is the maintainer's practice; the workf not check it. A tag is never moved or reused; a fix is a new version. `Admit the tag` takes `vX.Y.Z` or `vX.Y.Z-`, as SemVer 2.0.0 writes them, and -refuses build metadata (`+...`). `CI at the tag` then takes `vX.Y.Z` only, as an annotated -tag (`git tag -s` makes one): the evidence gate refuses a prerelease tag (`tag-syntax`) and -a lightweight tag (`tag-annotated`). The workflow still publishes a tag with a hyphen -(`v0.1.0-rc.1`) as a prerelease, never marked latest, but no such tag passes the gate -until the kit's gate takes prereleases. +requires an annotated tag (`git tag -s` makes one). The evidence gate accepts the same grammar: +`.github/ci-evidence.json` sets `tag.prerelease: true` (kit v0.11.0), so a prerelease such as +`v0.1.0-rc.1` passes the gate like a release and is published as a prerelease, never marked +latest. A lightweight tag is refused (`tag-annotated`). ## What the workflow does @@ -290,7 +289,7 @@ once: version. `run-unfinished` usually means the tag was pushed while main CI at the commit was still -running: wait for it to complete, then step 2. A prerelease or lightweight tag cannot pass; +running: wait for it to complete, then step 2. A lightweight tag cannot pass; see [Tag](#tag). The gate is first exercised live on the first push to `main` after it lands (the evidence diff --git a/scripts/dev/changed_checks.py b/scripts/dev/changed_checks.py index b4cbde1..6c1d5f3 100644 --- a/scripts/dev/changed_checks.py +++ b/scripts/dev/changed_checks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# from judgment-pack-devkit v0.10.0 (lib/changed_checks.py) +# from judgment-pack-devkit v0.11.0 (lib/changed_checks.py) """changed_checks.py --root ROOT --manifest MANIFEST [--base BASE] [--label TEXT] Selects the check-list steps that the changed paths of a clone can affect, for diff --git a/scripts/dev/check.sh b/scripts/dev/check.sh index 649e0b5..4a1aacc 100755 --- a/scripts/dev/check.sh +++ b/scripts/dev/check.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# from judgment-pack-devkit v0.10.0 (templates/check.sh) +# from judgment-pack-devkit v0.11.0 (templates/check.sh) # scripts/dev/check.sh [--setup] [--only a,b | --changed [BASE]] [--skip a,b] [--all] [--manifest FILE] # Runs the checks of dev.json in order and prints one summary block; exit status = failures. # --changed runs only the checks jpdev.json maps the paths changed since the merge base of BASE diff --git a/scripts/dev/handover_check.sh b/scripts/dev/handover_check.sh index 31ff467..377ebc6 100755 --- a/scripts/dev/handover_check.sh +++ b/scripts/dev/handover_check.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# from judgment-pack-devkit v0.10.0 (lib/handover_check.sh) +# from judgment-pack-devkit v0.11.0 (lib/handover_check.sh) set -eu python3 - <<'PY' diff --git a/tools/test_ci_evidence.py b/tools/test_ci_evidence.py index 0699886..a43f540 100644 --- a/tools/test_ci_evidence.py +++ b/tools/test_ci_evidence.py @@ -31,6 +31,7 @@ # from. A re-vendor row adds the release it moves to. KIT_GATE = { 'v0.10.0': 'd44af3df07f64036013ba9967a54cf799cec678068f0937395d6a2eef370e0e8', + 'v0.11.0': 'f272b7a43272bac31bb9b0c37a583d9c40d376972b04c812e576bd82c1ef18e3', } RUNNER_OS = {'ubuntu': 'Linux', 'macos': 'macOS'} GOOS = {'Linux': 'linux', 'macOS': 'darwin'}