integration: sync from a PR list, and the CI build that refuses a branch that does not verify

.github/workflows/build-integration.yml

@@ -0,0 +1,136 @@
+# Builds an integration branch ("bag of PRs", docs/INTEGRATION-BRANCHES.md) into an image on GHCR - but only a branch that
+# VERIFIES: its history is the base plus merges of pull-request heads and nothing else (scripts/integration.py verify).
+#
+#   Actions -> "Build integration image" -> Run workflow
+#     repo        owner/name holding the branch            e.g. sirosfoundation/vc
+#     branch      the integration branch                    e.g. integration/gdc-nymauth-v5
+#     component   image name                                e.g. vc-apigw
+#     pr_repos    repos searched for each merged head's PR  e.g. SUNET/vc,sirosfoundation/vc
+#
+# The image is ghcr.io/<owner>/integration/<component>:<branch-slug>-<sha7>, labelled with the branch tip, the PR list and the
+# sha256 of the report; environments pin it by DIGEST (the run summary prints component@sha256:...). The report is also the
+# `integration-report` artifact. Images here contain UNMERGED code: never `latest`, pin by digest, re-pin to a release when
+# the PRs land.
+#
+# NOT YET EXERCISED: this workflow has never run. Known gaps: a component that needs more than build args (the vc verifier's
+# circuit-artifact overlay, dockerfiles/verifier-nymauth.Dockerfile, needs a file that is not in any repo) is not covered; a
+# private repository needs the INTEGRATION_READ_TOKEN secret (a fine-grained token with contents:read).
+name: Build integration image
+
+on:
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: "owner/name of the repository holding the integration branch"
+        required: true
+      branch:
+        description: "the integration branch, e.g. integration/gdc-nymauth-v5"
+        required: true
+      component:
+        description: "image name, e.g. vc-apigw"
+        required: true
+      base:
+        description: "what the branch was built on (a branch of that repo)"
+        default: "main"
+      pr_repos:
+        description: "comma-separated owner/name list searched for each merged head's pull request"
+        required: true
+      dockerfile:
+        description: "Dockerfile path inside the repository"
+        default: "Dockerfile"
+      target:
+        description: "docker build --target (optional)"
+        default: ""
+      build_args:
+        description: "KEY=VALUE per line (e.g. SERVICE_NAME=apigw, VERSION=..., full 40-char ZK refs)"
+        default: ""
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Tools (this repository)
+        uses: actions/checkout@v4
+        with:
+          path: tools
+
+      - name: The integration branch (full history)
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo }}
+          ref: ${{ inputs.branch }}
+          fetch-depth: 0
+          path: src
+          token: ${{ secrets.INTEGRATION_READ_TOKEN || github.token }}
+
+      - name: Verify, and report what is in it
+        env:
+          GH_TOKEN: ${{ secrets.INTEGRATION_READ_TOKEN || github.token }}
+          BASE_BRANCH: ${{ inputs.base }}
+          BRANCH: ${{ inputs.branch }}
+          PR_REPOS: ${{ inputs.pr_repos }}
+        run: |
+          set -euo pipefail
+          python3 -m pip install --quiet pyyaml
+          BASE="origin/$BASE_BRANCH"
+          git -C src fetch --quiet origin "$BASE_BRANCH"
+          # A branch with a patch commit, a change hidden in a merge, a hand-resolved conflict or a head that is no PR fails here.
+          python3 tools/scripts/integration.py verify --path src --tip HEAD --base "$BASE" --repos "$PR_REPOS"
+          python3 tools/scripts/integration.py report --path src --tip HEAD --base "$BASE" --repos "$PR_REPOS" --json > report.json
+          SLUG=$(echo "$BRANCH" | tr '/' '-' | tr -c 'A-Za-z0-9._-\n' '-')
+          {
+            echo "REPORT_SHA=$(sha256sum report.json | cut -d' ' -f1)"
+            echo "TIP=$(git -C src rev-parse HEAD)"
+            echo "TAG=${SLUG}-$(git -C src rev-parse --short=7 HEAD)"
+            echo "PRS=$(python3 tools/scripts/integration.py report --path src --tip HEAD --base "$BASE" --repos "$PR_REPOS" --summary)"
+          } >> "$GITHUB_ENV"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: integration-report
+          path: report.json
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: src
+          file: src/${{ inputs.dockerfile }}
+          target: ${{ inputs.target }}
+          build-args: ${{ inputs.build_args }}
+          push: true
+          provenance: true
+          tags: ghcr.io/${{ github.repository_owner }}/integration/${{ inputs.component }}:${{ env.TAG }}
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/integration/${{ inputs.component }}:buildcache
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/integration/${{ inputs.component }}:buildcache,mode=max
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ inputs.repo }}
+            org.opencontainers.image.revision=${{ env.TIP }}
+            org.siros.integration.branch=${{ inputs.branch }}
+            org.siros.integration.prs=${{ env.PRS }}
+            org.siros.integration.report-sha256=${{ env.REPORT_SHA }}
+
+      - name: Summary (pin this)
+        env:
+          IMAGE: ghcr.io/${{ github.repository_owner }}/integration/${{ inputs.component }}
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          {
+            echo "### ${IMAGE}@${DIGEST}"
+            echo "- tag: \`${TAG}\`, branch tip \`${TIP}\`"
+            echo "- PRs: \`${PRS}\`"
+            echo "- report sha256: \`${REPORT_SHA}\`"
+          } >> "$GITHUB_STEP_SUMMARY"

docs/INTEGRATION-BRANCHES.md

@@ -24,7 +24,34 @@ inside the merge); and, with `--repos`, a merged head that is not the head of an
 regenerated from its PR list, never patched. `report` also shows when a PR's head has moved since it was merged (the branch is
 stale) and whether the PR has since been merged upstream (re-pin to a release).
 
-First run (2026-10-22): vc `integration/gdc-nymauth-v5` @ `c8c5b1ea` has 12 PRs but also 47 commits that are not merges of a
+## Creating and updating a branch: `sync`
+
+```sh
+# from a PR list (merge order as given; @sha pins a head and fails if the PR has moved)
+python3 scripts/integration.py sync --path <clone> --branch integration/<name> --base origin/main \
+    --pr SUNET/vc#840 --pr SUNET/vc#830@90f227fcd [--push origin]
+
+# the same PRs with their LATEST heads (an update); the old branch is only read
+python3 scripts/integration.py sync --path <clone> --branch integration/<name> --from integration/<old> --repos SUNET/vc,sirosfoundation/vc
+```
+Each PR is fetched from its own repository (`pull/<n>/head`) and merged with `--no-ff`, with the message
+`Merge PR owner/repo#N @<head9>: <title>`, so `report` can read the list back offline. A conflict stops the sync and leaves the tree
+clean: resolve it in a glue PR that goes into the list (before the PR it conflicts with), never by hand. `--push` force-pushes
+with a lease. A synced branch always passes `verify`.
+
+## Building it: `.github/workflows/build-integration.yml`
+
+`Actions -> Build integration image` (repo, branch, component, pr_repos, optional dockerfile/target/build args) checks the
+branch out with full history, runs `verify` (a branch that fails is not built), attaches the `report` as an artifact, builds with the
+component's own Dockerfile and a registry cache, and pushes `ghcr.io/<owner>/integration/<component>:<branch>-<sha7>` with labels
+(`org.opencontainers.image.revision`, `org.siros.integration.branch`, `org.siros.integration.prs`,
+`org.siros.integration.report-sha256`) and a provenance attestation. The run summary prints `component@sha256:...`: pin that
+digest in `environments/<env>.yaml`. **The workflow has never run** (it can only be dispatched from the default branch once
+merged) and does not cover builds that need more than build args, such as the vc verifier's circuit-artifact overlay.
+
+## First run (2026-10-22)
+
+vc `integration/gdc-nymauth-v5` @ `c8c5b1ea` has 12 PRs but also 47 commits that are not merges of a
 PR (the branch started on a feature line), and 51 problems in all; the engine branch `local/gdc-nymauth-integration-4`
 @ `1d0708b` has a conflict resolved by hand in its #163 merge and two local-only commits (`a25df83`, `1d0708b`).
 Neither can be built as a documented bundle until it is regenerated.

scripts/integration.py

@@ -18,10 +18,18 @@
   - a merge of a second parent that is not a PR head (checked with the API when `--repos` is given).
 A branch that fails is regenerated from its PR list, never patched.
 
+    integration.py sync --path <clone> --branch integration/<name> --base origin/main --pr owner/repo#N [--pr ...] [--push]
+    integration.py sync --path <clone> --branch integration/<name> --from integration/<old> --repos ...   # same PRs, latest heads
+
+`sync` (re)creates the branch from a PR list: base, then one `--no-ff` merge per PR head in the order given, each with the
+message `Merge PR owner/repo#N @<head7>: <title>` so `report` can read the list back offline. A conflict stops it: resolve it in
+a (draft, never-for-upstream) glue PR that goes in the list, never by hand in the merge.
+
 The first-parent path starts at the merge base of `--tip` and `--base`. A merge's second parent is the PR head that was merged.
 """
 import argparse
 import json
+import re
 import subprocess
 import sys
 from dataclasses import dataclass, field
@@ -104,12 +112,23 @@ def lookup_prs(repos, sha, run=subprocess.run):
             except json.JSONDecodeError:
                 continue
             if found:
-                return [{"repo": repo, "number": p["number"], "title": p["title"], "state": p["state"] if not p.get("merged_at")
+                # The repository a PR number belongs to is the PR's BASE repository: asking a fork returns the upstream's PRs
+                # for commits it shares, under the upstream's numbers.
+                return [{"repo": p.get("base", {}).get("repo", {}).get("full_name") or repo, "number": p["number"], "title": p["title"], "state": p["state"] if not p.get("merged_at")
                          else "merged", "draft": p.get("draft", False), "head": p["head"]["sha"],
                          "labels": [l["name"] for l in p.get("labels", [])]} for p in found]
     return []
 
 
+MESSAGE = re.compile(r"^Merge PR (?P<repo>[\w.-]+/[\w.-]+)#(?P<number>\d+) @(?P<sha>[0-9a-f]{7,40})(?:: (?P<title>.*))?$")
+
+
+def from_message(subject):
+    """The PR a `sync` merge message names (offline, no API), or None for any other message."""
+    m = MESSAGE.match(subject)
+    return m.groupdict() if m else None
+
+
 def report(analysis: Analysis, repos=(), lookup=lookup_prs) -> dict:
     """Ordered PRs (the latest merged head of each), plus everything that is not a PR merge."""
     heads = []                              # [(second parent, merge commit)] in merge order
@@ -118,7 +137,11 @@ def report(analysis: Analysis, repos=(), lookup=lookup_prs) -> dict:
             heads.append((m.parents[1], m))
     prs, unmapped, seen = [], [], {}
     for head, merge in heads:
+        named = from_message(merge.subject)
         found = lookup(repos, head) if repos else []
+        if not found and named:
+            found = [{"repo": named["repo"], "number": int(named["number"]), "title": named.get("title") or "", "state": "unknown",
+                      "draft": False, "head": head, "labels": []}]
         if not found:
             unmapped.append({"merged_head": head, "merge": merge.sha, "subject": merge.subject})
             continue
@@ -137,16 +160,79 @@ def report(analysis: Analysis, repos=(), lookup=lookup_prs) -> dict:
             "drift": [{"commit": s.sha, "subject": s.subject, "problem": p} for s, p in analysis.problems]}
 
 
+def parse_pr(spec):
+    """`owner/repo#N` or `owner/repo#N@<sha>` -> (repo, number, expected head or None)."""
+    m = re.fullmatch(r"([\w.-]+/[\w.-]+)#(\d+)(?:@([0-9a-f]{7,40}))?", spec)
+    if not m:
+        raise SystemExit(f"not a PR reference (owner/repo#N[@sha]): {spec}")
+    return m.group(1), int(m.group(2)), m.group(3)
+
+
+def ssh_url(repo):
+    return f"git@github.com:{repo}.git"
+
+
+def sync(path, branch, base_ref, prs, url_for=ssh_url, titles=None, push_to=None):
+    """(Re)create `branch` as base_ref plus one `--no-ff` merge per PR head, in order. Returns [(repo, number, head)]."""
+    path = Path(path)
+    if git(path, "status", "--porcelain").stdout.strip():
+        raise SystemExit("the working tree is not clean")
+    merged = []
+    git(path, "checkout", "-q", "-B", branch, base_ref)
+    for repo, number, expected in prs:
+        ref = f"refs/integration/{repo.replace('/', '_')}_pr{number}"
+        fetched = git(path, "fetch", "-q", url_for(repo), f"+pull/{number}/head:{ref}", check=False)
+        if fetched.returncode != 0:
+            raise SystemExit(f"cannot fetch {repo}#{number}: {fetched.stderr.strip()}")
+        head = git(path, "rev-parse", ref).stdout.strip()
+        if expected and not head.startswith(expected):
+            raise SystemExit(f"{repo}#{number} head is {head[:9]}, not the requested {expected}: the PR has moved")
+        title = (titles or {}).get((repo, number), "")
+        message = f"Merge PR {repo}#{number} @{head[:9]}" + (f": {title}" if title else "")
+        merge = git(path, "merge", "--no-ff", "-m", message, head, check=False)
+        if merge.returncode != 0:
+            git(path, "merge", "--abort", check=False)
+            raise SystemExit(f"{repo}#{number} @{head[:9]} conflicts with what is already on {branch}. Resolve it in a glue PR "
+                             f"added to the list (before this one), not by hand: {merge.stdout.strip()[:300]}")
+        merged.append((repo, number, head))
+    if push_to:
+        git(path, "push", "--force-with-lease", push_to, f"{branch}:{branch}")
+    return merged
+
+
 def main(argv=None):
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("command", choices=("report", "verify"))
+    ap.add_argument("command", choices=("report", "verify", "sync"))
     ap.add_argument("--path", required=True, help="a clone of the repository holding the integration branch")
-    ap.add_argument("--tip", required=True, help="the integration branch (or commit)")
+    ap.add_argument("--tip", help="the integration branch (or commit)")
+    ap.add_argument("--branch", help="sync: the branch to (re)create")
+    ap.add_argument("--pr", action="append", default=[], help="sync: owner/repo#N[@sha], in merge order (repeatable)")
+    ap.add_argument("--from", dest="source", help="sync: take the PR list (latest heads) from this existing branch")
+    ap.add_argument("--push", metavar="REMOTE", help="sync: force-with-lease push the branch to this remote")
     ap.add_argument("--base", default="origin/main", help="what the branch was built on (default origin/main)")
     ap.add_argument("--repos", default="", help="comma-separated owner/name list searched for each merged head's PR")
     ap.add_argument("--json", action="store_true")
+    ap.add_argument("--summary", action="store_true", help="report: one line, owner/repo#N@head9,... (an image label)")
     args = ap.parse_args(argv)
     repos = [r for r in args.repos.split(",") if r]
+    if args.command == "sync":
+        if not args.branch:
+            raise SystemExit("sync needs --branch")
+        prs = [parse_pr(p) for p in args.pr]
+        titles = {}
+        if args.source:
+            listed = report(analyze(Path(args.path), args.source, args.base), repos)
+            if listed["unmapped_merges"]:
+                raise SystemExit("some merges of the source branch name no PR; pass --repos or --pr explicitly")
+            prs += [(p["repo"], p["number"], None) for p in listed["prs"]]
+            titles = {(p["repo"], p["number"]): p["title"] for p in listed["prs"]}
+        if not prs:
+            raise SystemExit("sync needs at least one --pr (or --from)")
+        for repo, number, head in sync(args.path, args.branch, args.base, prs, titles=titles, push_to=args.push):
+            print(f"merged {repo}#{number} @{head[:9]}")
+        return 0
+    if not args.tip:
+        raise SystemExit(f"{args.command} needs --tip")
     analysis = analyze(Path(args.path), args.tip, args.base)
     data = report(analysis, repos)
     if args.command == "verify":
@@ -158,7 +244,9 @@ def main(argv=None):
             print(f"DRIFT {d['commit'][:9]} {d['subject'][:70]}: {d['problem']}", file=sys.stderr)
         print(f"{'FAILED' if data['drift'] else 'OK'}: {data['merges']} merges, {len(data['drift'])} problems", file=sys.stderr)
         return 1 if data["drift"] else 0
-    if args.json:
+    if args.summary:
+        print(",".join(f"{p['repo']}#{p['number']}@{p['merged_head'][:9]}" for p in data["prs"]))
+    elif args.json:
         print(json.dumps(data, indent=2))
     else:
         import yaml