ci: Fix sticky caches

.github/workflows/bench-main.yml

@@ -46,25 +46,29 @@ jobs:
   # with native code generation for r8i, with caches and measured results
   # isolated from other hardware and compiler flags.
   build:
-    runs-on: runs-on=${{ github.run_id }}-build-${{ github.run_attempt }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/extras=s3-cache
+    # The sticky disk is the native-r8i lineage bench-pr's build job shares:
+    # this push run leaves the lineage's `main` snapshot, which a
+    # `!benchmark` run dispatched on its PR branch reads without writing.
+    runs-on: runs-on=${{ github.run_id }}-build-${{ github.run_attempt }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/sticky=bench-build-r8i-native:50gb/extras=s3-cache
     steps:
-      - uses: runs-on/action@v2
       - uses: actions/checkout@v7
-      - name: Mount Lake build cache
-        uses: actions/cache@v6
+      # Lake and cargo artifacts live on the sticky disk; sccache serves the
+      # workspace crates cargo rebuilds after every fresh checkout (see
+      # ci.yml), under entries the native codegen flags keep apart from CI's.
+      - uses: runs-on/action@v2
         with:
-          path: .lake
-          key: ${{ env.BENCH_CACHE }}-lake-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }}-${{ github.sha }}
-          restore-keys: ${{ env.BENCH_CACHE }}-lake-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }}-
+          sccache: s3
+          sticky_cache: |
+            rust
+            custom,path=.lake,path=target
+      - uses: mozilla-actions/sccache-action@v0.0.11
       - name: Log build CPU
         uses: ./.github/actions/log-cpu
         with:
           label: Benchmark binary build CPU
-      # Rust toolchain + cargo cache, so the cargo step inside `lake build`
-      # does not recompile the Plonky3/multi-stark dependencies every run.
       - uses: ./.github/actions/setup-rust-toolchain
         with:
-          cache-key: ${{ env.BENCH_CACHE }}
+          use-github-cache: "false"
       - uses: leanprover/lean-action@v1
         with:
           auto-config: false
@@ -126,7 +130,10 @@ jobs:
     # job title.
     name: compile-${{ matrix.env }}
     needs: build
-    runs-on: runs-on=${{ github.run_id }}-compile-${{ github.run_attempt }}-${{ strategy.job-index }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/extras=s3-cache
+    # One sticky lineage per env: concurrent jobs on one lineage keep only
+    # the last clean completion, so a shared disk would drop every other
+    # env's Lake packages and oleans each run.
+    runs-on: runs-on=${{ github.run_id }}-compile-${{ github.run_attempt }}-${{ strategy.job-index }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/sticky=bench-compile-${{ matrix.env }}-r8i-native:100gb/extras=s3-cache
     permissions:
       contents: read
       checks: write
@@ -148,10 +155,16 @@ jobs:
           # shared Compile package; no mathlib cache needed.
           - { env: ISLB }
           - { env: Mathlib, mathlib: true }
-          - { env: FLT, cache_pkg: flt, mathlib: true }
+          - { env: FLT, mathlib: true }
     steps:
-      - uses: runs-on/action@v2
       - uses: actions/checkout@v7
+      # The compile workspace's `.lake` persists on the sticky disk: its
+      # Lake packages, the Mathlib oleans `cache get` fetched, and the env's
+      # own build, so later runs replay instead of refetching and rebuilding.
+      - uses: runs-on/action@v2
+        with:
+          sticky_cache: |
+            custom,path=${{ env.COMPILE_DIR }}/.lake
       # `lake build` below clones this package's Lake dependencies.
       - uses: ./.github/actions/authenticate-github-fetches
       - uses: actions/cache/restore@v6
@@ -174,12 +187,6 @@ jobs:
           auto-config: false
           use-github-cache: false
           use-mathlib-cache: ${{ matrix.mathlib && 'true' || 'false' }}
-      # FLT takes a few minutes to rebuild, so cache its build artifacts.
-      - if: matrix.cache_pkg
-        uses: actions/cache@v6
-        with:
-          path: ${{ env.COMPILE_DIR }}/.lake/packages/${{ matrix.cache_pkg }}/.lake/build
-          key: ${{ env.BENCH_CACHE }}-${{ matrix.cache_pkg }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}/lean-toolchain', env.COMPILE_DIR)) }}-${{ hashFiles(format('{0}/lake-manifest.json', env.COMPILE_DIR)) }}
       - run: lake build Compile${{ matrix.env }}
         working-directory: ${{ env.COMPILE_DIR }}
       # The measured compile: serializes the env to `<env>.ixe` at the

.github/workflows/bench-pr.yml

@@ -114,21 +114,32 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       actions: write
-      pull-requests: read
+      pull-requests: write
     steps:
       # issue_comment doesn't carry the PR's base/head; this action looks
       # them up.
       - uses: xt0rted/pull-request-comment-branch@v3
         id: comment-branch
-      - name: Dispatch the trusted run
+      # Dispatched on the PR's branch, not the default branch: RunsOn files
+      # each run's sticky-disk snapshot under its ref, so the build lands on
+      # the branch and only reads `main`'s (bench-main's) snapshot. A fork
+      # branch cannot be dispatched, and would write `main`'s otherwise.
+      - name: Dispatch on the PR branch
         env:
           GH_TOKEN: ${{ github.token }}
           COMMENT_BODY: ${{ github.event.comment.body }}
           PR_NUMBER: ${{ github.event.issue.number }}
+          HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }}
         run: |
+          head_repo=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.head.repo.full_name')
+          if [ "$head_repo" != "$GITHUB_REPOSITORY" ]; then
+            printf '%s\n' "Benchmarks run on branches of this repository only; \`$head_repo\` is a fork." > comment-body.md
+            gh pr comment "$PR_NUMBER" --body-file comment-body.md
+            exit 1
+          fi
           gh workflow run bench-pr.yml \
             --repo "$GITHUB_REPOSITORY" \
-            --ref "${{ github.event.repository.default_branch }}" \
+            --ref "$HEAD_REF" \
             -f pr="$PR_NUMBER" \
             -f base-sha="${{ steps.comment-branch.outputs.base_sha }}" \
             -f base-ref="${{ steps.comment-branch.outputs.base_ref }}" \
@@ -147,7 +158,10 @@ jobs:
   # available.
   build:
     if: github.event_name == 'workflow_dispatch'
-    runs-on: runs-on=${{ github.run_id }}-build-${{ github.run_attempt }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/extras=s3-cache
+    # bench-main's build lineage: its push-to-main runs leave the `main`
+    # snapshot this run falls back to; this run's own snapshot goes to
+    # the PR branch the relay dispatched on.
+    runs-on: runs-on=${{ github.run_id }}-build-${{ github.run_attempt }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/sticky=bench-build-r8i-native:50gb/extras=s3-cache
     outputs:
       revision: ${{ steps.target.outputs.revision }}
       matrix: ${{ steps.parse.outputs.matrix }}
@@ -159,7 +173,6 @@ jobs:
       # rejected; the failure comment quotes it.
       parse-error: ${{ steps.parse.outputs.parse-error }}
     steps:
-      - uses: runs-on/action@v2
       # Comment dispatch requires an org MEMBER/OWNER; manual dispatch requires
       # repo write access. Emit the validated full SHA once for every checkout.
       - name: Validate SHAs
@@ -176,6 +189,16 @@ jobs:
           ref: ${{ steps.target.outputs.revision }}
           # The job builds and runs PR code; never leave the token in .git.
           persist-credentials: false
+      # Lake and cargo artifacts live on the sticky disk; sccache serves the
+      # workspace crates cargo rebuilds after every fresh checkout (see
+      # ci.yml), under entries the native codegen flags keep apart from CI's.
+      - uses: runs-on/action@v2
+        with:
+          sccache: s3
+          sticky_cache: |
+            rust
+            custom,path=.lake,path=target
+      - uses: mozilla-actions/sccache-action@v0.0.11
       - id: bins
         uses: actions/cache/restore@v6
         with:
@@ -186,7 +209,7 @@ jobs:
       - if: steps.bins.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-rust-toolchain
         with:
-          cache-key: ${{ env.BENCH_CACHE }}
+          use-github-cache: "false"
       # A persistent hit still needs the matching toolchain for libleanshared.
       - uses: leanprover/lean-action@v1
         with:
@@ -212,7 +235,7 @@ jobs:
       - if: steps.parse.outputs.fresh == '1' && steps.bins.outputs.cache-hit == 'true'
         uses: ./.github/actions/setup-rust-toolchain
         with:
-          cache-key: ${{ env.BENCH_CACHE }}
+          use-github-cache: "false"
       - name: Log build CPU
         uses: ./.github/actions/log-cpu
         with:
@@ -310,20 +333,25 @@ jobs:
     name: compile-${{ matrix.env }}
     needs: build
     if: needs.build.outputs.envs != '[]'
-    runs-on: runs-on=${{ github.run_id }}-compile-${{ github.run_attempt }}-${{ strategy.job-index }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/extras=s3-cache
+    # bench-main's per-env compile lineage (see there): this run reads the
+    # env's `main` snapshot and leaves its own on the PR branch.
+    runs-on: runs-on=${{ github.run_id }}-compile-${{ github.run_attempt }}-${{ strategy.job-index }}/cpu=32/family=r8i.8xlarge/spot=false/image=ubuntu26-full-x64/volume=100gb/sticky=bench-compile-${{ matrix.env }}-r8i-native:100gb/extras=s3-cache
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         env: ${{ fromJSON(needs.build.outputs.envs) }}
     steps:
-      - uses: runs-on/action@v2
       - name: Checkout PR
         uses: actions/checkout@v7
         with:
           ref: ${{ needs.build.outputs.revision }}
           # The job runs PR code; never leave the token in .git.
           persist-credentials: false
+      - uses: runs-on/action@v2
+        with:
+          sticky_cache: |
+            custom,path=Benchmarks/Compile/.lake
       # `lake build` below clones this package's Lake dependencies.
       - uses: ./.github/actions/authenticate-github-fetches
       # Apply the allowlisted KEY=VALUE lines from the !benchmark comment,
@@ -469,7 +497,12 @@ jobs:
       HEAD_SHA: ${{ needs.build.outputs.revision }}
       FRESH: ${{ needs.build.outputs.fresh }}
     steps:
+      # sccache for the from-scratch base build below: its workspace crates
+      # are what the base's own bench-main run compiled (see ci.yml).
       - uses: runs-on/action@v2
+        with:
+          sccache: s3
+      - uses: mozilla-actions/sccache-action@v0.0.11
       # The PR is checked out at the workspace root (the local composite
       # actions resolve from there); the base checkout, when needed, goes
       # under base/.

.github/workflows/ci.yml

@@ -3,6 +3,12 @@ name: CI Jobs
 on:
   pull_request:
   merge_group:
+  # Pushes to main run the same jobs so their sticky disks get a `main`
+  # snapshot: RunsOn files each run's snapshot under its own ref, and a ref
+  # with none falls back to the default branch's. Without this run every
+  # new PR branch and every merge-queue entry starts from an empty disk.
+  push:
+    branches: [main]
   workflow_dispatch:
 
 permissions:
@@ -20,11 +26,17 @@ jobs:
     runs-on: runs-on=${{ github.run_id }}-lean-test-${{ github.run_attempt }}/cpu=16/family=r7i+r8i+r7a+r8a/image=ubuntu26-full-x64/volume=100gb/sticky=ci-lean-test-x86-64-v4:50gb/extras=s3-cache
     steps:
       - uses: actions/checkout@v7
+      # sccache keys rustc outputs by content, so the workspace crates, which
+      # cargo rebuilds after every fresh checkout because their mtimes moved,
+      # come back as cache hits. RunsOn configures it before anything can
+      # start an sccache server.
       - uses: runs-on/action@v2
         with:
+          sccache: s3
           sticky_cache: |
             rust
             custom,path=.lake,path=target
+      - uses: mozilla-actions/sccache-action@v0.0.11
       - uses: ./.github/actions/setup-rust-toolchain
         with:
           codegen: portable
@@ -41,17 +53,24 @@ jobs:
         run: lake lint -- --wfail -v
       - name: Build Ix.Tc formal verification
         run: lake build IxTcVerify
+      # A push to main already passed these checks in the merge queue; that
+      # run only exists to snapshot the build products above.
       - name: Check codegen'd IxVM kernel is up to date
+        if: github.event_name != 'push'
         run: lake exe ix codegen --check
       - name: Check Lean versions match for Ix and compiler bench
+        if: github.event_name != 'push'
         run: diff lean-toolchain Benchmarks/Compile/lean-toolchain
       # The primary test tier runs over the targets the lint step just built,
       # so the tests add no compilation of their own.
       - name: Run primary tests
+        if: github.event_name != 'push'
         run: lake test --wfail
       - name: Test Ix CLI
+        if: github.event_name != 'push'
         run: lake test --wfail -- cli
       - name: Test Ixon v3 contracts and consumers
+        if: github.event_name != 'push'
         run: |
           lake exe ixon-v3-primitives
           lake exe ixon-v3-tests --primitives
@@ -60,11 +79,17 @@ jobs:
     runs-on: runs-on=${{ github.run_id }}-rust-test-${{ github.run_attempt }}/cpu=8/family=r7i+r8i+r7a+r8a/image=ubuntu26-full-x64/volume=100gb/sticky=ci-rust-test-x86-64-v4:50gb/extras=s3-cache
     steps:
       - uses: actions/checkout@v7
+      # sccache keys rustc outputs by content, so the workspace crates, which
+      # cargo rebuilds after every fresh checkout because their mtimes moved,
+      # come back as cache hits. RunsOn configures it before anything can
+      # start an sccache server.
       - uses: runs-on/action@v2
         with:
+          sccache: s3
           sticky_cache: |
             rust
             custom,path=.lake,path=target
+      - uses: mozilla-actions/sccache-action@v0.0.11
       - uses: ./.github/actions/setup-rust-toolchain
         with:
           codegen: portable
@@ -75,19 +100,30 @@ jobs:
         with:
           auto-config: false
           use-github-cache: false
+      # A push to main already passed the lints and tests in the merge
+      # queue; that run only builds, so the sticky disk holds the check,
+      # clippy and test-binary artifacts the next PR run will fingerprint.
       - name: Check Rustfmt code style
+        if: github.event_name != 'push'
         uses: actions-rust-lang/rustfmt@v1
       - name: Check clippy warnings
         run: cargo clippy --release --workspace --all-targets --features ix-ffi/parallel,ix-ffi/net,ix-ffi/test-ffi -- -D warnings
       - name: Check *everything* compiles
         run: cargo check --release --workspace --all-targets --features ix-ffi/parallel,ix-ffi/net,ix-ffi/test-ffi
       - name: Tests
-        run: cargo nextest run --release --profile ci --workspace
+        run: cargo nextest run --release --profile ci --workspace ${{ github.event_name == 'push' && '--no-run' || '' }}
       - name: Get Rust version
+        if: github.event_name != 'push'
         run: |
           echo "RUST_VERSION=$(awk -F '"' '/^channel/ {print $2}' rust-toolchain.toml)" | tee -a $GITHUB_ENV
+      # A Docker action: the job's RUSTC_WRAPPER reaches its container, where
+      # sccache is not installed, and cargo fails before deny runs. Deny only
+      # reads metadata, and cargo treats an empty wrapper as none.
       - name: Cargo-deny
+        if: github.event_name != 'push'
         uses: EmbarkStudios/cargo-deny-action@v2
+        env:
+          RUSTC_WRAPPER: ""
         with:
           rust-version: ${{ env.RUST_VERSION }}
 
@@ -104,6 +140,12 @@ jobs:
       # sufficient for this downstream integration gate.
       MULTI_STARK_CUDA_ARCHS: "80"
     steps:
+      # No sccache here yet. Once the GPU benchmark lands after #644, wire
+      # it for both compilers: `sccache: s3` plus the sccache action for
+      # rustc (this job runs in a container, so its S3 credentials via the
+      # instance role need checking first), and an `NVCC` wrapper running
+      # `sccache nvcc` for the kernels, which multi-stark's build script
+      # must compile with `-c` per source for sccache to cache them.
       - uses: runs-on/action@v2
       - uses: actions/checkout@v7
       - name: Install toolchain bootstrap dependencies