bench.yml

name: bench

on:
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MARKER: "<!-- patchwork-bench -->"

jobs:
  # Say a run is under way, keeping the last results below it, marked as out
  # of date, until this one replaces them.
  announce:
    name: mark the comment out of date
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - uses: actions/github-script@v7
        with:
          script: |
            const marker = process.env.MARKER
            const previous = "<!-- previous -->"
            const { owner, repo } = context.repo
            const issue_number = context.issue.number
            const head = context.payload.pull_request.head.sha.slice(0, 7)
            const run = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`
            const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number })
            const existing = comments.find((c) => c.body.startsWith(marker))
            // A run that was cancelled leaves its notice; keep only the results.
            let last = existing?.body ?? ""
            if (last.includes(previous)) last = last.slice(last.indexOf(previous) + previous.length)
            else last = last.slice(marker.length)
            last = last.trim().replace(/^## (.*)$/m, "### Out of date: $1")
            const body = [
              marker,
              `## Benchmarks running for \`${head}\``,
              ``,
              `[A new run](${run}) is under way.${last ? " The results below are from the last run, for an earlier commit." : ""}`,
              previous,
              ``,
              last,
            ].join("\n")
            if (existing) {
              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })
            } else {
              await github.rest.issues.createComment({ owner, repo, issue_number, body })
            }

  # One runner per browser, so the browsers run at once without sharing a
  # machine. On each, main and the PR run one after the other: runner numbers
  # drift between runs, so a PR is only ever compared with main measured
  # alongside it, in the same browser. Only Patchwork as it ships (`patchwork`
  # mode, whose tests are titled "patchwork: …" or "S5 patchwork, …"): the
  # other modes and the storage adapters are alternatives the bench explores,
  # not what merging changes.
  bench:
    name: bench (${{ matrix.browser }})
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    steps:
      - uses: actions/checkout@v4
        with:
          path: pr

      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: main

      - uses: pnpm/action-setup@v4
        with:
          package_json_file: pr/package.json
          run_install: false

      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: pnpm
          cache-dependency-path: |
            pr/pnpm-lock.yaml
            main/pnpm-lock.yaml

      - name: build main
        working-directory: main
        run: pnpm install --frozen-lockfile && pnpm build

      - name: build the PR
        working-directory: pr
        run: pnpm install --frozen-lockfile && pnpm build

      - name: install ${{ matrix.browser }}
        run: |
          (cd main/sites/bench && pnpm exec playwright install --with-deps ${{ matrix.browser }})
          (cd pr/sites/bench && pnpm exec playwright install --with-deps ${{ matrix.browser }})

      # A single run of a benchmark can be off by more than the PR changes it,
      # so each side runs ROUNDS times, alternating, so that the runner
      # slowing down partway affects both. Failed tests still record their
      # rows, so every round reports whatever got measured. A side whose
      # bench.config.ts has no project for this browser records nothing.
      # Whether the PR passes is the report's call, from what's uploaded: a
      # PR whose tests threw leaves `threw` beside its runs.
      - name: bench main and the PR, alternating
        env:
          ROUNDS: 3
          BROWSER: ${{ matrix.browser }}
        run: |
          for side in main pr; do (cd $side/sites/bench && pnpm exec vite build); done
          mkdir -p runs
          for round in $(seq "$ROUNDS"); do
            for side in main pr; do
              echo "::group::$side, round $round"
              rm -f $side/sites/bench/bench-results/*.jsonl
              if ! (cd $side/sites/bench && pnpm exec playwright test -c bench.config.ts --project="$BROWSER" --grep "patchwork[:,]"); then
                if [ "$side" = pr ]; then touch runs/threw; fi
              fi
              mkdir -p "runs/$side-$round"
              cp $side/sites/bench/bench-results/*.jsonl "runs/$side-$round/" 2>/dev/null || true
              echo "::endgroup::"
            done
          done

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bench-runs-${{ matrix.browser }}
          path: runs/
          retention-days: 7

  # Every browser's runs side by side, in one comment, and the one check that
  # says whether the PR passes: it fails when the PR's benchmarks threw in any
  # browser, or a check (an "ok" row) that main passes failed in any round of
  # the PR (see compare.ts). Main failing doesn't fail the PR.
  report:
    name: benchmarks
    needs: bench
    # Not for a run cancelled by a newer push: that one reports instead.
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v4
        with:
          sparse-checkout: sites/bench/compare.ts
          sparse-checkout-cone-mode: false

      - uses: actions/setup-node@v4
        with:
          node-version: 24

      - uses: actions/download-artifact@v4
        with:
          pattern: bench-runs-*
          path: runs

      - name: compare
        run: |
          threw=""
          for browser in chromium firefox webkit; do
            dir="runs/bench-runs-$browser"
            if [ ! -d "$dir" ]; then
              threw="$threw $browser (recorded nothing)"
            elif [ -e "$dir/threw" ]; then
              threw="$threw $browser"
            fi
          done
          node sites/bench/compare.ts --main runs/*/main-* --pr runs/*/pr-* --why why.txt > compare.md || touch pr-regressed
          if [ -n "$threw" ]; then
            echo "$threw" > pr-failed
            { echo "**The PR's benchmarks threw in:$threw.** See the run's logs."; echo; cat compare.md; } > compare.tmp
            mv compare.tmp compare.md
          fi
          cat compare.md >> "$GITHUB_STEP_SUMMARY"

      - uses: actions/github-script@v7
        with:
          script: |
            const fs = require("fs")
            let compare = ""
            try { compare = fs.readFileSync("compare.md", "utf8") } catch {}
            if (!compare) return
            const marker = process.env.MARKER
            const { pull_request } = context.payload
            const head = pull_request.head.sha.slice(0, 7)
            const base = pull_request.base.sha.slice(0, 7)
            const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
            const body = [
              marker,
              `## Benchmarks: Patchwork with this PR against main`,
              ``,
              `\`${head}\` against main at \`${base}\`, Patchwork as it ships (\`patchwork\` mode), in chromium, firefox and webkit, each on its own runner with main and the PR measured one after the other ([run](${run})). Lower is better throughout. A row is in the table when every run of the PR came out on the same side of every run of main, with the medians more than 20% apart; every benchmark is below it.`,
              ``,
              compare,
            ].join("\n")
            const {owner, repo} = context.repo
            const issue_number = context.issue.number
            const comments = await github.paginate(github.rest.issues.listComments, {owner, repo, issue_number})
            const existing = comments.find(c => c.body.startsWith(marker))
            if (existing) {
              await github.rest.issues.updateComment({owner, repo, comment_id: existing.id, body})
            } else {
              await github.rest.issues.createComment({owner, repo, issue_number, body})
            }

      # Says why as error annotations, which show on the run's summary page and
      # the PR's checks. The comment has the same, with the tables.
      - name: fail if the PR's benchmarks failed
        run: |
          failed=0
          if [ -e pr-failed ]; then
            echo "::error title=benchmarks threw::The PR's benchmarks threw in:$(cat pr-failed). See that browser's bench job for the test that threw."
            failed=1
          fi
          if [ -e pr-regressed ]; then
            if [ -s why.txt ]; then
              while IFS= read -r line; do
                echo "::error title=check failed::${line//%/%25}"
              done < why.txt
            else
              echo "::error title=compare failed::compare.ts exited without naming a check that failed; see the compare step."
            fi
            failed=1
          fi
          exit $failed