From 3c2c392801abf5197a2435eadb197b916856ef07 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Tue, 25 Aug 2026 15:38:39 +0930 Subject: [PATCH 01/12] Add an optional preview performance check Adds an opt-in Lighthouse check that measures Core Web Vitals against the preview deployment and comments the median results on the pull request. The check is off by default and runs only when `performance-check` is true and `measured-paths` is non-empty, so existing callers are unaffected. The change is purely additive: no existing line is modified, and the deploy job gains only an `outputs.url` so the new job can consume the preview URL. The caller supplies the measured paths and a Lighthouse CI config holding run count and budgets. Documented with a worked example, including why the config should set `aggregationMethod: median` (LHCI defaults to `optimistic`, which takes the best run) and why the budgets should be advisory on a cold preview. Co-Authored-By: Claude Opus 5 --- .github/workflows/vercel-preview.yml | 308 +++++++++++++++++++++++++++ docs/vercel-preview.md | 118 +++++++++- 2 files changed, 417 insertions(+), 9 deletions(-) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index ed4bde9..2caa023 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -21,10 +21,43 @@ on: type: string required: false default: "Preview" + performance-check: + description: >- + Run a Lighthouse performance check against the preview and comment + the median Core Web Vitals on the pull request. Requires + measured-paths to be set. + type: boolean + required: false + default: false + measured-paths: + description: >- + Paths to measure, one per line, relative to the preview URL + (e.g. "/"). Newline-delimited so paths may contain query strings. + The performance check is skipped when this is empty. + type: string + required: false + default: "" + lighthouse-config-path: + description: >- + Path to a Lighthouse CI config file in the calling repository, + holding run count and budgets. + type: string + required: false + default: ".github/lighthouse/lighthouserc.json" + node-version-file: + description: "File to read the Node version from for the performance check" + type: string + required: false + default: ".nvmrc" secrets: vercel-token: description: "Vercel deployment token" required: true + vercel-automation-bypass-secret: + description: >- + Vercel Protection Bypass for Automation secret. Required for the + performance check when Deployment Protection is enabled. + required: false concurrency: group: vercel-preview-${{ github.event.pull_request.number }} @@ -41,6 +74,8 @@ jobs: environment: name: ${{ inputs.environment-name }} url: ${{ steps.deploy.outputs.url }} + outputs: + url: ${{ steps.deploy.outputs.url }} env: VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} VERCEL_PROJECT_ID: ${{ inputs.vercel-project-id }} @@ -101,3 +136,276 @@ jobs: else gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$body" fi + + # Opt-in: measures Core Web Vitals against the preview deployment and + # comments the median results on the pull request. Skipped entirely unless + # the caller sets performance-check and provides measured-paths, so existing + # callers are unaffected. + performance: + name: Preview Performance + needs: deploy-preview + if: inputs.performance-check && inputs.measured-paths != '' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + env: + # How long to wait for the preview deployment to finish building. + READY_TIMEOUT: 10m + MEASURED_PATHS: ${{ inputs.measured-paths }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0 + with: + persist-credentials: false + + - name: Install Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 + with: + node-version-file: ${{ inputs.node-version-file }} + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + # The deploy job uses `--no-wait`, so the deployment is still building + # when this job starts. `--wait` blocks until the build completes and + # exits non-zero if it fails, so no polling loop is needed. + - name: Wait for preview deployment + id: deployment + env: + VERCEL_TOKEN: ${{ secrets.vercel-token }} + VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} + DEPLOY_URL: ${{ needs.deploy-preview.outputs.url }} + run: | + set -euo pipefail + + if [ -z "$DEPLOY_URL" ]; then + echo "::error::The deploy job did not produce a preview URL." + exit 1 + fi + + # `vercel inspect` takes a deployment URL rather than a linked + # project, so it resolves scope from the token's default team + # instead of from VERCEL_ORG_ID the way `vercel deploy` does. Name + # the owning team explicitly or the deployment reads as out of + # scope. `--scope` accepts the team ID that VERCEL_ORG_ID holds, + # despite the docs describing it as a slug; `--team` is deprecated. + echo "Waiting for ${DEPLOY_URL} to become ready." + if ! vercel inspect "$DEPLOY_URL" \ + --token="$VERCEL_TOKEN" \ + --scope="$VERCEL_ORG_ID" \ + --wait \ + --timeout "$READY_TIMEOUT"; then + echo "::error::Preview deployment was not ready within ${READY_TIMEOUT}." + exit 1 + fi + + echo "url=$DEPLOY_URL" >> "$GITHUB_OUTPUT" + + # The first request to a cold preview pays serverless cold start and any + # on-demand page generation. Measuring that would report build latency + # rather than page performance, so discard it. Each path is warmed + # separately because they are generated on demand independently. + - name: Warm up preview deployment + id: urls + env: + BASE_URL: ${{ steps.deployment.outputs.url }} + BYPASS_SECRET: ${{ secrets.vercel-automation-bypass-secret }} + run: | + set -euo pipefail + + failed=0 + urls="" + + # Read line by line: a path may contain `?` and `&`, so splitting on + # whitespace would mangle it. + while IFS= read -r path; do + [ -z "$path" ] && continue + url="${BASE_URL}${path}" + urls="${urls}${url}"$'\n' + + for attempt in 1 2 3; do + status="$(curl -sS -o /dev/null -w '%{http_code}' -L \ + --max-time 60 \ + ${BYPASS_SECRET:+-H "x-vercel-protection-bypass: ${BYPASS_SECRET}"} \ + "$url" || echo 000)" + echo "Warm-up ${url} attempt ${attempt}: HTTP ${status}" + [ "$status" = "200" ] && break + [ "$attempt" = "3" ] && failed=1 + sleep 5 + done + done <<< "$MEASURED_PATHS" + + if [ "$failed" = "1" ]; then + echo "::error::One or more URLs did not return HTTP 200 after 3 warm-up requests." + echo "::error::If Deployment Protection is enabled, pass the vercel-automation-bypass-secret secret." + exit 1 + fi + + # Hand the fully-qualified URLs to the Lighthouse step so the list is + # defined once, by the caller, in measured-paths. + { + echo "list<> "$GITHUB_OUTPUT" + + # Run count and budgets come from the caller's Lighthouse CI config. Set + # `aggregationMethod: median` there: LHCI defaults to `optimistic`, which + # takes the best run and hides the variance that multiple runs exist to + # smooth out. + - name: Run Lighthouse + id: lighthouse + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12.6.1 + with: + urls: ${{ steps.urls.outputs.list }} + configPath: ${{ inputs.lighthouse-config-path }} + uploadArtifacts: true + artifactName: lighthouse-reports + + - name: Summarise results + id: summary + env: + RESULTS_PATH: ${{ steps.lighthouse.outputs.resultsPath }} + ASSERTION_RESULTS: ${{ steps.lighthouse.outputs.assertionResults }} + run: | + set -euo pipefail + + # Written out rather than inlined with `node -e` so the quoting stays + # manageable and the script is readable. + cat > summarise.mjs <<'SUMMARISE_EOF' + import { readFileSync } from 'node:fs'; + import { join } from 'node:path'; + + const METRICS = [ + ['largest-contentful-paint', 'LCP', 'ms'], + ['total-blocking-time', 'TBT', 'ms'], + ['cumulative-layout-shift', 'CLS', ''], + ['first-contentful-paint', 'FCP', 'ms'], + ['speed-index', 'SI', 'ms'], + ]; + + const format = (value, unit) => { + if (typeof value !== 'number') return 'n/a'; + if (unit !== 'ms') return value.toFixed(3); + return value >= 1000 + ? `${(value / 1000).toFixed(2)} s` + : `${Math.round(value)} ms`; + }; + + const toLabel = (url) => { + try { + return new URL(url).pathname || '/'; + } catch { + return url; + } + }; + + const manifest = JSON.parse( + readFileSync(join(process.env.RESULTS_PATH, 'manifest.json'), 'utf8') + ); + + // LHCI writes one manifest entry per run. The entry flagged as the + // representative run is the median run for that URL, so reporting it + // keeps this summary consistent with what the assertions checked. + const representative = manifest.filter((entry) => entry.isRepresentativeRun); + if (representative.length === 0) { + console.error('Manifest contained no representative run.'); + process.exit(1); + } + + // Assertion failures are keyed by URL so each row can be flagged. + const warningsByUrl = new Map(); + try { + for (const result of JSON.parse(process.env.ASSERTION_RESULTS || '[]')) { + if (!result.url) continue; + const existing = warningsByUrl.get(result.url) ?? new Set(); + existing.add(result.auditId); + warningsByUrl.set(result.url, existing); + } + } catch (error) { + console.error(`Could not parse assertion results: ${error.message}`); + } + + const lines = []; + let anyWarnings = false; + + for (const entry of representative) { + const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + const warned = warningsByUrl.get(entry.url) ?? new Set(); + if (warned.size > 0) anyWarnings = true; + + const score = entry.summary?.performance; + lines.push( + `#### \`${toLabel(entry.url)}\``, + '', + `**Performance score:** ${ + typeof score === 'number' ? Math.round(score * 100) : 'n/a' + }/100${warned.has('categories:performance') ? ' ⚠️' : ''}`, + '', + '| Metric | Median | |', + '| --- | --- | --- |' + ); + + for (const [id, label, unit] of METRICS) { + const value = report.audits?.[id]?.numericValue; + lines.push( + `| ${label} | ${format(value, unit)} | ${ + warned.has(id) ? '⚠️' : '✅' + } |` + ); + } + + lines.push(''); + } + + const runs = manifest.length / representative.length; + lines.push(`_Median of ${runs} run${runs === 1 ? '' : 's'} per URL._`); + + // Preview deployments are cold and CPU-shared on CI runners, so treat + // these as a smoke signal for large regressions, not a benchmark. + if (anyWarnings) { + lines.push( + '', + '⚠️ One or more metrics are over budget. Preview deployments ' + + 'are slower than production — confirm against a second run ' + + 'before treating this as a real regression.' + ); + } + + console.log(lines.join('\n')); + SUMMARISE_EOF + + node summarise.mjs > lighthouse-summary.md + cat lighthouse-summary.md + + # Posts (or updates in place) a single comment on the PR, separate from + # the preview URL comment above. + - name: Comment performance results on PR + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + marker='' + { + echo "$marker" + echo "### 📊 Preview performance" + echo + cat lighthouse-summary.md + echo + echo "[Full reports](${RUN_URL}) · _commit ${COMMIT_SHA}_" + } > comment.md + + comment_id="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq "[.[] | select(.body | startswith(\"${marker}\"))][0].id")" + + if [ -n "$comment_id" ] && [ "$comment_id" != "null" ]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body=@comment.md + else + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@comment.md + fi diff --git a/docs/vercel-preview.md b/docs/vercel-preview.md index 63a9ae2..0aad9de 100644 --- a/docs/vercel-preview.md +++ b/docs/vercel-preview.md @@ -5,17 +5,22 @@ pull request with the preview and inspect URLs. Intended to be called from a `pull_request` triggered workflow. #### **Inputs** -| Name | Required | Type | Default | Description | -|--------------------|----------|--------|----------|----------------------------------------| -| vercel-org-id | ✅ | string | | Vercel organisation ID | -| vercel-project-id | ✅ | string | | Vercel project ID | -| working-directory | ❌ | string | . | Directory to run the Vercel deploy from | -| environment-name | ❌ | string | Preview | GitHub Environment to deploy to | +| Name | Required | Type | Default | Description | +|------------------------|----------|---------|------------------------------------------|-----------------------------------------| +| vercel-org-id | ✅ | string | | Vercel organisation ID | +| vercel-project-id | ✅ | string | | Vercel project ID | +| working-directory | ❌ | string | . | Directory to run the Vercel deploy from | +| environment-name | ❌ | string | Preview | GitHub Environment to deploy to | +| performance-check | ❌ | boolean | false | Run a Lighthouse check against the preview | +| measured-paths | ❌ | string | | Paths to measure, one per line | +| lighthouse-config-path | ❌ | string | .github/lighthouse/lighthouserc.json | Lighthouse CI config in the caller repo | +| node-version-file | ❌ | string | .nvmrc | Node version file for the check | #### **Secrets** -| Name | Required | Description | -|---------------|----------|--------------------------| -| vercel-token | ✅ | Vercel deployment token | +| Name | Required | Description | +|---------------------------------|----------|------------------------------------------------------| +| vercel-token | ✅ | Vercel deployment token | +| vercel-automation-bypass-secret | ❌ | Protection Bypass for Automation, for the perf check | #### Example Usage @@ -34,3 +39,98 @@ jobs: secrets: vercel-token: ${{ secrets.VERCEL_TOKEN }} ``` + +## Performance Check + +Optionally measures Core Web Vitals against the preview deployment with +[Lighthouse CI](https://github.com/treosh/lighthouse-ci-action) and posts the +median results as a pull request comment. + +The check is **off by default**. It runs only when `performance-check` is true +*and* `measured-paths` is non-empty, so existing callers are unaffected. + +### How it works + +1. Waits for the preview deployment to finish building (the deploy step uses + `--no-wait`, so the deploy job itself returns as soon as it has a URL). +2. Warms each measured path. The first request to a cold preview pays + serverless cold start and any on-demand page generation, which would + otherwise be measured as page latency. +3. Runs Lighthouse against every path and comments the median of each metric. + +### Reading the results + +Preview deployments are cold and run on shared CI runners, so treat the numbers +as a smoke signal for large regressions rather than a benchmark. Two things +follow from that: + +- **Use several runs and aggregate on the median.** A single run on a cold + preview is not a usable signal. +- **Keep budgets loose.** Scoring a preview against production-grade + thresholds flags nearly every pull request, and a check that cries wolf gets + ignored. + +### Example Usage + +```yaml +on: + pull_request: + branches: + - main + +jobs: + deploy-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + performance-check: true + # One path per line. Paths may contain query strings. + measured-paths: | + / + /women/tops-women/jackets-women.html + /stellar-solar-jacket.html?categoryPath=jackets-women + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + # Only needed when Deployment Protection is enabled on the project. + vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} +``` + +The caller supplies the Lighthouse CI config, by default at +`.github/lighthouse/lighthouserc.json`: + +```json +{ + "ci": { + "collect": { + "numberOfRuns": 3, + "settings": { + "preset": "desktop", + "onlyCategories": ["performance"], + "maxWaitForLoad": 60000, + "chromeFlags": "--no-sandbox --disable-dev-shm-usage" + } + }, + "assert": { + "aggregationMethod": "median", + "assertions": { + "categories:performance": ["warn", { "minScore": 0.5 }], + "largest-contentful-paint": ["warn", { "maxNumericValue": 4000 }], + "total-blocking-time": ["warn", { "maxNumericValue": 600 }], + "cumulative-layout-shift": ["warn", { "maxNumericValue": 0.25 }], + "first-contentful-paint": ["warn", { "maxNumericValue": 3000 }], + "speed-index": ["warn", { "maxNumericValue": 5800 }] + } + } + } +} +``` + +Two things in that config are worth calling out: + +- **`aggregationMethod: median`** — Lighthouse CI defaults to `optimistic`, + which takes the *best* run and so discards the variance that multiple runs + exist to smooth out. Set this explicitly. +- **`warn` rather than `error`** — the job reports breaches without failing, so + a noisy preview cannot block a merge. Switch to `error` once you have enough + history to trust the thresholds. From 87dbc5e16cfe346dd2c8addcc2e8e4c44aaca23c Mon Sep 17 00:00:00 2001 From: Chris Park Date: Tue, 25 Aug 2026 15:49:44 +0930 Subject: [PATCH 02/12] Document Deployment Protection for the performance check The secret was named in the inputs table and the example, but nothing said where to obtain it, how to tell whether a project needs it, or what happens without it. Someone hitting a 401 during warm-up had no path from the error to the fix. Adds a Deployment Protection section covering how to generate the secret, the permissions required, the redeploy caveat when rotating it, and how to check whether a project has protection enabled. Co-Authored-By: Claude Opus 5 --- .github/workflows/vercel-preview.yml | 17 ++++---------- docs/vercel-preview.md | 34 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 2caa023..40eb93a 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -139,8 +139,7 @@ jobs: # Opt-in: measures Core Web Vitals against the preview deployment and # comments the median results on the pull request. Skipped entirely unless - # the caller sets performance-check and provides measured-paths, so existing - # callers are unaffected. + # the caller sets performance-check and provides measured-paths. performance: name: Preview Performance needs: deploy-preview @@ -159,17 +158,19 @@ jobs: with: persist-credentials: false + # `package-manager-cache: false` because this job never installs project + # dependencies — it only needs Node to summarise the Lighthouse results. - name: Install Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 with: node-version-file: ${{ inputs.node-version-file }} + package-manager-cache: false - name: Install Vercel CLI run: npm install --global vercel@latest # The deploy job uses `--no-wait`, so the deployment is still building - # when this job starts. `--wait` blocks until the build completes and - # exits non-zero if it fails, so no polling loop is needed. + # when this job starts. Wait for it to finish so the performance check can run against a fully built preview. - name: Wait for preview deployment id: deployment env: @@ -184,12 +185,6 @@ jobs: exit 1 fi - # `vercel inspect` takes a deployment URL rather than a linked - # project, so it resolves scope from the token's default team - # instead of from VERCEL_ORG_ID the way `vercel deploy` does. Name - # the owning team explicitly or the deployment reads as out of - # scope. `--scope` accepts the team ID that VERCEL_ORG_ID holds, - # despite the docs describing it as a slug; `--team` is deprecated. echo "Waiting for ${DEPLOY_URL} to become ready." if ! vercel inspect "$DEPLOY_URL" \ --token="$VERCEL_TOKEN" \ @@ -271,8 +266,6 @@ jobs: run: | set -euo pipefail - # Written out rather than inlined with `node -e` so the quoting stays - # manageable and the script is readable. cat > summarise.mjs <<'SUMMARISE_EOF' import { readFileSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/docs/vercel-preview.md b/docs/vercel-preview.md index 0aad9de..393988b 100644 --- a/docs/vercel-preview.md +++ b/docs/vercel-preview.md @@ -58,6 +58,40 @@ The check is **off by default**. It runs only when `performance-check` is true otherwise be measured as page latency. 3. Runs Lighthouse against every path and comments the median of each metric. +### Deployment Protection + +If the Vercel project has [Deployment +Protection](https://vercel.com/docs/deployment-protection) enabled, preview URLs +require a Vercel login. Automated requests receive an authentication page rather +than the site, so Lighthouse would score the login screen instead of the page +under test. + +To allow the check through, generate a [Protection Bypass for +Automation](https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/protection-bypass-automation) +secret in the Vercel project's Deployment Protection settings, add it to the +calling repository as a secret, and pass it as +`vercel-automation-bypass-secret`. The workflow sends it as the +`x-vercel-protection-bypass` header on both the warm-up requests and the +Lighthouse runs. + +Generating the secret requires at least the **member** team role, or the +**Project Administrator** role on the project. Note that regenerating or +deleting a secret invalidates it for existing deployments, which then need to +be redeployed. + +The secret is optional and is omitted from requests entirely when unset, which +is correct for a project without Deployment Protection. If protection *is* +enabled and the secret is missing, the warm-up step fails with a non-200 status +rather than reporting misleading scores. + +To check whether a project needs it, open a preview URL in a private browser +window: a login prompt means protection is enabled. + +> Vercel also exposes this value to the running deployment as the +> `VERCEL_AUTOMATION_BYPASS_SECRET` system environment variable, but that is not +> usable here — the workflow needs the secret *before* it can reach the +> deployment, so it must come from repository secrets. + ### Reading the results Preview deployments are cold and run on shared CI runners, so treat the numbers From d31e00a0a78f792785dd55f6fddd93e0265f2ffe Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 09:19:01 +0930 Subject: [PATCH 03/12] temp roll-back --- .github/workflows/vercel-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 40eb93a..b2ea0e6 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -164,7 +164,7 @@ jobs: uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 with: node-version-file: ${{ inputs.node-version-file }} - package-manager-cache: false + package-manager-cache: true - name: Install Vercel CLI run: npm install --global vercel@latest From f5b6ded2de219d103cb65767f8001ea1204c14a1 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 09:19:15 +0930 Subject: [PATCH 04/12] let performance check install its own dependencies --- .github/workflows/vercel-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index b2ea0e6..40eb93a 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -164,7 +164,7 @@ jobs: uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 with: node-version-file: ${{ inputs.node-version-file }} - package-manager-cache: true + package-manager-cache: false - name: Install Vercel CLI run: npm install --global vercel@latest From 8e3024bbc501cb7c45ce29bef177c93ca5811803 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 10:31:41 +0930 Subject: [PATCH 05/12] add mobile and baseline check --- .github/workflows/vercel-preview.yml | 239 +++++++++++++++++++++++---- 1 file changed, 207 insertions(+), 32 deletions(-) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 40eb93a..8674795 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -40,10 +40,38 @@ on: lighthouse-config-path: description: >- Path to a Lighthouse CI config file in the calling repository, - holding run count and budgets. + holding run count and budgets. Any "{form-factor}" placeholder is + replaced with "desktop" or "mobile", so each form factor can carry + its own budgets. Mobile Lighthouse throttles CPU and network, so its + budgets should be looser than desktop. type: string required: false - default: ".github/lighthouse/lighthouserc.json" + default: ".github/lighthouse/lighthouserc.{form-factor}.json" + baseline-comparison: + description: >- + Compare the measured medians against the most recent baseline + recorded from the default branch and report the delta. Requires the + caller to record baselines on the default branch with + `baseline-record: true`. + type: boolean + required: false + default: false + baseline-record: + description: >- + Record the measured medians as the baseline for later comparison, + rather than commenting on a pull request. Set this on a workflow + triggered by pushes to the default branch. + type: boolean + required: false + default: false + regression-threshold: + description: >- + Percentage a metric may worsen against the baseline before it is + flagged. Preview measurements are noisy, so keep this well above the + run-to-run variance you observe. + type: number + required: false + default: 20 node-version-file: description: "File to read the Node version from for the performance check" type: string @@ -60,7 +88,9 @@ on: required: false concurrency: - group: vercel-preview-${{ github.event.pull_request.number }} + # Falls back to the ref so a baseline-recording run on the default branch, + # which has no pull request number, still gets its own group. + group: vercel-preview-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -108,7 +138,10 @@ jobs: echo "inspect_url=$inspect_url" >> "$GITHUB_OUTPUT" # Posts (or updates in place) a single comment on the PR with the preview URL. + # Skipped when there is no pull request, e.g. a baseline-recording run + # triggered by a push to the default branch. - name: Comment preview URL on PR + if: github.event.pull_request.number env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -141,17 +174,24 @@ jobs: # comments the median results on the pull request. Skipped entirely unless # the caller sets performance-check and provides measured-paths. performance: - name: Preview Performance + name: Preview Performance (${{ matrix.form-factor }}) needs: deploy-preview if: inputs.performance-check && inputs.measured-paths != '' runs-on: ubuntu-latest permissions: contents: read pull-requests: write + strategy: + # Both form factors measure the same deployment independently, so one + # failing should not cancel the other. + fail-fast: false + matrix: + form-factor: [desktop, mobile] env: # How long to wait for the preview deployment to finish building. READY_TIMEOUT: 10m MEASURED_PATHS: ${{ inputs.measured-paths }} + FORM_FACTOR: ${{ matrix.form-factor }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0 @@ -245,6 +285,22 @@ jobs: echo "URLS_EOF" } >> "$GITHUB_OUTPUT" + - name: Resolve Lighthouse config + id: config + env: + CONFIG_TEMPLATE: ${{ inputs.lighthouse-config-path }} + run: | + set -euo pipefail + + path="${CONFIG_TEMPLATE//\{form-factor\}/$FORM_FACTOR}" + if [ ! -f "$path" ]; then + echo "::error::Lighthouse config not found at ${path}." + exit 1 + fi + + echo "Using Lighthouse config ${path}" + echo "path=$path" >> "$GITHUB_OUTPUT" + # Run count and budgets come from the caller's Lighthouse CI config. Set # `aggregationMethod: median` there: LHCI defaults to `optimistic`, which # takes the best run and hides the variance that multiple runs exist to @@ -254,20 +310,37 @@ jobs: uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12.6.1 with: urls: ${{ steps.urls.outputs.list }} - configPath: ${{ inputs.lighthouse-config-path }} + configPath: ${{ steps.config.outputs.path }} uploadArtifacts: true - artifactName: lighthouse-reports + artifactName: lighthouse-reports-${{ matrix.form-factor }} + + # Restores the most recent baseline recorded from the default branch. + # A pull request can read caches written by the default branch, but not + # the reverse, which is why recording runs on a separate trigger. + - name: Restore baseline + if: inputs.baseline-comparison && !inputs.baseline-record + uses: actions/cache/restore@v4 + with: + path: baseline.json + key: lh-baseline-${{ matrix.form-factor }}-${{ github.event.pull_request.base.sha }} + restore-keys: | + lh-baseline-${{ matrix.form-factor }}- - name: Summarise results id: summary env: RESULTS_PATH: ${{ steps.lighthouse.outputs.resultsPath }} ASSERTION_RESULTS: ${{ steps.lighthouse.outputs.assertionResults }} + REGRESSION_THRESHOLD: ${{ inputs.regression-threshold }} + BASELINE_PATH: ${{ inputs.baseline-comparison && 'baseline.json' || '' }} + BASELINE_RECORD: ${{ inputs.baseline-record }} run: | set -euo pipefail + # Written out rather than inlined with `node -e` so the quoting stays + # manageable and the script is readable. cat > summarise.mjs <<'SUMMARISE_EOF' - import { readFileSync } from 'node:fs'; + import { readFileSync, writeFileSync, existsSync } from 'node:fs'; import { join } from 'node:path'; const METRICS = [ @@ -294,20 +367,80 @@ jobs: } }; + // Percentage change from baseline. Positive means worse for every metric here + // (all are "lower is better"). Returns null when there is nothing to compare. + const delta = (current, baseline) => { + if (typeof current !== 'number' || typeof baseline !== 'number') return null; + // A zero baseline has no meaningful percentage. Treat an unchanged zero as + // no change; report any rise from zero as a regression rather than "n/a", + // which would hide a metric going from perfect to non-zero (TBT and CLS + // are commonly zero on a healthy page). + if (baseline === 0) return current === 0 ? 0 : Infinity; + return ((current - baseline) / baseline) * 100; + }; + + const formatDelta = (percent, threshold) => { + if (percent === null) return 'n/a'; + if (percent === Infinity) return 'new 🔺'; + const rounded = Math.round(percent); + if (Math.abs(rounded) < 1) return '±0%'; + const sign = rounded > 0 ? '+' : ''; + const flag = rounded > threshold ? ' 🔺' : ''; + return `${sign}${rounded}%${flag}`; + }; + + const resultsPath = process.env.RESULTS_PATH; + const formFactor = process.env.FORM_FACTOR || 'desktop'; + const threshold = Number(process.env.REGRESSION_THRESHOLD || '20'); + const baselinePath = process.env.BASELINE_PATH || ''; + const recordOnly = process.env.BASELINE_RECORD === 'true'; + const manifest = JSON.parse( - readFileSync(join(process.env.RESULTS_PATH, 'manifest.json'), 'utf8') + readFileSync(join(resultsPath, 'manifest.json'), 'utf8') ); // LHCI writes one manifest entry per run. The entry flagged as the - // representative run is the median run for that URL, so reporting it - // keeps this summary consistent with what the assertions checked. + // representative run is the median run for that URL, so reporting it keeps + // this summary consistent with what the assertions checked. const representative = manifest.filter((entry) => entry.isRepresentativeRun); if (representative.length === 0) { console.error('Manifest contained no representative run.'); process.exit(1); } - // Assertion failures are keyed by URL so each row can be flagged. + // Collect the medians into a plain object, both for the comparison below and + // so the run on the default branch can persist it as the next baseline. + const measured = {}; + for (const entry of representative) { + const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + const url = toLabel(entry.url); + measured[url] = { score: entry.summary?.performance ?? null }; + for (const [id] of METRICS) { + measured[url][id] = report.audits?.[id]?.numericValue ?? null; + } + } + + if (recordOnly) { + writeFileSync( + process.env.BASELINE_OUT || 'baseline.json', + JSON.stringify({ formFactor, measured }, null, 2) + ); + console.error(`Recorded baseline for ${formFactor}.`); + process.exit(0); + } + + // A baseline is optional: the first pull request after this is enabled has + // nothing to compare against, and should still report its measurements. + let baseline = null; + if (baselinePath && existsSync(baselinePath)) { + try { + const parsed = JSON.parse(readFileSync(baselinePath, 'utf8')); + if (parsed.formFactor === formFactor) baseline = parsed.measured; + } catch (error) { + console.error(`Could not read baseline: ${error.message}`); + } + } + const warningsByUrl = new Map(); try { for (const result of JSON.parse(process.env.ASSERTION_RESULTS || '[]')) { @@ -322,31 +455,46 @@ jobs: const lines = []; let anyWarnings = false; + let anyRegressions = false; for (const entry of representative) { - const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + const url = toLabel(entry.url); + const current = measured[url]; + const previous = baseline?.[url]; const warned = warningsByUrl.get(entry.url) ?? new Set(); if (warned.size > 0) anyWarnings = true; - const score = entry.summary?.performance; + const scoreDelta = + typeof previous?.score === 'number' && typeof current.score === 'number' + ? Math.round((current.score - previous.score) * 100) + : null; + lines.push( - `#### \`${toLabel(entry.url)}\``, + `#### \`${url}\``, '', `**Performance score:** ${ - typeof score === 'number' ? Math.round(score * 100) : 'n/a' - }/100${warned.has('categories:performance') ? ' ⚠️' : ''}`, + typeof current.score === 'number' + ? Math.round(current.score * 100) + : 'n/a' + }/100${warned.has('categories:performance') ? ' ⚠️' : ''}${ + scoreDelta === null + ? '' + : ` (${scoreDelta > 0 ? '+' : scoreDelta === 0 ? '±' : ''}${scoreDelta} vs baseline)` + }`, '', - '| Metric | Median | |', - '| --- | --- | --- |' + baseline ? '| Metric | Median | vs baseline | |' : '| Metric | Median | |', + baseline ? '| --- | --- | --- | --- |' : '| --- | --- | --- |' ); for (const [id, label, unit] of METRICS) { - const value = report.audits?.[id]?.numericValue; - lines.push( - `| ${label} | ${format(value, unit)} | ${ - warned.has(id) ? '⚠️' : '✅' - } |` - ); + const value = current[id]; + const percent = previous ? delta(value, previous[id]) : null; + if (percent !== null && percent > threshold) anyRegressions = true; + + const cells = [label, format(value, unit)]; + if (baseline) cells.push(formatDelta(percent, threshold)); + cells.push(warned.has(id) ? '⚠️' : '✅'); + lines.push(`| ${cells.join(' | ')} |`); } lines.push(''); @@ -355,14 +503,30 @@ jobs: const runs = manifest.length / representative.length; lines.push(`_Median of ${runs} run${runs === 1 ? '' : 's'} per URL._`); - // Preview deployments are cold and CPU-shared on CI runners, so treat - // these as a smoke signal for large regressions, not a benchmark. + if (anyRegressions) { + lines.push( + '', + `🔺 One or more metrics are more than ${threshold}% slower than the ` + + 'baseline recorded from the default branch.' + ); + } + + // Preview deployments are cold and CPU-shared on CI runners, so treat these as + // a smoke signal for large regressions, not a benchmark. if (anyWarnings) { lines.push( '', - '⚠️ One or more metrics are over budget. Preview deployments ' + - 'are slower than production — confirm against a second run ' + - 'before treating this as a real regression.' + '⚠️ One or more metrics are over budget. Preview deployments are ' + + 'slower than production — confirm against a second run before ' + + 'treating this as a real regression.' + ); + } + + if (!baseline && baselinePath) { + lines.push( + '', + '_No baseline available yet for this form factor; deltas will appear ' + + 'once the default branch has recorded one._' ); } @@ -370,11 +534,22 @@ jobs: SUMMARISE_EOF node summarise.mjs > lighthouse-summary.md - cat lighthouse-summary.md + if [ "${BASELINE_RECORD}" != "true" ]; then + cat lighthouse-summary.md + fi + + # Persists this run as the baseline for later pull requests. Only the + # default branch writes, so the value a pull request reads always came + # from merged code. + - name: Save baseline + if: inputs.baseline-record + uses: actions/cache/save@v4 + with: + path: baseline.json + key: lh-baseline-${{ matrix.form-factor }}-${{ github.sha }} - # Posts (or updates in place) a single comment on the PR, separate from - # the preview URL comment above. - name: Comment performance results on PR + if: github.event.pull_request.number && !inputs.baseline-record env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} From dec394c8c7603106835f9321b6758f6b1c5891aa Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 12:06:35 +0930 Subject: [PATCH 06/12] Split the performance check into its own reusable workflow The check was an input on the deploy workflow, which meant it could only ever measure a deployment that workflow made, and its cadence was fixed by whatever triggered the deploy. As a separate workflow taking a deployment URL, the caller decides what to measure and when: pair it with a deploy via `needs` on pull requests, call it again on push to the default branch to record a baseline, or point it at a second project. None of that is expressible as an input. vercel-preview.yml goes back to deploying only, losing six inputs and one secret. The two baseline booleans collapse into a `baseline-mode` enum, so the invalid "record and compare" combination is no longer representable. Co-Authored-By: Claude Opus 5 --- .github/workflows/vercel-performance.yml | 478 +++++++++++++++++++++++ .github/workflows/vercel-preview.yml | 475 +--------------------- docs/vercel-performance.md | 213 ++++++++++ docs/vercel-preview.md | 159 +------- 4 files changed, 711 insertions(+), 614 deletions(-) create mode 100644 .github/workflows/vercel-performance.yml create mode 100644 docs/vercel-performance.md diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml new file mode 100644 index 0000000..2f3d5ec --- /dev/null +++ b/.github/workflows/vercel-performance.yml @@ -0,0 +1,478 @@ +name: Vercel Preview Performance + +# Measures Core Web Vitals against an already-deployed Vercel preview and +# either comments the results on a pull request or records them as a baseline. +# +# The caller decides what to measure and when: pair it with a deploy workflow +# via `needs`, and call it again from a push-to-default-branch workflow with +# `baseline-mode: record` to keep the baseline current. + +on: + workflow_call: + inputs: + deployment-url: + description: "URL of the deployment to measure. May still be building." + type: string + required: true + vercel-org-id: + description: "Vercel organisation ID that owns the deployment" + type: string + required: true + measured-paths: + description: >- + Paths to measure, one per line, relative to the deployment URL + (e.g. "/"). Newline-delimited so paths may contain query strings. + type: string + required: true + baseline-mode: + description: >- + "compare" reports each metric against the recorded baseline, + "record" stores this run as the baseline instead of commenting, and + "none" reports the measurements on their own. Recording must run on + the default branch: a pull request can read a cache the default + branch wrote, but not the reverse. + type: string + required: false + default: "none" + lighthouse-config-path: + description: >- + Path to a Lighthouse CI config file in the calling repository, + holding run count and budgets. Any "{form-factor}" placeholder is + replaced with "desktop" or "mobile", so each form factor can carry + its own budgets. Mobile Lighthouse throttles CPU and network, so its + budgets should be looser than desktop. + type: string + required: false + default: ".github/lighthouse/lighthouserc.{form-factor}.json" + regression-threshold: + description: >- + Percentage a metric may worsen against the baseline before it is + flagged. Preview measurements are noisy, so keep this well above the + run-to-run variance you observe. + type: number + required: false + default: 20 + node-version-file: + description: "File to read the Node version from" + type: string + required: false + default: ".nvmrc" + secrets: + vercel-token: + description: "Vercel token, used to wait for the deployment to be ready" + required: true + vercel-automation-bypass-secret: + description: >- + Vercel Protection Bypass for Automation secret. Required when + Deployment Protection is enabled on the project. + required: false + +concurrency: + # Keyed by deployment so two callers measuring different deployments in the + # same pull request do not cancel each other. + group: vercel-performance-${{ inputs.deployment-url }} + cancel-in-progress: true + +jobs: + performance: + name: Preview Performance (${{ matrix.form-factor }}) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + strategy: + # Both form factors measure the same deployment independently, so one + # failing should not cancel the other. + fail-fast: false + matrix: + form-factor: [desktop, mobile] + env: + # How long to wait for the preview deployment to finish building. + READY_TIMEOUT: 10m + MEASURED_PATHS: ${{ inputs.measured-paths }} + FORM_FACTOR: ${{ matrix.form-factor }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0 + with: + persist-credentials: false + + # `package-manager-cache: false` because this job never installs project + # dependencies — it only needs Node to summarise the Lighthouse results. + - name: Install Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 + with: + node-version-file: ${{ inputs.node-version-file }} + package-manager-cache: false + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + # The deploy job uses `--no-wait`, so the deployment is still building + # when this job starts. Wait for it to finish so the performance check can run against a fully built preview. + - name: Wait for preview deployment + id: deployment + env: + VERCEL_TOKEN: ${{ secrets.vercel-token }} + VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} + DEPLOY_URL: ${{ inputs.deployment-url }} + run: | + set -euo pipefail + + if [ -z "$DEPLOY_URL" ]; then + echo "::error::The deploy job did not produce a preview URL." + exit 1 + fi + + echo "Waiting for ${DEPLOY_URL} to become ready." + if ! vercel inspect "$DEPLOY_URL" \ + --token="$VERCEL_TOKEN" \ + --scope="$VERCEL_ORG_ID" \ + --wait \ + --timeout "$READY_TIMEOUT"; then + echo "::error::Preview deployment was not ready within ${READY_TIMEOUT}." + exit 1 + fi + + echo "url=$DEPLOY_URL" >> "$GITHUB_OUTPUT" + + # The first request to a cold preview pays serverless cold start and any + # on-demand page generation. Measuring that would report build latency + # rather than page performance, so discard it. Each path is warmed + # separately because they are generated on demand independently. + - name: Warm up preview deployment + id: urls + env: + BASE_URL: ${{ steps.deployment.outputs.url }} + BYPASS_SECRET: ${{ secrets.vercel-automation-bypass-secret }} + run: | + set -euo pipefail + + failed=0 + urls="" + + # Read line by line: a path may contain `?` and `&`, so splitting on + # whitespace would mangle it. + while IFS= read -r path; do + [ -z "$path" ] && continue + url="${BASE_URL}${path}" + urls="${urls}${url}"$'\n' + + for attempt in 1 2 3; do + status="$(curl -sS -o /dev/null -w '%{http_code}' -L \ + --max-time 60 \ + ${BYPASS_SECRET:+-H "x-vercel-protection-bypass: ${BYPASS_SECRET}"} \ + "$url" || echo 000)" + echo "Warm-up ${url} attempt ${attempt}: HTTP ${status}" + [ "$status" = "200" ] && break + [ "$attempt" = "3" ] && failed=1 + sleep 5 + done + done <<< "$MEASURED_PATHS" + + if [ "$failed" = "1" ]; then + echo "::error::One or more URLs did not return HTTP 200 after 3 warm-up requests." + echo "::error::If Deployment Protection is enabled, pass the vercel-automation-bypass-secret secret." + exit 1 + fi + + # Hand the fully-qualified URLs to the Lighthouse step so the list is + # defined once, by the caller, in measured-paths. + { + echo "list<> "$GITHUB_OUTPUT" + + - name: Resolve Lighthouse config + id: config + env: + CONFIG_TEMPLATE: ${{ inputs.lighthouse-config-path }} + run: | + set -euo pipefail + + path="${CONFIG_TEMPLATE//\{form-factor\}/$FORM_FACTOR}" + if [ ! -f "$path" ]; then + echo "::error::Lighthouse config not found at ${path}." + exit 1 + fi + + echo "Using Lighthouse config ${path}" + echo "path=$path" >> "$GITHUB_OUTPUT" + + # Run count and budgets come from the caller's Lighthouse CI config. Set + # `aggregationMethod: median` there: LHCI defaults to `optimistic`, which + # takes the best run and hides the variance that multiple runs exist to + # smooth out. + - name: Run Lighthouse + id: lighthouse + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12.6.1 + with: + urls: ${{ steps.urls.outputs.list }} + configPath: ${{ steps.config.outputs.path }} + uploadArtifacts: true + artifactName: lighthouse-reports-${{ matrix.form-factor }} + + # Restores the most recent baseline recorded from the default branch. + # A pull request can read caches written by the default branch, but not + # the reverse, which is why recording runs on a separate trigger. + - name: Restore baseline + if: inputs.baseline-mode == 'compare' + uses: actions/cache/restore@v4 + with: + path: baseline.json + key: lh-baseline-${{ matrix.form-factor }}-${{ github.event.pull_request.base.sha }} + restore-keys: | + lh-baseline-${{ matrix.form-factor }}- + + - name: Summarise results + id: summary + env: + RESULTS_PATH: ${{ steps.lighthouse.outputs.resultsPath }} + ASSERTION_RESULTS: ${{ steps.lighthouse.outputs.assertionResults }} + REGRESSION_THRESHOLD: ${{ inputs.regression-threshold }} + BASELINE_PATH: ${{ inputs.baseline-mode == 'compare' && 'baseline.json' || '' }} + BASELINE_RECORD: ${{ inputs.baseline-mode == 'record' }} + run: | + set -euo pipefail + + # Written out rather than inlined with `node -e` so the quoting stays + # manageable and the script is readable. + cat > summarise.mjs <<'SUMMARISE_EOF' + import { readFileSync, writeFileSync, existsSync } from 'node:fs'; + import { join } from 'node:path'; + + const METRICS = [ + ['largest-contentful-paint', 'LCP', 'ms'], + ['total-blocking-time', 'TBT', 'ms'], + ['cumulative-layout-shift', 'CLS', ''], + ['first-contentful-paint', 'FCP', 'ms'], + ['speed-index', 'SI', 'ms'], + ]; + + const format = (value, unit) => { + if (typeof value !== 'number') return 'n/a'; + if (unit !== 'ms') return value.toFixed(3); + return value >= 1000 + ? `${(value / 1000).toFixed(2)} s` + : `${Math.round(value)} ms`; + }; + + const toLabel = (url) => { + try { + return new URL(url).pathname || '/'; + } catch { + return url; + } + }; + + // Percentage change from baseline. Positive means worse for every metric here + // (all are "lower is better"). Returns null when there is nothing to compare. + const delta = (current, baseline) => { + if (typeof current !== 'number' || typeof baseline !== 'number') return null; + // A zero baseline has no meaningful percentage. Treat an unchanged zero as + // no change; report any rise from zero as a regression rather than "n/a", + // which would hide a metric going from perfect to non-zero (TBT and CLS + // are commonly zero on a healthy page). + if (baseline === 0) return current === 0 ? 0 : Infinity; + return ((current - baseline) / baseline) * 100; + }; + + const formatDelta = (percent, threshold) => { + if (percent === null) return 'n/a'; + if (percent === Infinity) return 'new 🔺'; + const rounded = Math.round(percent); + if (Math.abs(rounded) < 1) return '±0%'; + const sign = rounded > 0 ? '+' : ''; + const flag = rounded > threshold ? ' 🔺' : ''; + return `${sign}${rounded}%${flag}`; + }; + + const resultsPath = process.env.RESULTS_PATH; + const formFactor = process.env.FORM_FACTOR || 'desktop'; + const threshold = Number(process.env.REGRESSION_THRESHOLD || '20'); + const baselinePath = process.env.BASELINE_PATH || ''; + const recordOnly = process.env.BASELINE_RECORD === 'true'; + + const manifest = JSON.parse( + readFileSync(join(resultsPath, 'manifest.json'), 'utf8') + ); + + // LHCI writes one manifest entry per run. The entry flagged as the + // representative run is the median run for that URL, so reporting it keeps + // this summary consistent with what the assertions checked. + const representative = manifest.filter((entry) => entry.isRepresentativeRun); + if (representative.length === 0) { + console.error('Manifest contained no representative run.'); + process.exit(1); + } + + // Collect the medians into a plain object, both for the comparison below and + // so the run on the default branch can persist it as the next baseline. + const measured = {}; + for (const entry of representative) { + const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + const url = toLabel(entry.url); + measured[url] = { score: entry.summary?.performance ?? null }; + for (const [id] of METRICS) { + measured[url][id] = report.audits?.[id]?.numericValue ?? null; + } + } + + if (recordOnly) { + writeFileSync( + process.env.BASELINE_OUT || 'baseline.json', + JSON.stringify({ formFactor, measured }, null, 2) + ); + console.error(`Recorded baseline for ${formFactor}.`); + process.exit(0); + } + + // A baseline is optional: the first pull request after this is enabled has + // nothing to compare against, and should still report its measurements. + let baseline = null; + if (baselinePath && existsSync(baselinePath)) { + try { + const parsed = JSON.parse(readFileSync(baselinePath, 'utf8')); + if (parsed.formFactor === formFactor) baseline = parsed.measured; + } catch (error) { + console.error(`Could not read baseline: ${error.message}`); + } + } + + const warningsByUrl = new Map(); + try { + for (const result of JSON.parse(process.env.ASSERTION_RESULTS || '[]')) { + if (!result.url) continue; + const existing = warningsByUrl.get(result.url) ?? new Set(); + existing.add(result.auditId); + warningsByUrl.set(result.url, existing); + } + } catch (error) { + console.error(`Could not parse assertion results: ${error.message}`); + } + + const lines = []; + let anyWarnings = false; + let anyRegressions = false; + + for (const entry of representative) { + const url = toLabel(entry.url); + const current = measured[url]; + const previous = baseline?.[url]; + const warned = warningsByUrl.get(entry.url) ?? new Set(); + if (warned.size > 0) anyWarnings = true; + + const scoreDelta = + typeof previous?.score === 'number' && typeof current.score === 'number' + ? Math.round((current.score - previous.score) * 100) + : null; + + lines.push( + `#### \`${url}\``, + '', + `**Performance score:** ${ + typeof current.score === 'number' + ? Math.round(current.score * 100) + : 'n/a' + }/100${warned.has('categories:performance') ? ' ⚠️' : ''}${ + scoreDelta === null + ? '' + : ` (${scoreDelta > 0 ? '+' : scoreDelta === 0 ? '±' : ''}${scoreDelta} vs baseline)` + }`, + '', + baseline ? '| Metric | Median | vs baseline | |' : '| Metric | Median | |', + baseline ? '| --- | --- | --- | --- |' : '| --- | --- | --- |' + ); + + for (const [id, label, unit] of METRICS) { + const value = current[id]; + const percent = previous ? delta(value, previous[id]) : null; + if (percent !== null && percent > threshold) anyRegressions = true; + + const cells = [label, format(value, unit)]; + if (baseline) cells.push(formatDelta(percent, threshold)); + cells.push(warned.has(id) ? '⚠️' : '✅'); + lines.push(`| ${cells.join(' | ')} |`); + } + + lines.push(''); + } + + const runs = manifest.length / representative.length; + lines.push(`_Median of ${runs} run${runs === 1 ? '' : 's'} per URL._`); + + if (anyRegressions) { + lines.push( + '', + `🔺 One or more metrics are more than ${threshold}% slower than the ` + + 'baseline recorded from the default branch.' + ); + } + + // Preview deployments are cold and CPU-shared on CI runners, so treat these as + // a smoke signal for large regressions, not a benchmark. + if (anyWarnings) { + lines.push( + '', + '⚠️ One or more metrics are over budget. Preview deployments are ' + + 'slower than production — confirm against a second run before ' + + 'treating this as a real regression.' + ); + } + + if (!baseline && baselinePath) { + lines.push( + '', + '_No baseline available yet for this form factor; deltas will appear ' + + 'once the default branch has recorded one._' + ); + } + + console.log(lines.join('\n')); + SUMMARISE_EOF + + node summarise.mjs > lighthouse-summary.md + if [ "${BASELINE_RECORD}" != "true" ]; then + cat lighthouse-summary.md + fi + + # Persists this run as the baseline for later pull requests. Only the + # default branch writes, so the value a pull request reads always came + # from merged code. + - name: Save baseline + if: inputs.baseline-mode == 'record' + uses: actions/cache/save@v4 + with: + path: baseline.json + key: lh-baseline-${{ matrix.form-factor }}-${{ github.sha }} + + - name: Comment performance results on PR + if: github.event.pull_request.number && inputs.baseline-mode != 'record' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + marker='' + { + echo "$marker" + echo "### 📊 Preview performance" + echo + cat lighthouse-summary.md + echo + echo "[Full reports](${RUN_URL}) · _commit ${COMMIT_SHA}_" + } > comment.md + + comment_id="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq "[.[] | select(.body | startswith(\"${marker}\"))][0].id")" + + if [ -n "$comment_id" ] && [ "$comment_id" != "null" ]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body=@comment.md + else + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@comment.md + fi diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 8674795..5047f1a 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -21,74 +21,13 @@ on: type: string required: false default: "Preview" - performance-check: - description: >- - Run a Lighthouse performance check against the preview and comment - the median Core Web Vitals on the pull request. Requires - measured-paths to be set. - type: boolean - required: false - default: false - measured-paths: - description: >- - Paths to measure, one per line, relative to the preview URL - (e.g. "/"). Newline-delimited so paths may contain query strings. - The performance check is skipped when this is empty. - type: string - required: false - default: "" - lighthouse-config-path: - description: >- - Path to a Lighthouse CI config file in the calling repository, - holding run count and budgets. Any "{form-factor}" placeholder is - replaced with "desktop" or "mobile", so each form factor can carry - its own budgets. Mobile Lighthouse throttles CPU and network, so its - budgets should be looser than desktop. - type: string - required: false - default: ".github/lighthouse/lighthouserc.{form-factor}.json" - baseline-comparison: - description: >- - Compare the measured medians against the most recent baseline - recorded from the default branch and report the delta. Requires the - caller to record baselines on the default branch with - `baseline-record: true`. - type: boolean - required: false - default: false - baseline-record: - description: >- - Record the measured medians as the baseline for later comparison, - rather than commenting on a pull request. Set this on a workflow - triggered by pushes to the default branch. - type: boolean - required: false - default: false - regression-threshold: - description: >- - Percentage a metric may worsen against the baseline before it is - flagged. Preview measurements are noisy, so keep this well above the - run-to-run variance you observe. - type: number - required: false - default: 20 - node-version-file: - description: "File to read the Node version from for the performance check" - type: string - required: false - default: ".nvmrc" secrets: vercel-token: description: "Vercel deployment token" required: true - vercel-automation-bypass-secret: - description: >- - Vercel Protection Bypass for Automation secret. Required for the - performance check when Deployment Protection is enabled. - required: false concurrency: - # Falls back to the ref so a baseline-recording run on the default branch, + # Falls back to the ref so a run on a branch rather than a pull request, # which has no pull request number, still gets its own group. group: vercel-preview-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -138,8 +77,8 @@ jobs: echo "inspect_url=$inspect_url" >> "$GITHUB_OUTPUT" # Posts (or updates in place) a single comment on the PR with the preview URL. - # Skipped when there is no pull request, e.g. a baseline-recording run - # triggered by a push to the default branch. + # Skipped when there is no pull request, e.g. a deploy triggered by a + # push to the default branch. - name: Comment preview URL on PR if: github.event.pull_request.number env: @@ -169,411 +108,3 @@ jobs: else gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$body" fi - - # Opt-in: measures Core Web Vitals against the preview deployment and - # comments the median results on the pull request. Skipped entirely unless - # the caller sets performance-check and provides measured-paths. - performance: - name: Preview Performance (${{ matrix.form-factor }}) - needs: deploy-preview - if: inputs.performance-check && inputs.measured-paths != '' - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - strategy: - # Both form factors measure the same deployment independently, so one - # failing should not cancel the other. - fail-fast: false - matrix: - form-factor: [desktop, mobile] - env: - # How long to wait for the preview deployment to finish building. - READY_TIMEOUT: 10m - MEASURED_PATHS: ${{ inputs.measured-paths }} - FORM_FACTOR: ${{ matrix.form-factor }} - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0 - with: - persist-credentials: false - - # `package-manager-cache: false` because this job never installs project - # dependencies — it only needs Node to summarise the Lighthouse results. - - name: Install Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 - with: - node-version-file: ${{ inputs.node-version-file }} - package-manager-cache: false - - - name: Install Vercel CLI - run: npm install --global vercel@latest - - # The deploy job uses `--no-wait`, so the deployment is still building - # when this job starts. Wait for it to finish so the performance check can run against a fully built preview. - - name: Wait for preview deployment - id: deployment - env: - VERCEL_TOKEN: ${{ secrets.vercel-token }} - VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} - DEPLOY_URL: ${{ needs.deploy-preview.outputs.url }} - run: | - set -euo pipefail - - if [ -z "$DEPLOY_URL" ]; then - echo "::error::The deploy job did not produce a preview URL." - exit 1 - fi - - echo "Waiting for ${DEPLOY_URL} to become ready." - if ! vercel inspect "$DEPLOY_URL" \ - --token="$VERCEL_TOKEN" \ - --scope="$VERCEL_ORG_ID" \ - --wait \ - --timeout "$READY_TIMEOUT"; then - echo "::error::Preview deployment was not ready within ${READY_TIMEOUT}." - exit 1 - fi - - echo "url=$DEPLOY_URL" >> "$GITHUB_OUTPUT" - - # The first request to a cold preview pays serverless cold start and any - # on-demand page generation. Measuring that would report build latency - # rather than page performance, so discard it. Each path is warmed - # separately because they are generated on demand independently. - - name: Warm up preview deployment - id: urls - env: - BASE_URL: ${{ steps.deployment.outputs.url }} - BYPASS_SECRET: ${{ secrets.vercel-automation-bypass-secret }} - run: | - set -euo pipefail - - failed=0 - urls="" - - # Read line by line: a path may contain `?` and `&`, so splitting on - # whitespace would mangle it. - while IFS= read -r path; do - [ -z "$path" ] && continue - url="${BASE_URL}${path}" - urls="${urls}${url}"$'\n' - - for attempt in 1 2 3; do - status="$(curl -sS -o /dev/null -w '%{http_code}' -L \ - --max-time 60 \ - ${BYPASS_SECRET:+-H "x-vercel-protection-bypass: ${BYPASS_SECRET}"} \ - "$url" || echo 000)" - echo "Warm-up ${url} attempt ${attempt}: HTTP ${status}" - [ "$status" = "200" ] && break - [ "$attempt" = "3" ] && failed=1 - sleep 5 - done - done <<< "$MEASURED_PATHS" - - if [ "$failed" = "1" ]; then - echo "::error::One or more URLs did not return HTTP 200 after 3 warm-up requests." - echo "::error::If Deployment Protection is enabled, pass the vercel-automation-bypass-secret secret." - exit 1 - fi - - # Hand the fully-qualified URLs to the Lighthouse step so the list is - # defined once, by the caller, in measured-paths. - { - echo "list<> "$GITHUB_OUTPUT" - - - name: Resolve Lighthouse config - id: config - env: - CONFIG_TEMPLATE: ${{ inputs.lighthouse-config-path }} - run: | - set -euo pipefail - - path="${CONFIG_TEMPLATE//\{form-factor\}/$FORM_FACTOR}" - if [ ! -f "$path" ]; then - echo "::error::Lighthouse config not found at ${path}." - exit 1 - fi - - echo "Using Lighthouse config ${path}" - echo "path=$path" >> "$GITHUB_OUTPUT" - - # Run count and budgets come from the caller's Lighthouse CI config. Set - # `aggregationMethod: median` there: LHCI defaults to `optimistic`, which - # takes the best run and hides the variance that multiple runs exist to - # smooth out. - - name: Run Lighthouse - id: lighthouse - uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12.6.1 - with: - urls: ${{ steps.urls.outputs.list }} - configPath: ${{ steps.config.outputs.path }} - uploadArtifacts: true - artifactName: lighthouse-reports-${{ matrix.form-factor }} - - # Restores the most recent baseline recorded from the default branch. - # A pull request can read caches written by the default branch, but not - # the reverse, which is why recording runs on a separate trigger. - - name: Restore baseline - if: inputs.baseline-comparison && !inputs.baseline-record - uses: actions/cache/restore@v4 - with: - path: baseline.json - key: lh-baseline-${{ matrix.form-factor }}-${{ github.event.pull_request.base.sha }} - restore-keys: | - lh-baseline-${{ matrix.form-factor }}- - - - name: Summarise results - id: summary - env: - RESULTS_PATH: ${{ steps.lighthouse.outputs.resultsPath }} - ASSERTION_RESULTS: ${{ steps.lighthouse.outputs.assertionResults }} - REGRESSION_THRESHOLD: ${{ inputs.regression-threshold }} - BASELINE_PATH: ${{ inputs.baseline-comparison && 'baseline.json' || '' }} - BASELINE_RECORD: ${{ inputs.baseline-record }} - run: | - set -euo pipefail - - # Written out rather than inlined with `node -e` so the quoting stays - # manageable and the script is readable. - cat > summarise.mjs <<'SUMMARISE_EOF' - import { readFileSync, writeFileSync, existsSync } from 'node:fs'; - import { join } from 'node:path'; - - const METRICS = [ - ['largest-contentful-paint', 'LCP', 'ms'], - ['total-blocking-time', 'TBT', 'ms'], - ['cumulative-layout-shift', 'CLS', ''], - ['first-contentful-paint', 'FCP', 'ms'], - ['speed-index', 'SI', 'ms'], - ]; - - const format = (value, unit) => { - if (typeof value !== 'number') return 'n/a'; - if (unit !== 'ms') return value.toFixed(3); - return value >= 1000 - ? `${(value / 1000).toFixed(2)} s` - : `${Math.round(value)} ms`; - }; - - const toLabel = (url) => { - try { - return new URL(url).pathname || '/'; - } catch { - return url; - } - }; - - // Percentage change from baseline. Positive means worse for every metric here - // (all are "lower is better"). Returns null when there is nothing to compare. - const delta = (current, baseline) => { - if (typeof current !== 'number' || typeof baseline !== 'number') return null; - // A zero baseline has no meaningful percentage. Treat an unchanged zero as - // no change; report any rise from zero as a regression rather than "n/a", - // which would hide a metric going from perfect to non-zero (TBT and CLS - // are commonly zero on a healthy page). - if (baseline === 0) return current === 0 ? 0 : Infinity; - return ((current - baseline) / baseline) * 100; - }; - - const formatDelta = (percent, threshold) => { - if (percent === null) return 'n/a'; - if (percent === Infinity) return 'new 🔺'; - const rounded = Math.round(percent); - if (Math.abs(rounded) < 1) return '±0%'; - const sign = rounded > 0 ? '+' : ''; - const flag = rounded > threshold ? ' 🔺' : ''; - return `${sign}${rounded}%${flag}`; - }; - - const resultsPath = process.env.RESULTS_PATH; - const formFactor = process.env.FORM_FACTOR || 'desktop'; - const threshold = Number(process.env.REGRESSION_THRESHOLD || '20'); - const baselinePath = process.env.BASELINE_PATH || ''; - const recordOnly = process.env.BASELINE_RECORD === 'true'; - - const manifest = JSON.parse( - readFileSync(join(resultsPath, 'manifest.json'), 'utf8') - ); - - // LHCI writes one manifest entry per run. The entry flagged as the - // representative run is the median run for that URL, so reporting it keeps - // this summary consistent with what the assertions checked. - const representative = manifest.filter((entry) => entry.isRepresentativeRun); - if (representative.length === 0) { - console.error('Manifest contained no representative run.'); - process.exit(1); - } - - // Collect the medians into a plain object, both for the comparison below and - // so the run on the default branch can persist it as the next baseline. - const measured = {}; - for (const entry of representative) { - const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); - const url = toLabel(entry.url); - measured[url] = { score: entry.summary?.performance ?? null }; - for (const [id] of METRICS) { - measured[url][id] = report.audits?.[id]?.numericValue ?? null; - } - } - - if (recordOnly) { - writeFileSync( - process.env.BASELINE_OUT || 'baseline.json', - JSON.stringify({ formFactor, measured }, null, 2) - ); - console.error(`Recorded baseline for ${formFactor}.`); - process.exit(0); - } - - // A baseline is optional: the first pull request after this is enabled has - // nothing to compare against, and should still report its measurements. - let baseline = null; - if (baselinePath && existsSync(baselinePath)) { - try { - const parsed = JSON.parse(readFileSync(baselinePath, 'utf8')); - if (parsed.formFactor === formFactor) baseline = parsed.measured; - } catch (error) { - console.error(`Could not read baseline: ${error.message}`); - } - } - - const warningsByUrl = new Map(); - try { - for (const result of JSON.parse(process.env.ASSERTION_RESULTS || '[]')) { - if (!result.url) continue; - const existing = warningsByUrl.get(result.url) ?? new Set(); - existing.add(result.auditId); - warningsByUrl.set(result.url, existing); - } - } catch (error) { - console.error(`Could not parse assertion results: ${error.message}`); - } - - const lines = []; - let anyWarnings = false; - let anyRegressions = false; - - for (const entry of representative) { - const url = toLabel(entry.url); - const current = measured[url]; - const previous = baseline?.[url]; - const warned = warningsByUrl.get(entry.url) ?? new Set(); - if (warned.size > 0) anyWarnings = true; - - const scoreDelta = - typeof previous?.score === 'number' && typeof current.score === 'number' - ? Math.round((current.score - previous.score) * 100) - : null; - - lines.push( - `#### \`${url}\``, - '', - `**Performance score:** ${ - typeof current.score === 'number' - ? Math.round(current.score * 100) - : 'n/a' - }/100${warned.has('categories:performance') ? ' ⚠️' : ''}${ - scoreDelta === null - ? '' - : ` (${scoreDelta > 0 ? '+' : scoreDelta === 0 ? '±' : ''}${scoreDelta} vs baseline)` - }`, - '', - baseline ? '| Metric | Median | vs baseline | |' : '| Metric | Median | |', - baseline ? '| --- | --- | --- | --- |' : '| --- | --- | --- |' - ); - - for (const [id, label, unit] of METRICS) { - const value = current[id]; - const percent = previous ? delta(value, previous[id]) : null; - if (percent !== null && percent > threshold) anyRegressions = true; - - const cells = [label, format(value, unit)]; - if (baseline) cells.push(formatDelta(percent, threshold)); - cells.push(warned.has(id) ? '⚠️' : '✅'); - lines.push(`| ${cells.join(' | ')} |`); - } - - lines.push(''); - } - - const runs = manifest.length / representative.length; - lines.push(`_Median of ${runs} run${runs === 1 ? '' : 's'} per URL._`); - - if (anyRegressions) { - lines.push( - '', - `🔺 One or more metrics are more than ${threshold}% slower than the ` + - 'baseline recorded from the default branch.' - ); - } - - // Preview deployments are cold and CPU-shared on CI runners, so treat these as - // a smoke signal for large regressions, not a benchmark. - if (anyWarnings) { - lines.push( - '', - '⚠️ One or more metrics are over budget. Preview deployments are ' + - 'slower than production — confirm against a second run before ' + - 'treating this as a real regression.' - ); - } - - if (!baseline && baselinePath) { - lines.push( - '', - '_No baseline available yet for this form factor; deltas will appear ' + - 'once the default branch has recorded one._' - ); - } - - console.log(lines.join('\n')); - SUMMARISE_EOF - - node summarise.mjs > lighthouse-summary.md - if [ "${BASELINE_RECORD}" != "true" ]; then - cat lighthouse-summary.md - fi - - # Persists this run as the baseline for later pull requests. Only the - # default branch writes, so the value a pull request reads always came - # from merged code. - - name: Save baseline - if: inputs.baseline-record - uses: actions/cache/save@v4 - with: - path: baseline.json - key: lh-baseline-${{ matrix.form-factor }}-${{ github.sha }} - - - name: Comment performance results on PR - if: github.event.pull_request.number && !inputs.baseline-record - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - COMMIT_SHA: ${{ github.event.pull_request.head.sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -euo pipefail - - marker='' - { - echo "$marker" - echo "### 📊 Preview performance" - echo - cat lighthouse-summary.md - echo - echo "[Full reports](${RUN_URL}) · _commit ${COMMIT_SHA}_" - } > comment.md - - comment_id="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ - --jq "[.[] | select(.body | startswith(\"${marker}\"))][0].id")" - - if [ -n "$comment_id" ] && [ "$comment_id" != "null" ]; then - gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body=@comment.md - else - gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@comment.md - fi diff --git a/docs/vercel-performance.md b/docs/vercel-performance.md new file mode 100644 index 0000000..6ebcfc1 --- /dev/null +++ b/docs/vercel-performance.md @@ -0,0 +1,213 @@ +# Vercel Preview Performance + +Measures Core Web Vitals against an already-deployed Vercel preview with +[Lighthouse CI](https://github.com/treosh/lighthouse-ci-action) and posts the +median results as a pull request comment. + +Desktop and mobile run concurrently as a matrix, each against its own budgets. + +The caller decides what to measure and when: pair it with +[Vercel Preview Deployment](vercel-preview.md) via `needs`, and call it again +from a push-to-default-branch workflow with `baseline-mode: record` to keep the +baseline current. + +#### **Inputs** +| Name | Required | Type | Default | Description | +|------------------------|----------|--------|-----------------------------------------------|------------------------------------------------| +| deployment-url | ✅ | string | | Deployment to measure. May still be building. | +| vercel-org-id | ✅ | string | | Vercel organisation ID owning the deployment | +| measured-paths | ✅ | string | | Paths to measure, one per line | +| baseline-mode | ❌ | string | none | `compare`, `record` or `none` | +| lighthouse-config-path | ❌ | string | .github/lighthouse/lighthouserc.{form-factor}.json | Lighthouse CI config in the caller repo | +| regression-threshold | ❌ | number | 20 | Percent a metric may worsen before flagging | +| node-version-file | ❌ | string | .nvmrc | File to read the Node version from | + +#### **Secrets** +| Name | Required | Description | +|---------------------------------|----------|----------------------------------------------------------| +| vercel-token | ✅ | Used to wait for the deployment to be ready | +| vercel-automation-bypass-secret | ❌ | Protection Bypass for Automation, if protection is on | + +### How it works + +1. Waits for the deployment to finish building, so the caller can deploy with + `--no-wait` and hand over the URL immediately. +2. Warms each measured path. The first request to a cold preview pays + serverless cold start and any on-demand page generation, which would + otherwise be measured as page latency. +3. Runs Lighthouse against every path, for desktop and mobile, and reports the + median of each metric. + +### Baseline comparison + +Absolute budgets cannot tell "this pull request made things worse" from "this +runner was busy". A regression from 1.2s to 3.5s passes a 4s budget silently. +Comparing against the default branch isolates what the change actually did. + +Recording must run on the **default branch**: a pull request can read a cache +the default branch wrote, but not the reverse. So the two modes live in two +caller workflows — `compare` on `pull_request`, `record` on push to the default +branch. + +Note that a cache expires after 7 days without being read. After a quiet period +the first pull request reports no baseline until the next merge records one. + +### Example Usage + +Measuring a pull request, comparing against the baseline: + +```yaml +on: + pull_request: + +jobs: + deploy-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + + performance: + needs: deploy-preview + uses: aligent/workflows/.github/workflows/vercel-performance.yml@main + with: + deployment-url: ${{ needs.deploy-preview.outputs.url }} + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + baseline-mode: compare + measured-paths: | + / + /category/example + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} +``` + +Recording the baseline after a merge. The measured paths must match those above, +or pull requests will have nothing to compare against: + +```yaml +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + deploy-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + + record-baseline: + needs: deploy-preview + uses: aligent/workflows/.github/workflows/vercel-performance.yml@main + with: + deployment-url: ${{ needs.deploy-preview.outputs.url }} + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + baseline-mode: record + measured-paths: | + / + /category/example + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} +``` + +The baseline is deliberately a **preview built from the default branch**, not a +production deployment. Production is warm, CDN-cached, and often points at +different data and environment variables, so a preview measured against it would +show deltas caused by the environment rather than by the change. + +### Lighthouse config + +The caller supplies one config per form factor. `{form-factor}` in the path is +replaced with `desktop` or `mobile`: + +```json +{ + "ci": { + "collect": { + "numberOfRuns": 3, + "settings": { + "preset": "desktop", + "onlyCategories": ["performance"], + "maxWaitForLoad": 60000, + "chromeFlags": "--no-sandbox --disable-dev-shm-usage" + } + }, + "assert": { + "aggregationMethod": "median", + "assertions": { + "categories:performance": ["warn", { "minScore": 0.5 }], + "largest-contentful-paint": ["warn", { "maxNumericValue": 4000 }], + "total-blocking-time": ["warn", { "maxNumericValue": 600 }], + "cumulative-layout-shift": ["warn", { "maxNumericValue": 0.25 }], + "first-contentful-paint": ["warn", { "maxNumericValue": 3000 }], + "speed-index": ["warn", { "maxNumericValue": 5800 }] + } + } + } +} +``` + +The mobile config sets `"formFactor": "mobile"` instead of `"preset": "desktop"`, +which applies mobile emulation and a 4x CPU slowdown. Its budgets should be +roughly 1.5x desktop as a result; CLS is unchanged, as layout shift is not +throttling-dependent. + +Three things in that config are worth calling out: + +- **`aggregationMethod: median`** — Lighthouse CI defaults to `optimistic`, + which takes the *best* run and so discards the variance that multiple runs + exist to smooth out. Set this explicitly. +- **`numberOfRuns: 3`** — a single run against a cold preview is not a usable + signal. An odd count means the median is a value that was actually measured. +- **`warn` rather than `error`** — the job reports breaches without failing, so + a noisy preview cannot block a merge. Switch to `error` once you have enough + history to trust the thresholds. + +### Reading the results + +Preview deployments are cold and run on shared CI runners, so treat the numbers +as a smoke signal for large regressions rather than a benchmark. Keep budgets +loose: scoring a preview against production-grade thresholds flags nearly every +pull request, and a check that cries wolf gets ignored. + +### Deployment Protection + +If the Vercel project has [Deployment +Protection](https://vercel.com/docs/deployment-protection) enabled, preview URLs +require a Vercel login. Automated requests receive an authentication page rather +than the site, so Lighthouse would score the login screen instead of the page +under test. + +To allow the check through, generate a [Protection Bypass for +Automation](https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/protection-bypass-automation) +secret in the Vercel project's Deployment Protection settings, add it to the +calling repository as a secret, and pass it as +`vercel-automation-bypass-secret`. The workflow sends it as the +`x-vercel-protection-bypass` header on both the warm-up requests and the +Lighthouse runs. + +Generating the secret requires at least the **member** team role, or the +**Project Administrator** role on the project. Note that regenerating or +deleting a secret invalidates it for existing deployments, which then need to +be redeployed. + +The secret is optional and is omitted from requests entirely when unset, which +is correct for a project without Deployment Protection. If protection *is* +enabled and the secret is missing, the warm-up step fails with a non-200 status +rather than reporting misleading scores. + +To check whether a project needs it, open a preview URL in a private browser +window: a login prompt means protection is enabled. + +> Vercel also exposes this value to the running deployment as the +> `VERCEL_AUTOMATION_BYPASS_SECRET` system environment variable, but that is not +> usable here — the workflow needs the secret *before* it can reach the +> deployment, so it must come from repository secrets. diff --git a/docs/vercel-preview.md b/docs/vercel-preview.md index 393988b..4ddaba6 100644 --- a/docs/vercel-preview.md +++ b/docs/vercel-preview.md @@ -5,22 +5,22 @@ pull request with the preview and inspect URLs. Intended to be called from a `pull_request` triggered workflow. #### **Inputs** -| Name | Required | Type | Default | Description | -|------------------------|----------|---------|------------------------------------------|-----------------------------------------| -| vercel-org-id | ✅ | string | | Vercel organisation ID | -| vercel-project-id | ✅ | string | | Vercel project ID | -| working-directory | ❌ | string | . | Directory to run the Vercel deploy from | -| environment-name | ❌ | string | Preview | GitHub Environment to deploy to | -| performance-check | ❌ | boolean | false | Run a Lighthouse check against the preview | -| measured-paths | ❌ | string | | Paths to measure, one per line | -| lighthouse-config-path | ❌ | string | .github/lighthouse/lighthouserc.json | Lighthouse CI config in the caller repo | -| node-version-file | ❌ | string | .nvmrc | Node version file for the check | +| Name | Required | Type | Default | Description | +|--------------------|----------|--------|----------|-----------------------------------------| +| vercel-org-id | ✅ | string | | Vercel organisation ID | +| vercel-project-id | ✅ | string | | Vercel project ID | +| working-directory | ❌ | string | . | Directory to run the Vercel deploy from | +| environment-name | ❌ | string | Preview | GitHub Environment to deploy to | + +#### **Outputs** +| Name | Description | +|------|-------------------------------------------------------------------------| +| url | The preview deployment URL. Still building when the job ends, as the deploy uses `--no-wait`. | #### **Secrets** -| Name | Required | Description | -|---------------------------------|----------|------------------------------------------------------| -| vercel-token | ✅ | Vercel deployment token | -| vercel-automation-bypass-secret | ❌ | Protection Bypass for Automation, for the perf check | +| Name | Required | Description | +|---------------|----------|--------------------------| +| vercel-token | ✅ | Vercel deployment token | #### Example Usage @@ -40,131 +40,6 @@ jobs: vercel-token: ${{ secrets.VERCEL_TOKEN }} ``` -## Performance Check - -Optionally measures Core Web Vitals against the preview deployment with -[Lighthouse CI](https://github.com/treosh/lighthouse-ci-action) and posts the -median results as a pull request comment. - -The check is **off by default**. It runs only when `performance-check` is true -*and* `measured-paths` is non-empty, so existing callers are unaffected. - -### How it works - -1. Waits for the preview deployment to finish building (the deploy step uses - `--no-wait`, so the deploy job itself returns as soon as it has a URL). -2. Warms each measured path. The first request to a cold preview pays - serverless cold start and any on-demand page generation, which would - otherwise be measured as page latency. -3. Runs Lighthouse against every path and comments the median of each metric. - -### Deployment Protection - -If the Vercel project has [Deployment -Protection](https://vercel.com/docs/deployment-protection) enabled, preview URLs -require a Vercel login. Automated requests receive an authentication page rather -than the site, so Lighthouse would score the login screen instead of the page -under test. - -To allow the check through, generate a [Protection Bypass for -Automation](https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/protection-bypass-automation) -secret in the Vercel project's Deployment Protection settings, add it to the -calling repository as a secret, and pass it as -`vercel-automation-bypass-secret`. The workflow sends it as the -`x-vercel-protection-bypass` header on both the warm-up requests and the -Lighthouse runs. - -Generating the secret requires at least the **member** team role, or the -**Project Administrator** role on the project. Note that regenerating or -deleting a secret invalidates it for existing deployments, which then need to -be redeployed. - -The secret is optional and is omitted from requests entirely when unset, which -is correct for a project without Deployment Protection. If protection *is* -enabled and the secret is missing, the warm-up step fails with a non-200 status -rather than reporting misleading scores. - -To check whether a project needs it, open a preview URL in a private browser -window: a login prompt means protection is enabled. - -> Vercel also exposes this value to the running deployment as the -> `VERCEL_AUTOMATION_BYPASS_SECRET` system environment variable, but that is not -> usable here — the workflow needs the secret *before* it can reach the -> deployment, so it must come from repository secrets. - -### Reading the results - -Preview deployments are cold and run on shared CI runners, so treat the numbers -as a smoke signal for large regressions rather than a benchmark. Two things -follow from that: - -- **Use several runs and aggregate on the median.** A single run on a cold - preview is not a usable signal. -- **Keep budgets loose.** Scoring a preview against production-grade - thresholds flags nearly every pull request, and a check that cries wolf gets - ignored. - -### Example Usage - -```yaml -on: - pull_request: - branches: - - main - -jobs: - deploy-preview: - uses: aligent/workflows/.github/workflows/vercel-preview.yml@main - with: - vercel-org-id: ${{ vars.VERCEL_ORG_ID }} - vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} - performance-check: true - # One path per line. Paths may contain query strings. - measured-paths: | - / - /women/tops-women/jackets-women.html - /stellar-solar-jacket.html?categoryPath=jackets-women - secrets: - vercel-token: ${{ secrets.VERCEL_TOKEN }} - # Only needed when Deployment Protection is enabled on the project. - vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} -``` - -The caller supplies the Lighthouse CI config, by default at -`.github/lighthouse/lighthouserc.json`: - -```json -{ - "ci": { - "collect": { - "numberOfRuns": 3, - "settings": { - "preset": "desktop", - "onlyCategories": ["performance"], - "maxWaitForLoad": 60000, - "chromeFlags": "--no-sandbox --disable-dev-shm-usage" - } - }, - "assert": { - "aggregationMethod": "median", - "assertions": { - "categories:performance": ["warn", { "minScore": 0.5 }], - "largest-contentful-paint": ["warn", { "maxNumericValue": 4000 }], - "total-blocking-time": ["warn", { "maxNumericValue": 600 }], - "cumulative-layout-shift": ["warn", { "maxNumericValue": 0.25 }], - "first-contentful-paint": ["warn", { "maxNumericValue": 3000 }], - "speed-index": ["warn", { "maxNumericValue": 5800 }] - } - } - } -} -``` - -Two things in that config are worth calling out: - -- **`aggregationMethod: median`** — Lighthouse CI defaults to `optimistic`, - which takes the *best* run and so discards the variance that multiple runs - exist to smooth out. Set this explicitly. -- **`warn` rather than `error`** — the job reports breaches without failing, so - a noisy preview cannot block a merge. Switch to `error` once you have enough - history to trust the thresholds. +To measure the performance of the resulting preview, pair this with +[Vercel Preview Performance](vercel-performance.md), which takes the `url` +output above. From 11999e08a653f46a44169328d5ca7ece868d528b Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 12:45:10 +0930 Subject: [PATCH 07/12] Scope baselines by key and add a keep-alive workflow Baseline caches were keyed only by form factor, so a repository measuring two deployments would have them overwrite each other and pull requests would compare against whichever recorded last. A `baseline-key` input now namespaces them; it defaults to "default", so existing single-deployment callers are unaffected. Adds vercel-performance-keepalive.yml. GitHub evicts a cache that has not been read for 7 days, and the baseline is only rewritten on a merge, so a quiet fortnight would drop it silently. Reading a cache resets that clock, so the workflow restores the baselines and does nothing else, costing seconds rather than the minutes a re-measurement would. It expands the caller's baseline keys across the form factors this repository measures, so callers do not duplicate that list. A full restore is used rather than `lookup-only`, because whether a metadata-only lookup resets the eviction clock is not documented, and the payload is a few hundred bytes either way. Co-Authored-By: Claude Opus 5 --- .../vercel-performance-keepalive.yml | 105 ++++++++++++++++++ .github/workflows/vercel-performance.yml | 19 +++- docs/vercel-performance.md | 44 +++++++- 3 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/vercel-performance-keepalive.yml diff --git a/.github/workflows/vercel-performance-keepalive.yml b/.github/workflows/vercel-performance-keepalive.yml new file mode 100644 index 0000000..56cbad7 --- /dev/null +++ b/.github/workflows/vercel-performance-keepalive.yml @@ -0,0 +1,105 @@ +name: Vercel Performance Baseline Keep-Alive + +# GitHub evicts a cache that has not been read for 7 days. A performance +# baseline is only rewritten when something merges to the default branch, so a +# quiet fortnight would silently drop it and pull requests would report no +# baseline until the next merge. +# +# Reading a cache resets that clock, so this restores the baselines and does +# nothing else. It deploys nothing and measures nothing, costing seconds rather +# than the minutes a re-measurement would. +# +# The form factors are declared here rather than by the caller so they stay in +# step with the matrix in vercel-performance.yml. + +on: + workflow_call: + inputs: + baseline-keys: + description: >- + Baseline keys to keep alive, one per line, matching the + `baseline-key` given to vercel-performance.yml when recording. + type: string + required: false + default: "default" + +jobs: + # Expands the caller's keys across the form factors this repository measures, + # so callers do not have to know what those are. + plan: + name: Plan + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.plan.outputs.targets }} + env: + BASELINE_KEYS: ${{ inputs.baseline-keys }} + steps: + - name: Build target list + id: plan + env: + # Keep in step with the matrix in vercel-performance.yml. + FORM_FACTORS: 'desktop mobile' + run: | + set -euo pipefail + + targets='[]' + while IFS= read -r baseline_key; do + [ -z "$baseline_key" ] && continue + for form_factor in $FORM_FACTORS; do + targets="$(printf '%s' "$targets" | jq -c \ + --arg k "$baseline_key" --arg f "$form_factor" \ + '. + [{"baseline-key": $k, "form-factor": $f}]')" + done + done <<< "$BASELINE_KEYS" + + if [ "$targets" = '[]' ]; then + echo "::error::baseline-keys produced no targets." + exit 1 + fi + + echo "Refreshing: ${targets}" + echo "targets=${targets}" >> "$GITHUB_OUTPUT" + + touch: + name: Touch ${{ matrix.target.baseline-key }} (${{ matrix.target.form-factor }}) + needs: plan + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.plan.outputs.targets) }} + steps: + # A full restore rather than `lookup-only`: a download is unambiguously + # an access, whereas whether a metadata-only lookup resets the clock is + # not documented. The payload is a few hundred bytes. + - name: Restore baseline + id: restore + uses: actions/cache/restore@v4 + with: + path: baseline.json + # Baselines are keyed by the commit that recorded them, so match on + # the prefix to pick up whichever is most recent. + key: lh-baseline-${{ matrix.target.baseline-key }}-${{ matrix.target.form-factor }}- + restore-keys: | + lh-baseline-${{ matrix.target.baseline-key }}-${{ matrix.target.form-factor }}- + + # A missing baseline is reported rather than treated as an error: pull + # requests degrade gracefully without one, and the next merge records it + # again, so failing here would page someone over a self-healing check. + - name: Report + env: + MATCHED_KEY: ${{ steps.restore.outputs.cache-matched-key }} + BASELINE_KEY: ${{ matrix.target.baseline-key }} + FORM_FACTOR: ${{ matrix.target.form-factor }} + run: | + set -euo pipefail + + if [ -n "$MATCHED_KEY" ]; then + echo "Refreshed ${MATCHED_KEY}" + else + echo "::warning::No ${BASELINE_KEY} ${FORM_FACTOR} baseline found." + echo "::warning::It has expired, or none has been recorded yet." + echo "::warning::A merge to the default branch will record one." + fi diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index 2f3d5ec..80d9dcc 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -34,6 +34,16 @@ on: type: string required: false default: "none" + baseline-key: + description: >- + Name distinguishing this baseline from others in the same repository. + A repository measuring more than one deployment must give each its + own key, or they overwrite each other and pull requests compare + against whichever ran last. Must match between the recording and + comparing callers. + type: string + required: false + default: "default" lighthouse-config-path: description: >- Path to a Lighthouse CI config file in the calling repository, @@ -221,9 +231,12 @@ jobs: uses: actions/cache/restore@v4 with: path: baseline.json - key: lh-baseline-${{ matrix.form-factor }}-${{ github.event.pull_request.base.sha }} + # Baselines are keyed by the commit that recorded them. An exact hit + # is unlikely, so the prefix in restore-keys picks up whichever is + # most recent. + key: lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}- restore-keys: | - lh-baseline-${{ matrix.form-factor }}- + lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}- - name: Summarise results id: summary @@ -445,7 +458,7 @@ jobs: uses: actions/cache/save@v4 with: path: baseline.json - key: lh-baseline-${{ matrix.form-factor }}-${{ github.sha }} + key: lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}-${{ github.sha }} - name: Comment performance results on PR if: github.event.pull_request.number && inputs.baseline-mode != 'record' diff --git a/docs/vercel-performance.md b/docs/vercel-performance.md index 6ebcfc1..2b6e1cc 100644 --- a/docs/vercel-performance.md +++ b/docs/vercel-performance.md @@ -18,6 +18,7 @@ baseline current. | vercel-org-id | ✅ | string | | Vercel organisation ID owning the deployment | | measured-paths | ✅ | string | | Paths to measure, one per line | | baseline-mode | ❌ | string | none | `compare`, `record` or `none` | +| baseline-key | ❌ | string | default | Names the baseline; give each measured deployment its own | | lighthouse-config-path | ❌ | string | .github/lighthouse/lighthouserc.{form-factor}.json | Lighthouse CI config in the caller repo | | regression-threshold | ❌ | number | 20 | Percent a metric may worsen before flagging | | node-version-file | ❌ | string | .nvmrc | File to read the Node version from | @@ -49,8 +50,41 @@ the default branch wrote, but not the reverse. So the two modes live in two caller workflows — `compare` on `pull_request`, `record` on push to the default branch. -Note that a cache expires after 7 days without being read. After a quiet period -the first pull request reports no baseline until the next merge records one. +A repository measuring more than one deployment must give each its own +`baseline-key`. Baselines are cached per key and form factor, so without it two +deployments overwrite each other and pull requests compare against whichever +recorded last. The key must match between the recording and comparing callers. + +#### Keeping the baseline alive + +GitHub evicts a cache that has not been read for 7 days. Because the baseline +is only rewritten on a merge, a quiet fortnight drops it silently and pull +requests report no baseline until the next merge. + +`vercel-performance-keepalive.yml` restores the baselines and does nothing +else, which resets that clock for seconds of runner time. It knows which form +factors are measured, so the caller supplies only its baseline keys: + +```yaml +on: + schedule: + # Every third day, against a 7 day expiry. Scheduled runs are delayed under + # load and can be dropped, so this leaves slack rather than sitting at the + # boundary. + - cron: '0 17 */3 * *' + workflow_dispatch: + +jobs: + keep-baselines-alive: + uses: aligent/workflows/.github/workflows/vercel-performance-keepalive.yml@main + with: + baseline-keys: | + paas + accs +``` + +A missing baseline is reported as a warning rather than a failure: pull +requests degrade gracefully without one, and the next merge records it again. ### Example Usage @@ -76,6 +110,9 @@ jobs: deployment-url: ${{ needs.deploy-preview.outputs.url }} vercel-org-id: ${{ vars.VERCEL_ORG_ID }} baseline-mode: compare + # Names this baseline. Required only when measuring more than one + # deployment, but explicit here to show the pairing. + baseline-key: paas measured-paths: | / /category/example @@ -110,6 +147,9 @@ jobs: deployment-url: ${{ needs.deploy-preview.outputs.url }} vercel-org-id: ${{ vars.VERCEL_ORG_ID }} baseline-mode: record + # Names this baseline. Required only when measuring more than one + # deployment, but explicit here to show the pairing. + baseline-key: paas measured-paths: | / /category/example From 68da37153e2421d46f3b59cfa614c9a80b9b5f90 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 13:58:44 +0930 Subject: [PATCH 08/12] Scope the preview concurrency group by project Deploying several projects from one workflow had them share a group, so cancel-in-progress made each call cancel the previous one. --- .github/workflows/vercel-performance.yml | 4 +++- .github/workflows/vercel-preview.yml | 12 ++++++---- docs/vercel-preview.md | 30 +++++++++++++++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index 80d9dcc..417f8d7 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -130,7 +130,9 @@ jobs: set -euo pipefail if [ -z "$DEPLOY_URL" ]; then - echo "::error::The deploy job did not produce a preview URL." + echo "::error::No deployment URL was passed in." + echo "::error::The deploy this depends on may have failed or been cancelled;" + echo "::error::check its result rather than this job." exit 1 fi diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 5047f1a..087a37b 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -27,9 +27,12 @@ on: required: true concurrency: - # Falls back to the ref so a run on a branch rather than a pull request, - # which has no pull request number, still gets its own group. - group: vercel-preview-${{ github.event.pull_request.number || github.ref }} + # Scoped by project as well as by pull request, because a caller may deploy + # several projects from one workflow. Without the project, two such calls + # share a group and `cancel-in-progress` makes the second cancel the first. + # Falls back to the ref for a run on a branch rather than a pull request, + # which has no pull request number. + group: vercel-preview-${{ inputs.vercel-project-id }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -88,10 +91,11 @@ jobs: DEPLOY_URL: ${{ steps.deploy.outputs.url }} INSPECT_URL: ${{ steps.deploy.outputs.inspect_url }} COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + ENVIRONMENT_NAME: ${{ inputs.environment-name }} run: | marker='' body="$marker - ### ⚡ Preview environment is deploying + ### ⚡ ${ENVIRONMENT_NAME} environment is deploying Wait ~5 minutes for completion, or monitor progress using the Inspect URL. diff --git a/docs/vercel-preview.md b/docs/vercel-preview.md index 4ddaba6..1e059ad 100644 --- a/docs/vercel-preview.md +++ b/docs/vercel-preview.md @@ -10,7 +10,7 @@ pull request with the preview and inspect URLs. Intended to be called from a | vercel-org-id | ✅ | string | | Vercel organisation ID | | vercel-project-id | ✅ | string | | Vercel project ID | | working-directory | ❌ | string | . | Directory to run the Vercel deploy from | -| environment-name | ❌ | string | Preview | GitHub Environment to deploy to | +| environment-name | ❌ | string | Preview | GitHub Environment to deploy to. Also used as the heading of the pull request comment | #### **Outputs** | Name | Description | @@ -40,6 +40,34 @@ jobs: vercel-token: ${{ secrets.VERCEL_TOKEN }} ``` +#### Deploying more than one project + +Call the workflow once per project and give each call a distinct +`environment-name`. The name keeps the deployments separate in the repository's +Environments list and labels each pull request comment, so the two are +distinguishable. + +```yaml +jobs: + deploy-paas-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + environment-name: "Preview PaaS" + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + + deploy-accs-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_ACCS_PROJECT_ID }} + environment-name: "Preview ACCS" + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} +``` + To measure the performance of the resulting preview, pair this with [Vercel Preview Performance](vercel-performance.md), which takes the `url` output above. From 461cc652bbb0ed53537ba865b7e15974d2d8ea24 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 14:02:57 +0930 Subject: [PATCH 09/12] Key the performance concurrency group on baseline-key The group used deployment-url, but that is only known once the deploy the call depends on has finished, while the group is evaluated when the call is queued. Every caller therefore shared an empty group and cancelled each other. --- .github/workflows/vercel-performance.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index 417f8d7..bae63c8 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -36,11 +36,12 @@ on: default: "none" baseline-key: description: >- - Name distinguishing this baseline from others in the same repository. - A repository measuring more than one deployment must give each its - own key, or they overwrite each other and pull requests compare - against whichever ran last. Must match between the recording and - comparing callers. + Name distinguishing this measurement from others in the same + repository. A repository measuring more than one deployment must give + each its own key: it namespaces both the cached baseline and the + concurrency group, so without it the deployments overwrite each + other's baselines and cancel each other's runs. Must match between + the recording and comparing callers. type: string required: false default: "default" @@ -78,9 +79,11 @@ on: required: false concurrency: - # Keyed by deployment so two callers measuring different deployments in the - # same pull request do not cancel each other. - group: vercel-performance-${{ inputs.deployment-url }} + # Keyed by baseline-key rather than by deployment-url: the URL is only known + # once the deploy this depends on has finished, but the group is evaluated + # when the call is queued, so it would still be empty and every caller in the + # workflow would share one group and cancel each other. + group: vercel-performance-${{ inputs.baseline-key }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: From e94bc15f866ac4adaee14cf7631988ac37ac6fe2 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 14:08:07 +0930 Subject: [PATCH 10/12] Expose the preview URL to callers The deploy job set an output, but a reusable workflow only exposes what it declares in on.workflow_call.outputs, so callers always read an empty string. --- .github/workflows/vercel-preview.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index 087a37b..038c3fc 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -21,6 +21,12 @@ on: type: string required: false default: "Preview" + outputs: + url: + description: >- + The preview deployment URL. Still building when the job ends, as the + deploy uses --no-wait. + value: ${{ jobs.deploy-preview.outputs.url }} secrets: vercel-token: description: "Vercel deployment token" From 0241142be4a5dbaeed5acb424d990d32492063ed Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 15:13:10 +0930 Subject: [PATCH 11/12] Scope the PR comment by baseline-key Two deployments measured in one pull request shared a comment marker, so whichever finished last overwrote the other's results. --- .github/workflows/vercel-performance.yml | 14 ++++++++++++-- docs/vercel-performance.md | 7 ++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index bae63c8..f0d4c47 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -473,13 +473,23 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} COMMIT_SHA: ${{ github.event.pull_request.head.sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + BASELINE_KEY: ${{ inputs.baseline-key }} run: | set -euo pipefail - marker='' + # Scoped by baseline-key so a repository measuring several + # deployments gets one comment each, rather than overwriting a + # shared one. The heading is only labelled when a key was given. + marker="" + if [ "$BASELINE_KEY" = "default" ]; then + heading="### 📊 Preview performance" + else + heading="### 📊 Preview performance (${BASELINE_KEY})" + fi + { echo "$marker" - echo "### 📊 Preview performance" + echo "$heading" echo cat lighthouse-summary.md echo diff --git a/docs/vercel-performance.md b/docs/vercel-performance.md index 2b6e1cc..77e1a4c 100644 --- a/docs/vercel-performance.md +++ b/docs/vercel-performance.md @@ -51,9 +51,10 @@ caller workflows — `compare` on `pull_request`, `record` on push to the defaul branch. A repository measuring more than one deployment must give each its own -`baseline-key`. Baselines are cached per key and form factor, so without it two -deployments overwrite each other and pull requests compare against whichever -recorded last. The key must match between the recording and comparing callers. +`baseline-key`. It namespaces the cached baseline, the concurrency group, and +the pull request comment, so without it two deployments overwrite each other's +baselines, cancel each other's runs, and fight over one comment. The key must +match between the recording and comparing callers. #### Keeping the baseline alive From 3551be9a9b8c8d111d438c3d86a0b2561d845529 Mon Sep 17 00:00:00 2001 From: Chris Park Date: Wed, 26 Aug 2026 15:43:17 +0930 Subject: [PATCH 12/12] Pin action hashes and correct version comments actions/cache was tag-pinned, which the blanket policy rejects. setup-node was hash-pinned to an unrelated Dependabot commit rather than to v6.0.0, and the lighthouse-ci-action comment named a v12.6.1 tag that does not exist. Both now use the versions already referenced elsewhere in this repo. --- .github/workflows/vercel-performance-keepalive.yml | 2 +- .github/workflows/vercel-performance.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/vercel-performance-keepalive.yml b/.github/workflows/vercel-performance-keepalive.yml index 56cbad7..af6d0c5 100644 --- a/.github/workflows/vercel-performance-keepalive.yml +++ b/.github/workflows/vercel-performance-keepalive.yml @@ -76,7 +76,7 @@ jobs: # not documented. The payload is a few hundred bytes. - name: Restore baseline id: restore - uses: actions/cache/restore@v4 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 with: path: baseline.json # Baselines are keyed by the commit that recorded them, so match on diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index f0d4c47..57130e3 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -113,7 +113,7 @@ jobs: # `package-manager-cache: false` because this job never installs project # dependencies — it only needs Node to summarise the Lighthouse results. - name: Install Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v6.0.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e #v6.4.0 with: node-version-file: ${{ inputs.node-version-file }} package-manager-cache: false @@ -221,7 +221,7 @@ jobs: # smooth out. - name: Run Lighthouse id: lighthouse - uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12.6.1 + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12 with: urls: ${{ steps.urls.outputs.list }} configPath: ${{ steps.config.outputs.path }} @@ -233,7 +233,7 @@ jobs: # the reverse, which is why recording runs on a separate trigger. - name: Restore baseline if: inputs.baseline-mode == 'compare' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 with: path: baseline.json # Baselines are keyed by the commit that recorded them. An exact hit @@ -460,7 +460,7 @@ jobs: # from merged code. - name: Save baseline if: inputs.baseline-mode == 'record' - uses: actions/cache/save@v4 + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 with: path: baseline.json key: lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}-${{ github.sha }}