diff --git a/.github/workflows/vercel-performance-keepalive.yml b/.github/workflows/vercel-performance-keepalive.yml new file mode 100644 index 0000000..af6d0c5 --- /dev/null +++ b/.github/workflows/vercel-performance-keepalive.yml @@ -0,0 +1,105 @@ +name: Vercel Performance Baseline Keep-Alive + +# GitHub evicts a cache that has not been read for 7 days. A performance +# baseline is only rewritten when something merges to the default branch, so a +# quiet fortnight would silently drop it and pull requests would report no +# baseline until the next merge. +# +# Reading a cache resets that clock, so this restores the baselines and does +# nothing else. It deploys nothing and measures nothing, costing seconds rather +# than the minutes a re-measurement would. +# +# The form factors are declared here rather than by the caller so they stay in +# step with the matrix in vercel-performance.yml. + +on: + workflow_call: + inputs: + baseline-keys: + description: >- + Baseline keys to keep alive, one per line, matching the + `baseline-key` given to vercel-performance.yml when recording. + type: string + required: false + default: "default" + +jobs: + # Expands the caller's keys across the form factors this repository measures, + # so callers do not have to know what those are. + plan: + name: Plan + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.plan.outputs.targets }} + env: + BASELINE_KEYS: ${{ inputs.baseline-keys }} + steps: + - name: Build target list + id: plan + env: + # Keep in step with the matrix in vercel-performance.yml. + FORM_FACTORS: 'desktop mobile' + run: | + set -euo pipefail + + targets='[]' + while IFS= read -r baseline_key; do + [ -z "$baseline_key" ] && continue + for form_factor in $FORM_FACTORS; do + targets="$(printf '%s' "$targets" | jq -c \ + --arg k "$baseline_key" --arg f "$form_factor" \ + '. + [{"baseline-key": $k, "form-factor": $f}]')" + done + done <<< "$BASELINE_KEYS" + + if [ "$targets" = '[]' ]; then + echo "::error::baseline-keys produced no targets." + exit 1 + fi + + echo "Refreshing: ${targets}" + echo "targets=${targets}" >> "$GITHUB_OUTPUT" + + touch: + name: Touch ${{ matrix.target.baseline-key }} (${{ matrix.target.form-factor }}) + needs: plan + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.plan.outputs.targets) }} + steps: + # A full restore rather than `lookup-only`: a download is unambiguously + # an access, whereas whether a metadata-only lookup resets the clock is + # not documented. The payload is a few hundred bytes. + - name: Restore baseline + id: restore + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 + with: + path: baseline.json + # Baselines are keyed by the commit that recorded them, so match on + # the prefix to pick up whichever is most recent. + key: lh-baseline-${{ matrix.target.baseline-key }}-${{ matrix.target.form-factor }}- + restore-keys: | + lh-baseline-${{ matrix.target.baseline-key }}-${{ matrix.target.form-factor }}- + + # A missing baseline is reported rather than treated as an error: pull + # requests degrade gracefully without one, and the next merge records it + # again, so failing here would page someone over a self-healing check. + - name: Report + env: + MATCHED_KEY: ${{ steps.restore.outputs.cache-matched-key }} + BASELINE_KEY: ${{ matrix.target.baseline-key }} + FORM_FACTOR: ${{ matrix.target.form-factor }} + run: | + set -euo pipefail + + if [ -n "$MATCHED_KEY" ]; then + echo "Refreshed ${MATCHED_KEY}" + else + echo "::warning::No ${BASELINE_KEY} ${FORM_FACTOR} baseline found." + echo "::warning::It has expired, or none has been recorded yet." + echo "::warning::A merge to the default branch will record one." + fi diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml new file mode 100644 index 0000000..57130e3 --- /dev/null +++ b/.github/workflows/vercel-performance.yml @@ -0,0 +1,506 @@ +name: Vercel Preview Performance + +# Measures Core Web Vitals against an already-deployed Vercel preview and +# either comments the results on a pull request or records them as a baseline. +# +# The caller decides what to measure and when: pair it with a deploy workflow +# via `needs`, and call it again from a push-to-default-branch workflow with +# `baseline-mode: record` to keep the baseline current. + +on: + workflow_call: + inputs: + deployment-url: + description: "URL of the deployment to measure. May still be building." + type: string + required: true + vercel-org-id: + description: "Vercel organisation ID that owns the deployment" + type: string + required: true + measured-paths: + description: >- + Paths to measure, one per line, relative to the deployment URL + (e.g. "/"). Newline-delimited so paths may contain query strings. + type: string + required: true + baseline-mode: + description: >- + "compare" reports each metric against the recorded baseline, + "record" stores this run as the baseline instead of commenting, and + "none" reports the measurements on their own. Recording must run on + the default branch: a pull request can read a cache the default + branch wrote, but not the reverse. + type: string + required: false + default: "none" + baseline-key: + description: >- + Name distinguishing this measurement from others in the same + repository. A repository measuring more than one deployment must give + each its own key: it namespaces both the cached baseline and the + concurrency group, so without it the deployments overwrite each + other's baselines and cancel each other's runs. Must match between + the recording and comparing callers. + type: string + required: false + default: "default" + lighthouse-config-path: + description: >- + Path to a Lighthouse CI config file in the calling repository, + holding run count and budgets. Any "{form-factor}" placeholder is + replaced with "desktop" or "mobile", so each form factor can carry + its own budgets. Mobile Lighthouse throttles CPU and network, so its + budgets should be looser than desktop. + type: string + required: false + default: ".github/lighthouse/lighthouserc.{form-factor}.json" + regression-threshold: + description: >- + Percentage a metric may worsen against the baseline before it is + flagged. Preview measurements are noisy, so keep this well above the + run-to-run variance you observe. + type: number + required: false + default: 20 + node-version-file: + description: "File to read the Node version from" + type: string + required: false + default: ".nvmrc" + secrets: + vercel-token: + description: "Vercel token, used to wait for the deployment to be ready" + required: true + vercel-automation-bypass-secret: + description: >- + Vercel Protection Bypass for Automation secret. Required when + Deployment Protection is enabled on the project. + required: false + +concurrency: + # Keyed by baseline-key rather than by deployment-url: the URL is only known + # once the deploy this depends on has finished, but the group is evaluated + # when the call is queued, so it would still be empty and every caller in the + # workflow would share one group and cancel each other. + group: vercel-performance-${{ inputs.baseline-key }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + performance: + name: Preview Performance (${{ matrix.form-factor }}) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + strategy: + # Both form factors measure the same deployment independently, so one + # failing should not cancel the other. + fail-fast: false + matrix: + form-factor: [desktop, mobile] + env: + # How long to wait for the preview deployment to finish building. + READY_TIMEOUT: 10m + MEASURED_PATHS: ${{ inputs.measured-paths }} + FORM_FACTOR: ${{ matrix.form-factor }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0 + with: + persist-credentials: false + + # `package-manager-cache: false` because this job never installs project + # dependencies โ€” it only needs Node to summarise the Lighthouse results. + - name: Install Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e #v6.4.0 + with: + node-version-file: ${{ inputs.node-version-file }} + package-manager-cache: false + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + # The deploy job uses `--no-wait`, so the deployment is still building + # when this job starts. Wait for it to finish so the performance check can run against a fully built preview. + - name: Wait for preview deployment + id: deployment + env: + VERCEL_TOKEN: ${{ secrets.vercel-token }} + VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} + DEPLOY_URL: ${{ inputs.deployment-url }} + run: | + set -euo pipefail + + if [ -z "$DEPLOY_URL" ]; then + echo "::error::No deployment URL was passed in." + echo "::error::The deploy this depends on may have failed or been cancelled;" + echo "::error::check its result rather than this job." + exit 1 + fi + + echo "Waiting for ${DEPLOY_URL} to become ready." + if ! vercel inspect "$DEPLOY_URL" \ + --token="$VERCEL_TOKEN" \ + --scope="$VERCEL_ORG_ID" \ + --wait \ + --timeout "$READY_TIMEOUT"; then + echo "::error::Preview deployment was not ready within ${READY_TIMEOUT}." + exit 1 + fi + + echo "url=$DEPLOY_URL" >> "$GITHUB_OUTPUT" + + # The first request to a cold preview pays serverless cold start and any + # on-demand page generation. Measuring that would report build latency + # rather than page performance, so discard it. Each path is warmed + # separately because they are generated on demand independently. + - name: Warm up preview deployment + id: urls + env: + BASE_URL: ${{ steps.deployment.outputs.url }} + BYPASS_SECRET: ${{ secrets.vercel-automation-bypass-secret }} + run: | + set -euo pipefail + + failed=0 + urls="" + + # Read line by line: a path may contain `?` and `&`, so splitting on + # whitespace would mangle it. + while IFS= read -r path; do + [ -z "$path" ] && continue + url="${BASE_URL}${path}" + urls="${urls}${url}"$'\n' + + for attempt in 1 2 3; do + status="$(curl -sS -o /dev/null -w '%{http_code}' -L \ + --max-time 60 \ + ${BYPASS_SECRET:+-H "x-vercel-protection-bypass: ${BYPASS_SECRET}"} \ + "$url" || echo 000)" + echo "Warm-up ${url} attempt ${attempt}: HTTP ${status}" + [ "$status" = "200" ] && break + [ "$attempt" = "3" ] && failed=1 + sleep 5 + done + done <<< "$MEASURED_PATHS" + + if [ "$failed" = "1" ]; then + echo "::error::One or more URLs did not return HTTP 200 after 3 warm-up requests." + echo "::error::If Deployment Protection is enabled, pass the vercel-automation-bypass-secret secret." + exit 1 + fi + + # Hand the fully-qualified URLs to the Lighthouse step so the list is + # defined once, by the caller, in measured-paths. + { + echo "list<> "$GITHUB_OUTPUT" + + - name: Resolve Lighthouse config + id: config + env: + CONFIG_TEMPLATE: ${{ inputs.lighthouse-config-path }} + run: | + set -euo pipefail + + path="${CONFIG_TEMPLATE//\{form-factor\}/$FORM_FACTOR}" + if [ ! -f "$path" ]; then + echo "::error::Lighthouse config not found at ${path}." + exit 1 + fi + + echo "Using Lighthouse config ${path}" + echo "path=$path" >> "$GITHUB_OUTPUT" + + # Run count and budgets come from the caller's Lighthouse CI config. Set + # `aggregationMethod: median` there: LHCI defaults to `optimistic`, which + # takes the best run and hides the variance that multiple runs exist to + # smooth out. + - name: Run Lighthouse + id: lighthouse + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 #v12 + with: + urls: ${{ steps.urls.outputs.list }} + configPath: ${{ steps.config.outputs.path }} + uploadArtifacts: true + artifactName: lighthouse-reports-${{ matrix.form-factor }} + + # Restores the most recent baseline recorded from the default branch. + # A pull request can read caches written by the default branch, but not + # the reverse, which is why recording runs on a separate trigger. + - name: Restore baseline + if: inputs.baseline-mode == 'compare' + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 + with: + path: baseline.json + # Baselines are keyed by the commit that recorded them. An exact hit + # is unlikely, so the prefix in restore-keys picks up whichever is + # most recent. + key: lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}- + restore-keys: | + lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}- + + - name: Summarise results + id: summary + env: + RESULTS_PATH: ${{ steps.lighthouse.outputs.resultsPath }} + ASSERTION_RESULTS: ${{ steps.lighthouse.outputs.assertionResults }} + REGRESSION_THRESHOLD: ${{ inputs.regression-threshold }} + BASELINE_PATH: ${{ inputs.baseline-mode == 'compare' && 'baseline.json' || '' }} + BASELINE_RECORD: ${{ inputs.baseline-mode == 'record' }} + run: | + set -euo pipefail + + # Written out rather than inlined with `node -e` so the quoting stays + # manageable and the script is readable. + cat > summarise.mjs <<'SUMMARISE_EOF' + import { readFileSync, writeFileSync, existsSync } from 'node:fs'; + import { join } from 'node:path'; + + const METRICS = [ + ['largest-contentful-paint', 'LCP', 'ms'], + ['total-blocking-time', 'TBT', 'ms'], + ['cumulative-layout-shift', 'CLS', ''], + ['first-contentful-paint', 'FCP', 'ms'], + ['speed-index', 'SI', 'ms'], + ]; + + const format = (value, unit) => { + if (typeof value !== 'number') return 'n/a'; + if (unit !== 'ms') return value.toFixed(3); + return value >= 1000 + ? `${(value / 1000).toFixed(2)} s` + : `${Math.round(value)} ms`; + }; + + const toLabel = (url) => { + try { + return new URL(url).pathname || '/'; + } catch { + return url; + } + }; + + // Percentage change from baseline. Positive means worse for every metric here + // (all are "lower is better"). Returns null when there is nothing to compare. + const delta = (current, baseline) => { + if (typeof current !== 'number' || typeof baseline !== 'number') return null; + // A zero baseline has no meaningful percentage. Treat an unchanged zero as + // no change; report any rise from zero as a regression rather than "n/a", + // which would hide a metric going from perfect to non-zero (TBT and CLS + // are commonly zero on a healthy page). + if (baseline === 0) return current === 0 ? 0 : Infinity; + return ((current - baseline) / baseline) * 100; + }; + + const formatDelta = (percent, threshold) => { + if (percent === null) return 'n/a'; + if (percent === Infinity) return 'new ๐Ÿ”บ'; + const rounded = Math.round(percent); + if (Math.abs(rounded) < 1) return 'ยฑ0%'; + const sign = rounded > 0 ? '+' : ''; + const flag = rounded > threshold ? ' ๐Ÿ”บ' : ''; + return `${sign}${rounded}%${flag}`; + }; + + const resultsPath = process.env.RESULTS_PATH; + const formFactor = process.env.FORM_FACTOR || 'desktop'; + const threshold = Number(process.env.REGRESSION_THRESHOLD || '20'); + const baselinePath = process.env.BASELINE_PATH || ''; + const recordOnly = process.env.BASELINE_RECORD === 'true'; + + const manifest = JSON.parse( + readFileSync(join(resultsPath, 'manifest.json'), 'utf8') + ); + + // LHCI writes one manifest entry per run. The entry flagged as the + // representative run is the median run for that URL, so reporting it keeps + // this summary consistent with what the assertions checked. + const representative = manifest.filter((entry) => entry.isRepresentativeRun); + if (representative.length === 0) { + console.error('Manifest contained no representative run.'); + process.exit(1); + } + + // Collect the medians into a plain object, both for the comparison below and + // so the run on the default branch can persist it as the next baseline. + const measured = {}; + for (const entry of representative) { + const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + const url = toLabel(entry.url); + measured[url] = { score: entry.summary?.performance ?? null }; + for (const [id] of METRICS) { + measured[url][id] = report.audits?.[id]?.numericValue ?? null; + } + } + + if (recordOnly) { + writeFileSync( + process.env.BASELINE_OUT || 'baseline.json', + JSON.stringify({ formFactor, measured }, null, 2) + ); + console.error(`Recorded baseline for ${formFactor}.`); + process.exit(0); + } + + // A baseline is optional: the first pull request after this is enabled has + // nothing to compare against, and should still report its measurements. + let baseline = null; + if (baselinePath && existsSync(baselinePath)) { + try { + const parsed = JSON.parse(readFileSync(baselinePath, 'utf8')); + if (parsed.formFactor === formFactor) baseline = parsed.measured; + } catch (error) { + console.error(`Could not read baseline: ${error.message}`); + } + } + + const warningsByUrl = new Map(); + try { + for (const result of JSON.parse(process.env.ASSERTION_RESULTS || '[]')) { + if (!result.url) continue; + const existing = warningsByUrl.get(result.url) ?? new Set(); + existing.add(result.auditId); + warningsByUrl.set(result.url, existing); + } + } catch (error) { + console.error(`Could not parse assertion results: ${error.message}`); + } + + const lines = []; + let anyWarnings = false; + let anyRegressions = false; + + for (const entry of representative) { + const url = toLabel(entry.url); + const current = measured[url]; + const previous = baseline?.[url]; + const warned = warningsByUrl.get(entry.url) ?? new Set(); + if (warned.size > 0) anyWarnings = true; + + const scoreDelta = + typeof previous?.score === 'number' && typeof current.score === 'number' + ? Math.round((current.score - previous.score) * 100) + : null; + + lines.push( + `#### \`${url}\``, + '', + `**Performance score:** ${ + typeof current.score === 'number' + ? Math.round(current.score * 100) + : 'n/a' + }/100${warned.has('categories:performance') ? ' โš ๏ธ' : ''}${ + scoreDelta === null + ? '' + : ` (${scoreDelta > 0 ? '+' : scoreDelta === 0 ? 'ยฑ' : ''}${scoreDelta} vs baseline)` + }`, + '', + baseline ? '| Metric | Median | vs baseline | |' : '| Metric | Median | |', + baseline ? '| --- | --- | --- | --- |' : '| --- | --- | --- |' + ); + + for (const [id, label, unit] of METRICS) { + const value = current[id]; + const percent = previous ? delta(value, previous[id]) : null; + if (percent !== null && percent > threshold) anyRegressions = true; + + const cells = [label, format(value, unit)]; + if (baseline) cells.push(formatDelta(percent, threshold)); + cells.push(warned.has(id) ? 'โš ๏ธ' : 'โœ…'); + lines.push(`| ${cells.join(' | ')} |`); + } + + lines.push(''); + } + + const runs = manifest.length / representative.length; + lines.push(`_Median of ${runs} run${runs === 1 ? '' : 's'} per URL._`); + + if (anyRegressions) { + lines.push( + '', + `๐Ÿ”บ One or more metrics are more than ${threshold}% slower than the ` + + 'baseline recorded from the default branch.' + ); + } + + // Preview deployments are cold and CPU-shared on CI runners, so treat these as + // a smoke signal for large regressions, not a benchmark. + if (anyWarnings) { + lines.push( + '', + 'โš ๏ธ One or more metrics are over budget. Preview deployments are ' + + 'slower than production โ€” confirm against a second run before ' + + 'treating this as a real regression.' + ); + } + + if (!baseline && baselinePath) { + lines.push( + '', + '_No baseline available yet for this form factor; deltas will appear ' + + 'once the default branch has recorded one._' + ); + } + + console.log(lines.join('\n')); + SUMMARISE_EOF + + node summarise.mjs > lighthouse-summary.md + if [ "${BASELINE_RECORD}" != "true" ]; then + cat lighthouse-summary.md + fi + + # Persists this run as the baseline for later pull requests. Only the + # default branch writes, so the value a pull request reads always came + # from merged code. + - name: Save baseline + if: inputs.baseline-mode == 'record' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae #v5.0.5 + with: + path: baseline.json + key: lh-baseline-${{ inputs.baseline-key }}-${{ matrix.form-factor }}-${{ github.sha }} + + - name: Comment performance results on PR + if: github.event.pull_request.number && inputs.baseline-mode != 'record' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + BASELINE_KEY: ${{ inputs.baseline-key }} + run: | + set -euo pipefail + + # Scoped by baseline-key so a repository measuring several + # deployments gets one comment each, rather than overwriting a + # shared one. The heading is only labelled when a key was given. + marker="" + if [ "$BASELINE_KEY" = "default" ]; then + heading="### ๐Ÿ“Š Preview performance" + else + heading="### ๐Ÿ“Š Preview performance (${BASELINE_KEY})" + fi + + { + echo "$marker" + echo "$heading" + echo + cat lighthouse-summary.md + echo + echo "[Full reports](${RUN_URL}) ยท _commit ${COMMIT_SHA}_" + } > comment.md + + comment_id="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq "[.[] | select(.body | startswith(\"${marker}\"))][0].id")" + + if [ -n "$comment_id" ] && [ "$comment_id" != "null" ]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body=@comment.md + else + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@comment.md + fi diff --git a/.github/workflows/vercel-preview.yml b/.github/workflows/vercel-preview.yml index d29e7b1..9ffd829 100644 --- a/.github/workflows/vercel-preview.yml +++ b/.github/workflows/vercel-preview.yml @@ -21,6 +21,12 @@ on: type: string required: false default: "Preview" + outputs: + url: + description: >- + The preview deployment URL. Still building when the job ends, as the + deploy uses --no-wait. + value: ${{ jobs.deploy-preview.outputs.url }} secrets: vercel-token: description: "Vercel deployment token" @@ -29,7 +35,10 @@ on: # Scoped per project so a repo deploying several Vercel projects from one PR # gets an independent group per project instead of the jobs cancelling each other. concurrency: - group: vercel-preview-${{ github.event.pull_request.number }}-${{ inputs.vercel-project-id }} + # Falls back to the ref for a run on a branch rather than a pull request, + # which has no pull request number: this workflow is also called on pushes + # to the default branch to record a performance baseline. + group: vercel-preview-${{ github.event.pull_request.number || github.ref }}-${{ inputs.vercel-project-id }} cancel-in-progress: true jobs: @@ -43,6 +52,8 @@ jobs: environment: name: ${{ inputs.environment-name }} url: ${{ steps.deploy.outputs.url }} + outputs: + url: ${{ steps.deploy.outputs.url }} env: VERCEL_ORG_ID: ${{ inputs.vercel-org-id }} VERCEL_PROJECT_ID: ${{ inputs.vercel-project-id }} @@ -76,8 +87,11 @@ jobs: # Posts (or updates in place) a single comment on the PR with the preview URL. # The marker is scoped per project so each Vercel project keeps its own - # comment rather than overwriting another project's. + # comment rather than overwriting another project's. Skipped when there + # is no pull request, e.g. a deploy triggered by a push to the default + # branch. - name: Comment preview URL on PR + if: github.event.pull_request.number env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -85,10 +99,11 @@ jobs: DEPLOY_URL: ${{ steps.deploy.outputs.url }} INSPECT_URL: ${{ steps.deploy.outputs.inspect_url }} COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + ENVIRONMENT_NAME: ${{ inputs.environment-name }} run: | marker="" body="$marker - ### โšก Preview environment is deploying + ### โšก ${ENVIRONMENT_NAME} environment is deploying Wait ~5 minutes for completion, or monitor progress using the Inspect URL. diff --git a/docs/vercel-performance.md b/docs/vercel-performance.md new file mode 100644 index 0000000..77e1a4c --- /dev/null +++ b/docs/vercel-performance.md @@ -0,0 +1,254 @@ +# Vercel Preview Performance + +Measures Core Web Vitals against an already-deployed Vercel preview with +[Lighthouse CI](https://github.com/treosh/lighthouse-ci-action) and posts the +median results as a pull request comment. + +Desktop and mobile run concurrently as a matrix, each against its own budgets. + +The caller decides what to measure and when: pair it with +[Vercel Preview Deployment](vercel-preview.md) via `needs`, and call it again +from a push-to-default-branch workflow with `baseline-mode: record` to keep the +baseline current. + +#### **Inputs** +| Name | Required | Type | Default | Description | +|------------------------|----------|--------|-----------------------------------------------|------------------------------------------------| +| deployment-url | โœ… | string | | Deployment to measure. May still be building. | +| vercel-org-id | โœ… | string | | Vercel organisation ID owning the deployment | +| measured-paths | โœ… | string | | Paths to measure, one per line | +| baseline-mode | โŒ | string | none | `compare`, `record` or `none` | +| baseline-key | โŒ | string | default | Names the baseline; give each measured deployment its own | +| lighthouse-config-path | โŒ | string | .github/lighthouse/lighthouserc.{form-factor}.json | Lighthouse CI config in the caller repo | +| regression-threshold | โŒ | number | 20 | Percent a metric may worsen before flagging | +| node-version-file | โŒ | string | .nvmrc | File to read the Node version from | + +#### **Secrets** +| Name | Required | Description | +|---------------------------------|----------|----------------------------------------------------------| +| vercel-token | โœ… | Used to wait for the deployment to be ready | +| vercel-automation-bypass-secret | โŒ | Protection Bypass for Automation, if protection is on | + +### How it works + +1. Waits for the deployment to finish building, so the caller can deploy with + `--no-wait` and hand over the URL immediately. +2. Warms each measured path. The first request to a cold preview pays + serverless cold start and any on-demand page generation, which would + otherwise be measured as page latency. +3. Runs Lighthouse against every path, for desktop and mobile, and reports the + median of each metric. + +### Baseline comparison + +Absolute budgets cannot tell "this pull request made things worse" from "this +runner was busy". A regression from 1.2s to 3.5s passes a 4s budget silently. +Comparing against the default branch isolates what the change actually did. + +Recording must run on the **default branch**: a pull request can read a cache +the default branch wrote, but not the reverse. So the two modes live in two +caller workflows โ€” `compare` on `pull_request`, `record` on push to the default +branch. + +A repository measuring more than one deployment must give each its own +`baseline-key`. It namespaces the cached baseline, the concurrency group, and +the pull request comment, so without it two deployments overwrite each other's +baselines, cancel each other's runs, and fight over one comment. The key must +match between the recording and comparing callers. + +#### Keeping the baseline alive + +GitHub evicts a cache that has not been read for 7 days. Because the baseline +is only rewritten on a merge, a quiet fortnight drops it silently and pull +requests report no baseline until the next merge. + +`vercel-performance-keepalive.yml` restores the baselines and does nothing +else, which resets that clock for seconds of runner time. It knows which form +factors are measured, so the caller supplies only its baseline keys: + +```yaml +on: + schedule: + # Every third day, against a 7 day expiry. Scheduled runs are delayed under + # load and can be dropped, so this leaves slack rather than sitting at the + # boundary. + - cron: '0 17 */3 * *' + workflow_dispatch: + +jobs: + keep-baselines-alive: + uses: aligent/workflows/.github/workflows/vercel-performance-keepalive.yml@main + with: + baseline-keys: | + paas + accs +``` + +A missing baseline is reported as a warning rather than a failure: pull +requests degrade gracefully without one, and the next merge records it again. + +### Example Usage + +Measuring a pull request, comparing against the baseline: + +```yaml +on: + pull_request: + +jobs: + deploy-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + + performance: + needs: deploy-preview + uses: aligent/workflows/.github/workflows/vercel-performance.yml@main + with: + deployment-url: ${{ needs.deploy-preview.outputs.url }} + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + baseline-mode: compare + # Names this baseline. Required only when measuring more than one + # deployment, but explicit here to show the pairing. + baseline-key: paas + measured-paths: | + / + /category/example + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} +``` + +Recording the baseline after a merge. The measured paths must match those above, +or pull requests will have nothing to compare against: + +```yaml +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + deploy-preview: + uses: aligent/workflows/.github/workflows/vercel-preview.yml@main + with: + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + vercel-project-id: ${{ vars.VERCEL_PROJECT_ID }} + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + + record-baseline: + needs: deploy-preview + uses: aligent/workflows/.github/workflows/vercel-performance.yml@main + with: + deployment-url: ${{ needs.deploy-preview.outputs.url }} + vercel-org-id: ${{ vars.VERCEL_ORG_ID }} + baseline-mode: record + # Names this baseline. Required only when measuring more than one + # deployment, but explicit here to show the pairing. + baseline-key: paas + measured-paths: | + / + /category/example + secrets: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-automation-bypass-secret: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} +``` + +The baseline is deliberately a **preview built from the default branch**, not a +production deployment. Production is warm, CDN-cached, and often points at +different data and environment variables, so a preview measured against it would +show deltas caused by the environment rather than by the change. + +### Lighthouse config + +The caller supplies one config per form factor. `{form-factor}` in the path is +replaced with `desktop` or `mobile`: + +```json +{ + "ci": { + "collect": { + "numberOfRuns": 3, + "settings": { + "preset": "desktop", + "onlyCategories": ["performance"], + "maxWaitForLoad": 60000, + "chromeFlags": "--no-sandbox --disable-dev-shm-usage" + } + }, + "assert": { + "aggregationMethod": "median", + "assertions": { + "categories:performance": ["warn", { "minScore": 0.5 }], + "largest-contentful-paint": ["warn", { "maxNumericValue": 4000 }], + "total-blocking-time": ["warn", { "maxNumericValue": 600 }], + "cumulative-layout-shift": ["warn", { "maxNumericValue": 0.25 }], + "first-contentful-paint": ["warn", { "maxNumericValue": 3000 }], + "speed-index": ["warn", { "maxNumericValue": 5800 }] + } + } + } +} +``` + +The mobile config sets `"formFactor": "mobile"` instead of `"preset": "desktop"`, +which applies mobile emulation and a 4x CPU slowdown. Its budgets should be +roughly 1.5x desktop as a result; CLS is unchanged, as layout shift is not +throttling-dependent. + +Three things in that config are worth calling out: + +- **`aggregationMethod: median`** โ€” Lighthouse CI defaults to `optimistic`, + which takes the *best* run and so discards the variance that multiple runs + exist to smooth out. Set this explicitly. +- **`numberOfRuns: 3`** โ€” a single run against a cold preview is not a usable + signal. An odd count means the median is a value that was actually measured. +- **`warn` rather than `error`** โ€” the job reports breaches without failing, so + a noisy preview cannot block a merge. Switch to `error` once you have enough + history to trust the thresholds. + +### Reading the results + +Preview deployments are cold and run on shared CI runners, so treat the numbers +as a smoke signal for large regressions rather than a benchmark. Keep budgets +loose: scoring a preview against production-grade thresholds flags nearly every +pull request, and a check that cries wolf gets ignored. + +### Deployment Protection + +If the Vercel project has [Deployment +Protection](https://vercel.com/docs/deployment-protection) enabled, preview URLs +require a Vercel login. Automated requests receive an authentication page rather +than the site, so Lighthouse would score the login screen instead of the page +under test. + +To allow the check through, generate a [Protection Bypass for +Automation](https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/protection-bypass-automation) +secret in the Vercel project's Deployment Protection settings, add it to the +calling repository as a secret, and pass it as +`vercel-automation-bypass-secret`. The workflow sends it as the +`x-vercel-protection-bypass` header on both the warm-up requests and the +Lighthouse runs. + +Generating the secret requires at least the **member** team role, or the +**Project Administrator** role on the project. Note that regenerating or +deleting a secret invalidates it for existing deployments, which then need to +be redeployed. + +The secret is optional and is omitted from requests entirely when unset, which +is correct for a project without Deployment Protection. If protection *is* +enabled and the secret is missing, the warm-up step fails with a non-200 status +rather than reporting misleading scores. + +To check whether a project needs it, open a preview URL in a private browser +window: a login prompt means protection is enabled. + +> Vercel also exposes this value to the running deployment as the +> `VERCEL_AUTOMATION_BYPASS_SECRET` system environment variable, but that is not +> usable here โ€” the workflow needs the secret *before* it can reach the +> deployment, so it must come from repository secrets. diff --git a/docs/vercel-preview.md b/docs/vercel-preview.md index 079b767..0a7ed01 100644 --- a/docs/vercel-preview.md +++ b/docs/vercel-preview.md @@ -5,12 +5,17 @@ pull request with the preview and inspect URLs. Intended to be called from a `pull_request` triggered workflow. #### **Inputs** -| Name | Required | Type | Default | Description | -|--------------------|----------|--------|----------|----------------------------------------| -| vercel-org-id | โœ… | string | | Vercel organisation ID | -| vercel-project-id | โœ… | string | | Vercel project ID | +| Name | Required | Type | Default | Description | +|--------------------|----------|--------|----------|-----------------------------------------| +| vercel-org-id | โœ… | string | | Vercel organisation ID | +| vercel-project-id | โœ… | string | | Vercel project ID | | working-directory | โŒ | string | . | Directory to run the Vercel deploy from | -| environment-name | โŒ | string | Preview | GitHub Environment to deploy to | +| environment-name | โŒ | string | Preview | GitHub Environment to deploy to. Also used as the heading of the pull request comment | + +#### **Outputs** +| Name | Description | +|------|-------------------------------------------------------------------------| +| url | The preview deployment URL. Still building when the job ends, as the deploy uses `--no-wait`. | #### **Secrets** | Name | Required | Description | @@ -69,3 +74,7 @@ jobs: ``` Each project gets its own PR comment, keyed on the project ID. + +To measure the performance of the resulting preview, pair this with +[Vercel Preview Performance](vercel-performance.md), which takes the `url` +output above.