diff --git a/.github/workflows/python-regen-diff.yml b/.github/workflows/python-regen-diff.yml new file mode 100644 index 0000000000..c3a0b9c39c --- /dev/null +++ b/.github/workflows/python-regen-diff.yml @@ -0,0 +1,76 @@ +name: Python Regen Diff + +# Regenerates the typespec-python emitter test output for a PR and renders an +# HTML diff against the assets baseline (assets.json -> the configured assets +# repo). This workflow runs with the (possibly fork) pull_request token, which +# is read-only, so it only *builds* the diff and uploads it as an artifact. The +# shared companion workflow (regen-diff-publish.yml) runs via workflow_run with +# write permissions to publish the HTML to GitHub Pages and post the PR comment. +# No PAT / extra secret is required. + +on: + pull_request: + branches: ["main"] + paths: + - "packages/typespec-python/**" + - ".github/workflows/python-regen-diff.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + render: + name: "Render generated-test diff" + runs-on: ubuntu-latest + if: | + !startsWith(github.head_ref, 'dependabot/') && + !startsWith(github.head_ref, 'publish/') && + !startsWith(github.head_ref, 'backmerge/') && + !startsWith(github.head_ref, 'revert-') + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + + - uses: ./.github/actions/setup + + - uses: ./.github/actions/setup-python + + - name: Install dependencies + run: pnpm install + + - name: Build + run: pnpm turbo run --filter "@azure-tools/typespec-python..." build + + - name: Install regen-diff tool + working-directory: eng/common/scripts/regen-diff + run: npm install --no-package-lock + + - name: Prepare Python environment + run: pnpm run prepare + working-directory: packages/typespec-python + + - name: Regenerate + run: pnpm run regenerate + working-directory: packages/typespec-python + + - name: Render HTML diff + working-directory: packages/typespec-python + env: + DIFF_TITLE: "typespec-python generated test diff (PR #${{ github.event.pull_request.number || 'manual' }})" + run: pnpm run regenerate:render-diff -- --output "${{ runner.temp }}/diff-site" --title "$DIFF_TITLE" + + - name: Record PR metadata + run: echo "${{ github.event.pull_request.number }}" > "${{ runner.temp }}/diff-site/pr.txt" + + - name: Upload diff site + uses: actions/upload-artifact@v7 + with: + name: regen-diff-site + path: ${{ runner.temp }}/diff-site + retention-days: 7 diff --git a/.github/workflows/regen-diff-cleanup.yml b/.github/workflows/regen-diff-cleanup.yml new file mode 100644 index 0000000000..b789a6df7d --- /dev/null +++ b/.github/workflows/regen-diff-cleanup.yml @@ -0,0 +1,70 @@ +name: Regen Diff Cleanup + +# Shared, language-agnostic cleanup. Removes a PR's published diff previews from +# GitHub Pages when the PR is closed. Uses pull_request_target (closed) so it +# runs in the base-repo context with a write token even for fork PRs. It does +# NOT check out or run any PR code — it only deletes the pr// folder from +# gh-pages and updates the sticky comment(s). + +on: + pull_request_target: + types: [closed] + paths: + - "packages/typespec-python/**" + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + cleanup: + name: "Remove diff preview" + runs-on: ubuntu-latest + steps: + - name: Remove PR folder from GitHub Pages + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + url="https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" + tmp="$(mktemp -d)" + if ! git clone --depth 1 --branch gh-pages "$url" "$tmp" 2>/dev/null; then + echo "No gh-pages branch; nothing to clean up." + exit 0 + fi + if [ ! -d "$tmp/pr/${PR}" ]; then + echo "No preview folder for PR #${PR}; nothing to clean up." + exit 0 + fi + git -C "$tmp" rm -r "pr/${PR}" >/dev/null + git -C "$tmp" config user.name "github-actions[bot]" + git -C "$tmp" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$tmp" commit -m "Remove diff previews for closed PR #${PR}" + git -C "$tmp" push origin gh-pages + + - name: Update PR comments + uses: actions/github-script@v7 + env: + PR: ${{ github.event.pull_request.number }} + with: + script: | + const prNumber = Number(process.env.PR); + const owner = context.repo.owner; + const repo = context.repo.repo; + const markerPrefix = '`; + const statsLine = summary.changed + ? `**${files}** files changed · \`+${adds}\` / \`-${dels}\`` + : '✅ No differences from the baseline.'; + const tagLine = tag + ? `Baseline tag: \`${tag}\`` + : '_Baseline not bootstrapped yet — entire output shown as added._'; + const noteLine = note ? `\n> ⚠️ ${note}\n` : ''; + + const body = [ + marker, + `### ${title}`, + '', + statsLine, + '', + `👉 **[View the rendered HTML diff](${pagesUrl})**`, + noteLine, + `${tagLine} · updated for the latest commit. Re-runs of CI refresh this preview.`, + ].join('\n'); + + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: prNumber, per_page: 100, + }); + const existing = comments.find((c) => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + core.info(`Updated comment ${existing.id}`); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); + core.info('Created new diff comment'); + } diff --git a/eng/common/scripts/regen-diff/README.md b/eng/common/scripts/regen-diff/README.md new file mode 100644 index 0000000000..881c802d5c --- /dev/null +++ b/eng/common/scripts/regen-diff/README.md @@ -0,0 +1,155 @@ +# regen-diff — shared generated-test diff previews + +A small, self-contained tool that lets any TypeSpec **emitter language** preview +the impact of a change on its generated test output, the same way the Azure SDK +[test-proxy](https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/README.md) +externalizes recordings. + +Instead of committing a large generated-test tree (or force-pushing it to a side +branch), the **baseline** ("last accepted output") lives in an external, public +**assets repo**, pinned by a tag in an `assets.json` file (test-proxy +convention). On every PR the emitter is regenerated, diffed against the restored +baseline, rendered to **HTML**, published to **GitHub Pages**, and linked from a +**sticky PR comment** plus a `regen-diff/` commit status. + +``` +PR touches packages//** + │ + ▼ +-regen-diff.yml (pull_request, read-only token) + build → regenerate → render HTML diff → upload artifact + │ + ▼ workflow_run (base-repo context, GITHUB_TOKEN write) +regen-diff-publish.yml (shared, language-agnostic) + publish to gh-pages: pr/// · sticky comment · commit status +``` + +Everything uses only the built-in `GITHUB_TOKEN` (no PAT / extra secret) and is +safe for fork PRs. + +## Layout + +| Path | What it is | +| ----------------------------------------------- | ------------------------------------------------------------ | +| `eng/common/scripts/regen-diff/` | This shared tool (one copy, used by every language). | +| `packages//assets.json` | Baseline pointer (assets repo + pinned tag). | +| `packages//regen-diff.config.json` | Per-language config (slug, flavors, generated dir, title). | +| `.github/workflows/-regen-diff.yml` | Thin per-language workflow: build → regenerate → render. | +| `.github/workflows/regen-diff-publish.yml` | Shared: publish to Pages + comment + status. | +| `.github/workflows/regen-diff-cleanup.yml` | Shared: remove a PR's preview on close. | + +## CLI + +``` +tsx eng/common/scripts/regen-diff/src/cli.ts --package [options] +``` + +| Command | Purpose | +| ------------- | ----------------------------------------------------------------------- | +| `render` | Render the HTML diff (current generated output vs the assets baseline). | +| `push-assets` | Publish the current output as a new baseline and bump `assets.json`. | + +`render` options: `--output `, `--generated `, `--title `, +`--open` (browser), `--vscode` (native side-by-side editor diffs), `--max `. +`push-assets` options: `--message `, `--branch `, `--dry-run`. + +The tool has its own `package.json`/`package-lock.json`, so it is **independent +of the consuming package's package manager** (npm, pnpm, …). Install its deps once with `npm install` in this directory before running it. + +## Onboarding a new emitter language + +1. **Pick an assets repo + prefix.** It must be a public repo you can push tags + to. Use a unique `AssetsRepoPrefixPath` so languages don't collide. Add + `packages//assets.json`: + + ```json + { + "AssetsRepo": "/", + "AssetsRepoPrefixPath": "", + "TagPrefix": "/tests", + "Tag": "" + } + ``` + + Leave `Tag` empty until you bootstrap (step 4); the diff then shows the whole + output as "added", which is harmless. + +2. **Describe the output.** Add `packages//regen-diff.config.json`: + + ```json + { + "slug": "", + "title": " emitter — generated test diff", + "generatedDir": "tests/generated", + "flavors": ["azure", "unbranded"], + "pruneFilePrefixes": [] + } + ``` + + - `slug` — URL/identifier-safe, namespaces the Pages folder + status context. + - `flavors` — the top-level folders under `generatedDir` that make up the + baseline (use `["."]` if there's a single flat tree). + - `pruneFilePrefixes` — filename prefixes to drop as transient codegen noise + (e.g. `[".tsp-codegen-"]`); leave `[]` if none. + +3. **Wire up scripts** in `packages//package.json` (paths are relative + to the package; adjust depth as needed): + + ```jsonc + "regenerate:render-diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package .", + "regenerate:diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package . --open", + "regenerate:vscode-diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package . --vscode", + "regenerate:review": " && ", + "regenerate:push-assets": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts push-assets --package ." + ``` + +4. **Bootstrap the baseline** (maintainer, local, one time). Generate the output + and publish the first tag: + + ```bash + + npm install --no-package-lock --prefix eng/common/scripts/regen-diff + # pushes a tag to the assets repo + bumps assets.json + ``` + + Commit the updated `assets.json` (its `Tag` now points at the baseline). + +5. **Add the diff workflow** `.github/workflows/-regen-diff.yml` — copy + `python-regen-diff.yml` and adapt the build/regenerate steps + the + `paths:` filter to your package. It must: + - run `npm install` in `eng/common/scripts/regen-diff` before rendering, + - render with `--output "${{ runner.temp }}/diff-site"`, + - write the PR number to `diff-site/pr.txt`, + - upload the artifact as **`regen-diff-site`**. + +6. **Register with the shared publish workflow.** Add your diff workflow's + `name:` to the `workflow_run.workflows` list in `regen-diff-publish.yml`. + The shared cleanup workflow already matches `packages/http-client-*/**`; add + your package glob there too if it lives elsewhere. + +7. **Enable GitHub Pages** (gh-pages branch) on the repo if it isn't already, + and optionally make `regen-diff/` a required status via branch + protection so a drift without an `assets.json` bump blocks merge. + +## Local developer experience + +```bash +npm install --no-package-lock --prefix eng/common/scripts/regen-diff # one time + # produce fresh output + # render + open the HTML diff +# or: # open native VS Code diffs +``` + +No token is needed — the baseline is restored with an anonymous clone of the +public assets repo. When the changes are intentional, run `regenerate:push-assets` +to refresh the baseline and bump `assets.json`, then commit it so the check +passes with **0 differences**. + +## Notes + +- The renderer is fully generic: it groups changed files by folder, paginates a + side-by-side page per file, and bounds memory by capping per-file render size. +- `assets.ts` here is a self-contained copy of the test-proxy restore helper. + Emitters whose `regenerate` step also seeds a baseline (e.g. legacy smoke + tests) keep their own small `assets.ts`; this tool does not depend on it. + diff --git a/eng/common/scripts/regen-diff/package.json b/eng/common/scripts/regen-diff/package.json new file mode 100644 index 0000000000..6c4f25edcc --- /dev/null +++ b/eng/common/scripts/regen-diff/package.json @@ -0,0 +1,20 @@ +{ + "name": "@typespec/internal-regen-diff", + "version": "0.1.0", + "private": true, + "description": "Shared tool to render generated-test diffs against an external assets baseline and publish new baselines. Used by the TypeSpec emitter languages.", + "type": "module", + "scripts": { + "regen-diff": "tsx ./src/cli.ts", + "build": "tsc -p .", + "test:tsc": "tsc -p ." + }, + "dependencies": { + "picocolors": "~1.1.1" + }, + "devDependencies": { + "@types/node": "~25.0.2", + "tsx": "^4.21.0", + "typescript": "~5.9.2" + } +} diff --git a/eng/common/scripts/regen-diff/src/assets.ts b/eng/common/scripts/regen-diff/src/assets.ts new file mode 100644 index 0000000000..02ee1d6d62 --- /dev/null +++ b/eng/common/scripts/regen-diff/src/assets.ts @@ -0,0 +1,145 @@ +/* eslint-disable no-console */ +/** + * Test-proxy-style baseline assets helpers, shared across emitter languages. + * + * The generated-test baseline ("the last accepted regeneration output") is + * stored in an external, public assets repo and pinned by a tag, following the + * Azure SDK test-proxy `assets.json` convention. Restoring the baseline is an + * anonymous clone of that public repo, so it needs **no token**. + * + * `assets.json` (next to a package's `package.json`) looks like: + * { + * "AssetsRepo": "Azure/azure-sdk-assets", + * "AssetsRepoPrefixPath": "typespec/python", + * "TagPrefix": "typespec/python/tests", + * "Tag": "typespec/python/tests_" + * } + * + * In the assets repo the baseline lives under + * / + * at the commit pointed to by , one folder per configured flavor. + */ + +import { execSync } from "child_process"; +import { existsSync, readFileSync, rmSync } from "fs"; +import { cp, mkdir, mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { join, resolve } from "path"; +import pc from "picocolors"; + +/** Parsed `assets.json` describing where the baseline lives. */ +export interface AssetsConfig { + /** "owner/repo" of the public assets repo. */ + assetsRepo: string; + /** Subdirectory in the assets repo that contains the language baseline. */ + prefixPath: string; + /** Tag namespace, e.g. "typespec/python/tests". */ + tagPrefix: string; + /** Concrete tag the baseline is pinned to (empty until first bootstrap). */ + tag: string; + /** Absolute path to the `assets.json` file this was read from. */ + configPath: string; +} + +/** + * Reads `assets.json` from `packageRoot`. Returns `undefined` if the file does + * not exist or has no `AssetsRepo`, so callers can treat that as "no baseline". + */ +export function readAssetsConfig(packageRoot: string): AssetsConfig | undefined { + const configPath = resolve(packageRoot, "assets.json"); + if (!existsSync(configPath)) { + return undefined; + } + + const raw = JSON.parse(readFileSync(configPath, "utf8")) as Record; + const assetsRepo = raw.AssetsRepo?.trim() ?? ""; + if (!assetsRepo) { + return undefined; + } + + return { + assetsRepo, + prefixPath: (raw.AssetsRepoPrefixPath ?? "").trim(), + tagPrefix: (raw.TagPrefix ?? "").trim(), + tag: (raw.Tag ?? "").trim(), + configPath, + }; +} + +/** HTTPS URL for the public assets repo (anonymous, no credentials). */ +export function assetsRepoUrl(config: AssetsConfig): string { + return `https://github.com/${config.assetsRepo}.git`; +} + +/** + * Anonymously checks out the assets repo at `config.tag` into a fresh temp dir + * and invokes `handler` with the absolute path to `/` (the + * directory that contains the per-flavor baseline folders). The temp dir is + * always cleaned up afterwards. + * + * Throws if `config.tag` is empty (baseline not bootstrapped yet). + */ +export async function withBaselineCheckout( + config: AssetsConfig, + handler: (baselineRoot: string) => Promise, +): Promise { + if (!config.tag) { + throw new Error( + `assets.json has an empty "Tag"; bootstrap the baseline first ` + + `(run " regenerate:push-assets").`, + ); + } + + const repoUrl = assetsRepoUrl(config); + const tempDir = await mkdtemp(join(tmpdir(), "typespec-assets-")); + + try { + console.log(pc.dim(`Cloning ${config.assetsRepo}@${config.tag} into ${tempDir}`)); + const run = (cmd: string) => + execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); + + run(`git init`); + run(`git config core.longpaths true`); + run(`git remote add origin ${repoUrl}`); + if (config.prefixPath) { + run(`git config core.sparseCheckout true`); + run(`git sparse-checkout init --cone`); + run(`git sparse-checkout set ${config.prefixPath}`); + } + // Fetch the specific tag shallowly. Tags are fetched under refs/tags/. + run(`git fetch --depth 1 origin "refs/tags/${config.tag}:refs/tags/${config.tag}"`); + run(`git checkout "tags/${config.tag}"`); + + const baselineRoot = config.prefixPath + ? join(tempDir, ...config.prefixPath.split("/")) + : tempDir; + return await handler(baselineRoot); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +/** + * Restores the **full** baseline (every configured flavor folder) from the + * assets repo into `destDir` as `destDir/`. `destDir/` is wiped + * before copying. Used by the diff renderer to obtain the "before" snapshot. + */ +export async function restoreFullBaseline( + config: AssetsConfig, + destDir: string, + flavors: string[], +): Promise { + await withBaselineCheckout(config, async (baselineRoot) => { + for (const flavor of flavors) { + const src = join(baselineRoot, flavor); + const dest = join(destDir, flavor); + if (!existsSync(src)) { + console.warn(pc.yellow(`Baseline flavor folder not found in assets repo: ${flavor}`)); + continue; + } + rmSync(dest, { recursive: true, force: true }); + await mkdir(dest, { recursive: true }); + await cp(src, dest, { recursive: true }); + } + }); +} diff --git a/eng/common/scripts/regen-diff/src/cli.ts b/eng/common/scripts/regen-diff/src/cli.ts new file mode 100644 index 0000000000..5a9cd536f9 --- /dev/null +++ b/eng/common/scripts/regen-diff/src/cli.ts @@ -0,0 +1,118 @@ +/* eslint-disable no-console */ +/** + * Shared regen-diff CLI. One self-contained tool used by every emitter language + * to render generated-test diffs against an external assets baseline and to + * publish a new baseline. + * + * Commands: + * render Render the HTML diff (current generated output vs baseline). + * push-assets Publish the current output as a new baseline + bump assets.json. + * + * Every command takes `--package `, the emitter package root that holds the + * `regen-diff.config.json` and `assets.json`. Run it with `tsx`: + * + * tsx eng/common/scripts/regen-diff/src/cli.ts render --package packages/http-client-python + * + * Most consumers wire this up behind package.json scripts (see the README). + */ + +import pc from "picocolors"; +import { parseArgs } from "util"; + +import { pushAssets } from "./push-assets.js"; +import { render } from "./render-diff.js"; + +const HELP = `${pc.bold("regen-diff")} — shared generated-test diff tool + +${pc.bold("Usage:")} + regen-diff --package [options] + +${pc.bold("Commands:")} + render Render an HTML diff of the current generated output vs the + assets baseline. + --package Emitter package root (required). + --output Output directory (default: /temp/diff-site). + --generated Current generated dir (default: from config). + --title Title shown on the diff page (default: from config). + --open Open the rendered diff in your default browser. + --vscode Open each changed file as a native VS Code editor diff. + --max Max files to open in VS Code (default 40). + + push-assets Publish the current generated output as a new baseline and bump + assets.json to the new tag (maintainer-run, local). + --package Emitter package root (required). + --message Commit message (default auto-generated). + --branch Assets-repo branch to push to (default: main). + --dry-run Build the commit/tag locally but do not push. + + -h, --help Show this help. +`; + +async function main(): Promise { + const [command, ...rest] = process.argv.slice(2); + + if (!command || command === "-h" || command === "--help") { + console.log(HELP); + process.exit(command ? 0 : 1); + } + + const { values } = parseArgs({ + args: rest, + options: { + package: { type: "string", short: "p" }, + output: { type: "string", short: "o" }, + generated: { type: "string", short: "g" }, + title: { type: "string", short: "t" }, + open: { type: "boolean" }, + vscode: { type: "boolean" }, + max: { type: "string" }, + message: { type: "string", short: "m" }, + branch: { type: "string", short: "b" }, + "dry-run": { type: "boolean" }, + help: { type: "boolean", short: "h" }, + }, + }); + + if (values.help) { + console.log(HELP); + process.exit(0); + } + + const packageRoot = values.package; + if (!packageRoot) { + console.error(pc.red(`Missing required --package .`)); + console.log(HELP); + process.exit(1); + } + + switch (command) { + case "render": + await render({ + packageRoot, + output: values.output, + generated: values.generated, + title: values.title, + open: values.open, + vscode: values.vscode, + max: values.max ? Number(values.max) : undefined, + }); + break; + case "push-assets": + await pushAssets({ + packageRoot, + message: values.message, + branch: values.branch, + dryRun: values["dry-run"], + }); + break; + default: + console.error(pc.red(`Unknown command: ${command}`)); + console.log(HELP); + process.exit(1); + } +} + +main().catch((err) => { + console.error(pc.red(`Fatal error: ${err?.stack ?? err}`)); + process.exit(1); +}); diff --git a/eng/common/scripts/regen-diff/src/config.ts b/eng/common/scripts/regen-diff/src/config.ts new file mode 100644 index 0000000000..a14f65465c --- /dev/null +++ b/eng/common/scripts/regen-diff/src/config.ts @@ -0,0 +1,97 @@ +/* eslint-disable no-console */ +/** + * Loads a package's `regen-diff.config.json`, the per-language configuration for + * the shared regen-diff tool. It sits next to the package's test-proxy-style + * `assets.json` (which points at the external baseline) and describes the bits + * that are specific to one emitter: where the generated output lives, which + * top-level "flavor" folders make it up, and how to label the rendered diff. + * + * Example `regen-diff.config.json`: + * { + * "slug": "python", + * "title": "Python emitter — generated test diff", + * "generatedDir": "tests/generated", + * "flavors": ["azure", "unbranded"], + * "pruneFilePrefixes": [".tsp-codegen-"] + * } + */ + +import { existsSync, readFileSync } from "fs"; +import { resolve } from "path"; + +/** Resolved per-language regen-diff configuration. */ +export interface RegenDiffConfig { + /** + * Short, URL/identifier-safe language slug (e.g. "python", "java", "csharp"). + * Used to namespace the Pages preview (`pr///`), the commit-status + * context (`regen-diff/`) and the sticky PR comment marker. + */ + slug: string; + /** Human-readable title shown on the diff page and the PR comment. */ + title: string; + /** Generated-output directory, relative to the package root. */ + generatedDir: string; + /** Top-level folders under `generatedDir` that make up the baseline. */ + flavors: string[]; + /** + * Filename prefixes to drop from both sides before diffing (transient codegen + * artifacts that embed machine-local paths and would otherwise be diff noise). + */ + pruneFilePrefixes: string[]; + /** Absolute path to the package root the config was loaded from. */ + packageRoot: string; +} + +const SLUG_RE = /^[a-z0-9][a-z0-9-]*$/; + +/** + * Reads and validates `/regen-diff.config.json`. Throws a friendly + * error if the file is missing or malformed, since every command needs it. + */ +export function readRegenDiffConfig(packageRoot: string): RegenDiffConfig { + const root = resolve(packageRoot); + const configPath = resolve(root, "regen-diff.config.json"); + if (!existsSync(configPath)) { + throw new Error( + `No regen-diff.config.json found at ${configPath}. ` + + `See eng/common/scripts/regen-diff/README.md for the onboarding steps.`, + ); + } + + const raw = JSON.parse(readFileSync(configPath, "utf8")) as Record; + + const slug = typeof raw.slug === "string" ? raw.slug.trim() : ""; + if (!SLUG_RE.test(slug)) { + throw new Error( + `regen-diff.config.json: "slug" must match ${SLUG_RE} (got ${JSON.stringify(raw.slug)}).`, + ); + } + + const flavors = Array.isArray(raw.flavors) ? raw.flavors.map((f) => String(f)) : []; + if (flavors.length === 0) { + throw new Error(`regen-diff.config.json: "flavors" must be a non-empty array of folder names.`); + } + + const pruneFilePrefixes = Array.isArray(raw.pruneFilePrefixes) + ? raw.pruneFilePrefixes.map((p) => String(p)) + : []; + + const title = + typeof raw.title === "string" && raw.title.trim() + ? raw.title.trim() + : `${slug} emitter — generated test diff`; + + const generatedDir = + typeof raw.generatedDir === "string" && raw.generatedDir.trim() + ? raw.generatedDir.trim() + : "tests/generated"; + + return { + slug, + title, + generatedDir, + flavors, + pruneFilePrefixes, + packageRoot: root, + }; +} diff --git a/eng/common/scripts/regen-diff/src/push-assets.ts b/eng/common/scripts/regen-diff/src/push-assets.ts new file mode 100644 index 0000000000..f120e4e4ea --- /dev/null +++ b/eng/common/scripts/regen-diff/src/push-assets.ts @@ -0,0 +1,201 @@ +/* eslint-disable no-console */ +/** + * Publishes the current generated output as a new baseline in the external + * assets repo and bumps `assets.json` to the new tag. Shared across emitter + * languages; the generated dir, flavor folders and commit-label come from the + * package's `regen-diff.config.json`. + * + * This is a **maintainer-run, local** tool. It pushes to the assets repo using + * whatever git credentials are already configured on the machine (or a + * `GH_TOKEN`/`GITHUB_TOKEN` env var if present), so it needs **no CI secret**. + * + * Typical flow: + * 1. # produce fresh generated output + * 2. regenerate:push-assets # publish it + bump assets.json + * 3. commit the assets.json change and open a PR + * + * Usage: + * push-assets --package [--message ""] [--branch ] [--dry-run] + */ + +import { execFileSync } from "child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { cp, mkdir, mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import pc from "picocolors"; + +import { assetsRepoUrl, readAssetsConfig } from "./assets.js"; +import { readRegenDiffConfig } from "./config.js"; + +export interface PushOptions { + packageRoot: string; + message?: string; + branch?: string; + dryRun?: boolean; +} + +/** Returns an authenticated push URL when a token is in the environment. */ +function pushUrl(repoSlug: string): string { + const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; + if (token) { + return `https://x-access-token:${token}@github.com/${repoSlug}.git`; + } + return `https://github.com/${repoSlug}.git`; +} + +/** + * Recursively rewrites every text file under `dir` with all `\r` bytes removed, + * so the stored baseline is pure LF regardless of the maintainer's OS. Files + * containing a NUL byte are treated as binary and left untouched. + */ +function normalizeEol(dir: string): void { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + normalizeEol(full); + } else if (entry.isFile()) { + const buf = readFileSync(full); + if (buf.includes(0)) continue; // binary + if (buf.includes(0x0d)) { + writeFileSync(full, buf.toString("utf8").replace(/\r/g, "")); + } + } + } +} + +/** + * Removes transient codegen handoff files (configured via `pruneFilePrefixes`) + * that embed absolute, machine-local paths and must never be published into the + * portable baseline. + */ +function pruneIntermediates(dir: string, prefixes: string[]): void { + if (prefixes.length === 0) return; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + pruneIntermediates(full, prefixes); + } else if (entry.isFile() && prefixes.some((p) => entry.name.startsWith(p))) { + rmSync(full, { force: true }); + } + } +} + +/** Publishes the current generated output as a new baseline + tag. */ +export async function pushAssets(options: PushOptions): Promise { + const cfg = readRegenDiffConfig(options.packageRoot); + const generatedDir = join(cfg.packageRoot, ...cfg.generatedDir.split("/")); + const branch = options.branch ?? "main"; + const dryRun = options.dryRun ?? false; + + const config = readAssetsConfig(cfg.packageRoot); + if (!config) { + throw new Error(`No usable assets.json found at ${cfg.packageRoot}/assets.json`); + } + + // Validate the generated output is present. + for (const flavor of cfg.flavors) { + if (!existsSync(join(generatedDir, flavor))) { + throw new Error( + `Missing ${join(generatedDir, flavor)}. Run the package's regenerate step before pushing assets.`, + ); + } + } + + const repoUrl = assetsRepoUrl(config); + const tempDir = await mkdtemp(join(tmpdir(), "typespec-assets-push-")); + + try { + const git = (args: string[], opts: { allowFail?: boolean } = {}) => { + try { + return execFileSync("git", args, { + cwd: tempDir, + stdio: ["ignore", "pipe", "inherit"], + encoding: "utf8", + }).trim(); + } catch (err) { + if (opts.allowFail) return ""; + throw err; + } + }; + + console.log(pc.cyan(`Preparing assets push to ${config.assetsRepo} (branch ${branch})`)); + + git(["init"]); + git(["config", "core.longpaths", "true"]); + // Store the baseline with LF regardless of the maintainer's OS. + git(["config", "core.autocrlf", "false"]); + git(["remote", "add", "origin", repoUrl]); + + // Try to base the new commit on the existing branch; if the repo/branch is + // empty this fails harmlessly and we start an orphan history. + const fetched = git(["fetch", "--depth", "1", "origin", branch], { allowFail: true }); + if (fetched !== "" || git(["rev-parse", "--verify", "FETCH_HEAD"], { allowFail: true })) { + git(["checkout", "-B", branch, "FETCH_HEAD"], { allowFail: true }); + } + if (!git(["rev-parse", "--verify", "HEAD"], { allowFail: true })) { + git(["checkout", "--orphan", branch], { allowFail: true }); + } + + // Replace the per-flavor baseline under / with the fresh output. + const prefixRoot = config.prefixPath ? join(tempDir, ...config.prefixPath.split("/")) : tempDir; + for (const flavor of cfg.flavors) { + const dest = join(prefixRoot, flavor); + rmSync(dest, { recursive: true, force: true }); + await mkdir(dirname(dest), { recursive: true }); + await cp(join(generatedDir, flavor), dest, { recursive: true }); + } + + // Normalize line endings to LF in the committed blobs. + writeFileSync(join(tempDir, ".gitattributes"), "* text=auto eol=lf\n"); + normalizeEol(prefixRoot); + pruneIntermediates(prefixRoot, cfg.pruneFilePrefixes); + + git(["add", "-A"]); + const status = git(["status", "--porcelain"], { allowFail: true }); + if (!status) { + console.log(pc.green("Baseline already up to date in the assets repo; nothing to push.")); + return; + } + + const message = + options.message ?? + `[${cfg.slug}] Update generated test baseline (${new Date().toISOString()})`; + git(["config", "user.name", process.env.GIT_AUTHOR_NAME || "typespec-assets-bot"]); + git([ + "config", + "user.email", + process.env.GIT_AUTHOR_EMAIL || "typespec-assets-bot@users.noreply.github.com", + ]); + git(["commit", "-m", message]); + + const shortSha = git(["rev-parse", "--short=10", "HEAD"]); + const tag = `${config.tagPrefix}_${shortSha}`; + git(["tag", tag]); + + if (dryRun) { + console.log(pc.yellow(`[dry-run] Would push branch ${branch} and tag ${tag}.`)); + } else { + const authUrl = pushUrl(config.assetsRepo); + git(["remote", "set-url", "origin", authUrl]); + git(["push", "origin", `HEAD:${branch}`]); + git(["push", "origin", tag]); + console.log(pc.green(`Pushed baseline and tag ${tag} to ${config.assetsRepo}.`)); + } + + // Bump assets.json's Tag locally so the maintainer can commit it. + bumpAssetsTag(config.configPath, tag); + console.log(pc.green(`Updated ${config.configPath} -> "Tag": "${tag}"`)); + console.log(pc.cyan(`Next: commit assets.json and open a PR.`)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +/** Rewrites only the `Tag` field of assets.json, preserving the other fields. */ +function bumpAssetsTag(configPath: string, tag: string): void { + const raw = readFileSync(configPath, "utf8"); + const parsed = JSON.parse(raw) as Record; + parsed.Tag = tag; + writeFileSync(configPath, JSON.stringify(parsed, null, 2) + "\n"); +} diff --git a/eng/common/scripts/regen-diff/src/render-diff.ts b/eng/common/scripts/regen-diff/src/render-diff.ts new file mode 100644 index 0000000000..85848c9ef8 --- /dev/null +++ b/eng/common/scripts/regen-diff/src/render-diff.ts @@ -0,0 +1,779 @@ +/* eslint-disable no-console */ +/** + * Renders an HTML diff between the **assets baseline** (the last accepted + * regeneration output, restored from the assets repo) and the **current** + * generated output (produced by the package's regenerate step beforehand). + * + * This is the shared, language-agnostic renderer. The per-language specifics + * (generated dir, flavor folders, title, transient files to prune) come from + * the package's `regen-diff.config.json`; the baseline pointer comes from its + * `assets.json`. + * + * Output (default `/temp/diff-site/`): + * - index.html A folder-grouped, searchable tree of changed files. + * - files/NN.html One side-by-side page per changed file (diff2html). + * - summary.json { slug, title, changed, filesChanged, additions, + * deletions, baselineTag, note } for the publish step. + * + * Restoring the baseline is an anonymous clone of the public assets repo, so + * this needs no token. If assets.json has no Tag yet (not bootstrapped), the + * whole current output is treated as "added". + * + * Usage: + * render-diff --package [--output ] [--title ] [--open] + * [--vscode] [--max ] + */ + +import { execFileSync, execSync } from "child_process"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { cp, mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { join, resolve } from "path"; +import pc from "picocolors"; +import { pathToFileURL } from "url"; + +import { readAssetsConfig, restoreFullBaseline } from "./assets.js"; +import { readRegenDiffConfig, type RegenDiffConfig } from "./config.js"; + + +export interface RenderOptions { + packageRoot: string; + output?: string; + generated?: string; + title?: string; + open?: boolean; + vscode?: boolean; + max?: number; +} + +// Each changed file is rendered as its own page, so we never build one giant +// HTML string (which throws `RangeError: Invalid string length` past ~512MB). +// A single file whose diff exceeds this is shown as a raw
 instead.
+const MAX_FILE_DIFF_BYTES = 2 * 1024 * 1024;
+
+interface DiffSummary {
+  slug: string;
+  title: string;
+  changed: boolean;
+  filesChanged: number;
+  additions: number;
+  deletions: number;
+  baselineTag: string;
+  note?: string;
+}
+
+// Resolved once per run from options + config; read by the render helpers.
+let CONFIG: RegenDiffConfig;
+let GENERATED_DIR: string;
+let OUTPUT_DIR: string;
+let TITLE: string;
+
+function git(args: string[], cwd: string, allowFail = false): string {
+  try {
+    return execFileSync("git", args, {
+      cwd,
+      stdio: ["ignore", "pipe", "ignore"],
+      encoding: "utf8",
+      maxBuffer: 256 * 1024 * 1024,
+    });
+  } catch (err) {
+    if (allowFail) {
+      const e = err as { stdout?: string };
+      return e.stdout ?? "";
+    }
+    throw err;
+  }
+}
+
+/** Removes carriage returns so diff2html doesn't render literal `^M` markers. */
+function stripCr(text: string): string {
+  return text.replace(/\r/g, "");
+}
+
+/**
+ * Recursively rewrites every text file under `dir` with all `\r` bytes removed,
+ * so line endings are pure LF. The baseline may have been generated on Windows,
+ * where writing `\r\n` to a text-mode file yields `\r\r\n` (double CR);
+ * `git diff --ignore-cr-at-eol` only ignores a *single* trailing CR, so without
+ * this those lines show as spurious changes. Normalizing both trees to LF makes
+ * the comparison truly line-ending agnostic. Files containing a NUL byte are
+ * treated as binary and left untouched.
+ */
+function normalizeEol(dir: string): void {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      normalizeEol(full);
+    } else if (entry.isFile()) {
+      const buf = readFileSync(full);
+      if (buf.includes(0)) continue; // binary
+      if (buf.includes(0x0d)) {
+        writeFileSync(full, buf.toString("utf8").replace(/\r/g, ""));
+      }
+    }
+  }
+}
+
+/**
+ * Removes transient codegen handoff files (configured via `pruneFilePrefixes`)
+ * from a tree. These embed absolute machine-local paths and are not real
+ * generated output, so they would otherwise show up as noise in the diff.
+ */
+function pruneIntermediates(dir: string): void {
+  if (CONFIG.pruneFilePrefixes.length === 0) return;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      pruneIntermediates(full);
+    } else if (entry.isFile() && CONFIG.pruneFilePrefixes.some((p) => entry.name.startsWith(p))) {
+      rmSync(full, { force: true });
+    }
+  }
+}
+
+/** Parses `git diff --numstat` output into aggregate counts. */
+function parseNumstat(numstat: string): { files: number; additions: number; deletions: number } {
+  let files = 0;
+  let additions = 0;
+  let deletions = 0;
+  for (const line of numstat.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const [add, del] = trimmed.split("\t");
+    files += 1;
+    // Binary files report "-" for counts.
+    if (add !== "-") additions += Number(add) || 0;
+    if (del !== "-") deletions += Number(del) || 0;
+  }
+  return { files, additions, deletions };
+}
+
+/** Renders the HTML diff for one package. Returns the produced summary. */
+export async function render(options: RenderOptions): Promise {
+  CONFIG = readRegenDiffConfig(options.packageRoot);
+  GENERATED_DIR = options.generated
+    ? resolve(options.generated)
+    : resolve(CONFIG.packageRoot, CONFIG.generatedDir);
+  OUTPUT_DIR = options.output
+    ? resolve(options.output)
+    : resolve(CONFIG.packageRoot, "temp/diff-site");
+  TITLE = options.title ?? CONFIG.title;
+
+  // Validate current output exists.
+  for (const flavor of CONFIG.flavors) {
+    if (!existsSync(join(GENERATED_DIR, flavor))) {
+      throw new Error(
+        `Missing ${join(GENERATED_DIR, flavor)}. Run the package's regenerate step before render-diff.`,
+      );
+    }
+  }
+
+  const config = readAssetsConfig(CONFIG.packageRoot);
+  const baselineTag = config?.tag ?? "";
+
+  const workDir = await mkdtemp(join(tmpdir(), "typespec-diff-"));
+  const baselineDir = join(workDir, "baseline");
+  const currentDir = join(workDir, "current");
+
+  try {
+    // "current" = freshly regenerated output.
+    await cp(GENERATED_DIR, currentDir, { recursive: true });
+
+    // "baseline" = last accepted output from the assets repo (empty if no tag).
+    mkdirSync(baselineDir, { recursive: true });
+    let note: string | undefined;
+    if (config && config.tag) {
+      console.log(pc.cyan(`Restoring baseline ${config.assetsRepo}@${config.tag}...`));
+      await restoreFullBaseline(config, baselineDir, CONFIG.flavors);
+    } else {
+      note =
+        "No baseline tag is configured in assets.json yet; the entire current output is shown as added.";
+      console.warn(pc.yellow(note));
+    }
+
+    // Normalize line endings to LF on both sides so EOL artifacts don't
+    // masquerade as real diffs.
+    normalizeEol(currentDir);
+    normalizeEol(baselineDir);
+
+    // Drop transient codegen handoff files so they never appear in the diff.
+    pruneIntermediates(currentDir);
+    pruneIntermediates(baselineDir);
+
+    // git diff --no-index returns exit code 1 when there are differences.
+    // --ignore-cr-at-eol makes the diff line-ending agnostic.
+    const diffArgs = (extra: string[]) => [
+      "-c",
+      "core.quotepath=false",
+      "diff",
+      "--no-index",
+      ...extra,
+      "--ignore-cr-at-eol",
+      "--",
+      "baseline",
+      "current",
+    ];
+    const diffText = stripCr(git(diffArgs(["--no-color"]), workDir, true));
+    const numstat = git(diffArgs(["--numstat"]), workDir, true);
+    const counts = parseNumstat(numstat);
+
+    const summary: DiffSummary = {
+      slug: CONFIG.slug,
+      title: TITLE,
+      changed: diffText.trim().length > 0,
+      filesChanged: counts.files,
+      additions: counts.additions,
+      deletions: counts.deletions,
+      baselineTag,
+      note,
+    };
+
+    rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    mkdirSync(OUTPUT_DIR, { recursive: true });
+    writeFileSync(join(OUTPUT_DIR, "summary.json"), JSON.stringify(summary, null, 2) + "\n");
+    writeSite(diffText, summary);
+
+    const indexPath = join(OUTPUT_DIR, "index.html");
+    console.log(
+      pc.green(
+        `Diff rendered to ${OUTPUT_DIR} ` +
+          `(${summary.filesChanged} files, +${summary.additions}/-${summary.deletions}).`,
+      ),
+    );
+    console.log(pc.cyan(`View it at ${pathToFileURL(indexPath).href}`));
+    if (options.open) {
+      openInBrowser(indexPath);
+    }
+
+    // Open the diff natively in VS Code: persist both normalized trees to a
+    // stable path and pop a side-by-side editor for each changed file. The
+    // generated output is git-ignored, so it never shows in Source Control.
+    if (options.vscode) {
+      const treesDir = resolve(CONFIG.packageRoot, "temp/diff-trees");
+      const baselineOut = join(treesDir, "baseline");
+      const currentOut = join(treesDir, "current");
+      rmSync(treesDir, { recursive: true, force: true });
+      mkdirSync(treesDir, { recursive: true });
+      await cp(baselineDir, baselineOut, { recursive: true });
+      await cp(currentDir, currentOut, { recursive: true });
+      const max = options.max && options.max > 0 ? options.max : 40;
+      openInVscode(diffText, baselineOut, currentOut, summary.filesChanged, max);
+    }
+
+    return summary;
+  } finally {
+    rmSync(workDir, { recursive: true, force: true });
+  }
+}
+
+/** Opens a local file in the OS default browser; never fails the run. */
+function openInBrowser(target: string): void {
+  try {
+    if (process.platform === "win32") {
+      execFileSync("cmd", ["/c", "start", "", target], { stdio: "ignore" });
+    } else if (process.platform === "darwin") {
+      execFileSync("open", [target], { stdio: "ignore" });
+    } else {
+      execFileSync("xdg-open", [target], { stdio: "ignore" });
+    }
+  } catch (err) {
+    console.warn(pc.yellow(`Could not open a browser automatically: ${err}`));
+  }
+}
+
+/**
+ * Parses `diff --git a/baseline/ b/current/` header lines to recover
+ * the per-file relative paths that changed.
+ */
+function changedRelPaths(diffText: string): string[] {
+  const paths: string[] = [];
+  for (const line of diffText.split("\n")) {
+    const m = /^diff --git a\/baseline\/(.+?) b\/current\/(.+)$/.exec(line);
+    if (m) paths.push(m[2]);
+  }
+  return paths;
+}
+
+/**
+ * Opens each changed file as a native VS Code editor diff (`code --diff
+ *  `). Added/removed files (one side missing) are opened on
+ * their own. Capped at `max` tabs.
+ */
+function openInVscode(
+  diffText: string,
+  baselineDir: string,
+  currentDir: string,
+  filesChanged: number,
+  max: number,
+): void {
+  const rels = changedRelPaths(diffText);
+
+  console.log(pc.cyan(`Baseline tree:  ${baselineDir}`));
+  console.log(pc.cyan(`Current tree:   ${currentDir}`));
+
+  if (rels.length === 0) {
+    console.log(pc.green("No changed files — nothing to open in VS Code."));
+    return;
+  }
+
+  // `code` is a shell script / .cmd on most platforms, so it must be launched
+  // through a shell (Node refuses to execFile a .cmd directly). Compose a single
+  // quoted command string so the shell parses paths with spaces correctly.
+  const q = (p: string) => `"${p.replace(/"/g, '\\"')}"`;
+  const code = (parts: string[]) => execSync(["code", ...parts].join(" "), { stdio: "ignore" });
+
+  const toOpen = rels.slice(0, max);
+  if (rels.length > max) {
+    console.warn(
+      pc.yellow(
+        `${filesChanged} files changed; opening the first ${max} in VS Code. ` +
+          `Use --max  to open more, or the HTML page for all of them.`,
+      ),
+    );
+  }
+
+  let opened = 0;
+  for (const rel of toOpen) {
+    const baseFile = join(baselineDir, rel);
+    const curFile = join(currentDir, rel);
+    const hasBase = existsSync(baseFile);
+    const hasCur = existsSync(curFile);
+    try {
+      if (hasBase && hasCur) {
+        code(["--diff", q(baseFile), q(curFile)]);
+      } else if (hasCur) {
+        code([q(curFile)]); // added
+      } else if (hasBase) {
+        code([q(baseFile)]); // removed
+      }
+      opened += 1;
+    } catch (err) {
+      console.warn(
+        pc.yellow(
+          `Could not launch VS Code (is the "code" command on PATH?). ` +
+            `Compare the two trees above manually. Details: ${err}`,
+        ),
+      );
+      return;
+    }
+  }
+  console.log(pc.green(`Opened ${opened} file diff(s) in VS Code.`));
+}
+
+interface FileDiff {
+  /** Display path (baseline/current prefixes stripped). */
+  path: string;
+  /** Raw unified-diff chunk for just this file. */
+  chunk: string;
+  additions: number;
+  deletions: number;
+  status: "added" | "removed" | "modified";
+}
+
+/**
+ * Rewrites a file chunk's diff header so both sides share the same path.
+ *
+ * The diff comes from `git diff --no-index baseline current`, so every header
+ * reads `a/baseline/` vs `b/current/`. Because those two paths
+ * differ only by the temp-dir prefix, diff2html mistakes every file for a
+ * RENAME. Stripping the `baseline/`/`current/` prefixes makes old === new path,
+ * so it renders as a normal modification.
+ */
+function normalizeChunkHeader(chunk: string): string {
+  const lines = chunk.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.startsWith("@@")) break; // header is done once hunks begin
+    if (line.startsWith("diff --git ")) {
+      lines[i] = line.replace(/ a\/baseline\//g, " a/").replace(/ b\/current\//g, " b/");
+    } else if (line.startsWith("--- ")) {
+      lines[i] = line.replace(/^--- a\/baseline\//, "--- a/");
+    } else if (line.startsWith("+++ ")) {
+      lines[i] = line.replace(/^\+\+\+ b\/current\//, "+++ b/");
+    } else if (line.startsWith("rename from ")) {
+      lines[i] = line.replace(/^rename from baseline\//, "rename from ");
+    } else if (line.startsWith("rename to ")) {
+      lines[i] = line.replace(/^rename to current\//, "rename to ");
+    }
+  }
+  return lines.join("\n");
+}
+
+/** Splits a `git diff --no-index` blob into one chunk per file. */
+function splitDiffByFile(diffText: string): FileDiff[] {
+  const files: FileDiff[] = [];
+  const sections = diffText.split(/(?=^diff --git )/m).filter((s) => s.startsWith("diff --git "));
+  for (const chunk of sections) {
+    const lines = chunk.split("\n");
+    let oldPath = "";
+    let newPath = "";
+    let additions = 0;
+    let deletions = 0;
+    for (const line of lines) {
+      if (line.startsWith("--- ")) {
+        oldPath = line.slice(4).trim();
+      } else if (line.startsWith("+++ ")) {
+        newPath = line.slice(4).trim();
+      } else if (line.startsWith("+") && !line.startsWith("+++")) {
+        additions += 1;
+      } else if (line.startsWith("-") && !line.startsWith("---")) {
+        deletions += 1;
+      }
+    }
+    const strip = (p: string): string =>
+      p
+        .replace(/^["ab]\//, "")
+        .replace(/^a\//, "")
+        .replace(/^b\//, "")
+        .replace(/^baseline\//, "")
+        .replace(/^current\//, "");
+    const isAdded = oldPath === "/dev/null";
+    const isRemoved = newPath === "/dev/null";
+    const display = strip(isAdded ? newPath : oldPath) || strip(newPath) || "(unknown)";
+    files.push({
+      path: display,
+      chunk: normalizeChunkHeader(chunk),
+      additions,
+      deletions,
+      status: isAdded ? "added" : isRemoved ? "removed" : "modified",
+    });
+  }
+  files.sort((a, b) => a.path.localeCompare(b.path));
+  return files;
+}
+
+/** Writes the full multi-page diff site to OUTPUT_DIR. */
+function writeSite(diffText: string, summary: DiffSummary): void {
+  writeFileSync(join(OUTPUT_DIR, "styles.css"), SITE_CSS);
+
+  if (!summary.changed) {
+    writeFileSync(
+      join(OUTPUT_DIR, "index.html"),
+      pageShell(
+        TITLE,
+        headerHtml(summary),
+        `
✅ No differences from the baseline.
`, + ".", + ), + ); + return; + } + + // Keep a full raw diff available for download. + writeFileSync(join(OUTPUT_DIR, "diff.txt"), diffText); + + const files = splitDiffByFile(diffText); + const filesDir = join(OUTPUT_DIR, "files"); + mkdirSync(filesDir, { recursive: true }); + + const pad = String(files.length).length; + files.forEach((file, i) => { + const name = `${String(i + 1).padStart(pad, "0")}.html`; + writeFileSync(join(filesDir, name), renderFilePage(file, files, i)); + }); + + writeFileSync(join(OUTPUT_DIR, "index.html"), renderIndexPage(files, summary, pad)); +} + +/** Index page: a folder-grouped, searchable tree of all changed files. */ +function renderIndexPage(files: FileDiff[], summary: DiffSummary, pad: number): string { + const root = buildTree(files, pad); + const tree = renderTreeChildren(root, 0); + + const body = ` + +
+ + + Grouped by folder · download the full raw diff. +
+
+${tree} +
+`; + + return pageShell(TITLE, headerHtml(summary), body, "."); +} + +interface TreeNode { + dirs: Map; + files: { name: string; href: string; file: FileDiff }[]; + additions: number; + deletions: number; + count: number; +} + +function newTreeNode(): TreeNode { + return { dirs: new Map(), files: [], additions: 0, deletions: 0, count: 0 }; +} + +/** Builds a directory tree from the (sorted) flat file list. */ +function buildTree(files: FileDiff[], pad: number): TreeNode { + const root = newTreeNode(); + files.forEach((file, i) => { + const href = `files/${String(i + 1).padStart(pad, "0")}.html`; + const segments = file.path.split("/"); + const fileName = segments.pop() ?? file.path; + let node = root; + node.count += 1; + node.additions += file.additions; + node.deletions += file.deletions; + for (const seg of segments) { + let child = node.dirs.get(seg); + if (!child) { + child = newTreeNode(); + node.dirs.set(seg, child); + } + child.count += 1; + child.additions += file.additions; + child.deletions += file.deletions; + node = child; + } + node.files.push({ name: fileName, href, file }); + }); + return root; +} + +function renderTreeChildren(node: TreeNode, depth: number): string { + const dirNames = [...node.dirs.keys()].sort((a, b) => a.localeCompare(b)); + const dirHtml = dirNames + .map((name) => renderDir(name, node.dirs.get(name)!, depth)) + .join("\n"); + const fileHtml = node.files.map((f) => renderFileRow(f.name, f.href, f.file)).join("\n"); + return dirHtml + (dirHtml && fileHtml ? "\n" : "") + fileHtml; +} + +function renderDir(name: string, node: TreeNode, depth: number): string { + // Open the top two levels (flavor + spec) by default; collapse deeper ones. + const open = depth < 2 ? " open" : ""; + return `
+ ${escapeHtml(name)}/ ${node.count} files +${node.additions} -${node.deletions} +
+${renderTreeChildren(node, depth + 1)} +
+
`; +} + +function renderFileRow(name: string, href: string, file: FileDiff): string { + const badge = + file.status === "added" + ? `A` + : file.status === "removed" + ? `D` + : `M`; + return `
+ ${badge}${escapeHtml(name)}+${file.additions} -${file.deletions} +
`; +} + +/** + * Renders a unified-diff chunk as line-numbered, colorized HTML rows. + * Replaces the former diff2html dependency with a small self-contained renderer + * so the tool has zero runtime rendering dependencies. + */ +function renderUnifiedDiff(chunk: string): string { + const rows: string[] = []; + let oldLn = 0; + let newLn = 0; + const row = (cls: string, oldNo: string, newNo: string, text: string): string => + `
${oldNo}${newNo}` + + `${escapeHtml(text.length ? text : " ")}
`; + + for (const raw of chunk.split("\n")) { + const line = raw.replace(/\r$/, ""); + if (line.startsWith("@@")) { + const m = /^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line); + if (m) { + oldLn = Number(m[1]); + newLn = Number(m[2]); + } + rows.push(row("hunk", "", "", line)); + continue; + } + if ( + line.startsWith("diff ") || + line.startsWith("index ") || + line.startsWith("--- ") || + line.startsWith("+++ ") || + line.startsWith("new file") || + line.startsWith("deleted file") || + line.startsWith("rename ") || + line.startsWith("similarity ") || + line.startsWith("old mode") || + line.startsWith("new mode") || + line.startsWith("\\") + ) { + rows.push(row("meta", "", "", line)); + continue; + } + if (line.startsWith("+")) { + rows.push(row("add", "", String(newLn++), line)); + } else if (line.startsWith("-")) { + rows.push(row("del", String(oldLn++), "", line)); + } else { + rows.push(row("ctx", String(oldLn++), String(newLn++), line)); + } + } + return `
${rows.join("")}
`; +} + +/** One page per changed file: line-numbered colorized diff with prev/next nav. */ +function renderFilePage(file: FileDiff, files: FileDiff[], index: number): string { + const pad = String(files.length).length; + const fileName = (i: number): string => `${String(i + 1).padStart(pad, "0")}.html`; + const prev = + index > 0 ? `← Prev` : `← Prev`; + const next = + index < files.length - 1 + ? `Next →` + : `Next →`; + + const chunkBytes = Buffer.byteLength(file.chunk, "utf8"); + let diffBody: string; + if (chunkBytes > MAX_FILE_DIFF_BYTES) { + diffBody = `
⚠️ This file's diff is too large to render (${( + chunkBytes / + (1024 * 1024) + ).toFixed(1)} MB). View it in the raw diff.
`; + } else { + diffBody = renderUnifiedDiff(file.chunk); + } + + const nav = ``; + + const header = `
+

${escapeHtml(file.path)}

+
+${file.additions} / -${file.deletions} · ${file.status}
+
`; + + return pageShell(`${file.path} · ${TITLE}`, header + nav, diffBody, "..", nav); +} + +function headerHtml(summary: DiffSummary): string { + const tagLine = summary.baselineTag + ? `Baseline tag: ${escapeHtml(summary.baselineTag)}` + : "Baseline: none (not bootstrapped)"; + const noteHtml = summary.note ? `

⚠️ ${escapeHtml(summary.note)}

` : ""; + return `
+

${escapeHtml(TITLE)}

+
${tagLine}  ·  ${summary.filesChanged} files changed  ·  +${summary.additions} / -${summary.deletions}
+
${noteHtml}`; +} + +/** Wraps body content in a full HTML document linking the shared stylesheet. */ +function pageShell( + title: string, + headerAndNav: string, + body: string, + cssBase: string, + footerNav = "", +): string { + return ` + + + + +${escapeHtml(title)} + + + +${headerAndNav} +
+${body} +
+${footerNav} + + +`; +} + +/** Site chrome shared across all pages (appended to the diff2html stylesheet). */ +const SITE_CSS = ` +body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; color: #1f2328; } +header { padding: 16px 20px; background: #24292f; color: #fff; } +header h1 { margin: 0 0 6px; font-size: 18px; word-break: break-all; } +header .meta { font-size: 13px; opacity: 0.9; } +header code { background: rgba(255,255,255,0.15); padding: 1px 5px; border-radius: 4px; } +.add { color: #3fb950; } +.del { color: #f85149; } +.note { color: #9a6700; background: #fff8c5; margin: 12px 20px; padding: 10px 14px; border-radius: 6px; } +.no-changes { margin: 40px 20px; font-size: 16px; color: #1a7f37; } +.content { padding: 12px 16px; } +.hint { color: #57606a; font-size: 13px; margin: 8px 0 16px; } +#filter { width: 100%; box-sizing: border-box; padding: 8px 12px; font-size: 14px; border: 1px solid #d0d7de; border-radius: 6px; margin-top: 12px; } +.treebar { display: flex; align-items: center; gap: 10px; margin: 10px 0 14px; flex-wrap: wrap; } +.treebar button { font-size: 12px; padding: 4px 10px; border: 1px solid #d0d7de; background: #f6f8fa; border-radius: 6px; cursor: pointer; } +.treebar button:hover { background: #eaeef2; } +.treebar .hint { margin: 0; } +.tree { font-size: 13px; } +details.dir { margin: 0; } +details.dir > summary { cursor: pointer; padding: 3px 6px; border-radius: 6px; list-style-position: inside; display: flex; align-items: center; gap: 8px; } +details.dir > summary:hover { background: #f0f3f6; } +details.dir > summary .dirname { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-weight: 600; } +details.dir > summary .counts { font-size: 11px; } +.counts { margin-left: auto; white-space: nowrap; font-variant-numeric: tabular-nums; display: inline-flex; gap: 8px; } +.children { margin-left: 16px; border-left: 1px solid #eaeef2; padding-left: 8px; } +.file-row { display: flex; align-items: center; gap: 8px; padding: 2px 6px; } +.file-row:hover { background: #f6f8fa; } +.file-row a { color: #0969da; text-decoration: none; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.file-row a:hover { text-decoration: underline; } +.st { font-size: 10px; font-weight: 700; width: 16px; height: 16px; line-height: 16px; text-align: center; border-radius: 4px; flex: none; } +.st.added { background: #dafbe1; color: #1a7f37; } +.st.removed { background: #ffebe9; color: #cf222e; } +.st.modified { background: #ddf4ff; color: #0969da; } +nav.filenav { display: flex; align-items: center; gap: 14px; padding: 8px 16px; background: #f6f8fa; border-bottom: 1px solid #d0d7de; font-size: 13px; } +nav.filenav .spacer { flex: 1; } +nav.filenav a { color: #0969da; text-decoration: none; } +nav.filenav .muted { color: #8c959f; } +nav.filenav .counter { color: #57606a; } +pre.raw { white-space: pre-wrap; word-break: break-all; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; background: #f6f8fa; padding: 12px; border-radius: 6px; } +.udiff { border: 1px solid #d0d7de; border-radius: 6px; overflow-x: auto; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; line-height: 20px; background: #fff; } +.dl { display: flex; white-space: pre; } +.dl .ln { flex: 0 0 auto; width: 3.5em; padding: 0 8px; text-align: right; color: #6e7781; background: #f6f8fa; border-right: 1px solid #eaeef2; user-select: none; -webkit-user-select: none; font-variant-numeric: tabular-nums; } +.dl .tx { flex: 1 1 auto; padding: 0 10px; } +.dl.add { background: #e6ffec; } +.dl.add .ln { background: #ccffd8; color: #1a7f37; } +.dl.del { background: #ffebe9; } +.dl.del .ln { background: #ffd7d5; color: #cf222e; } +.dl.hunk { background: #ddf4ff; color: #0550ae; } +.dl.hunk .ln { background: #ddf4ff; } +.dl.hunk .tx { color: #0550ae; } +.dl.meta { color: #6e7781; background: #fbfbfb; } +`; + +function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} diff --git a/eng/common/scripts/regen-diff/tsconfig.json b/eng/common/scripts/regen-diff/tsconfig.json new file mode 100644 index 0000000000..4c33094044 --- /dev/null +++ b/eng/common/scripts/regen-diff/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/typespec-python/assets.json b/packages/typespec-python/assets.json new file mode 100644 index 0000000000..ea0b56fb84 --- /dev/null +++ b/packages/typespec-python/assets.json @@ -0,0 +1,6 @@ +{ + "AssetsRepo": "l0lawrence/typespec-assets", + "AssetsRepoPrefixPath": "typespec-azure/python", + "TagPrefix": "typespec-azure/python/tests", + "Tag": "" +} diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 2239e14317..75a4040db6 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -38,6 +38,11 @@ "format:extra": "tsx ./eng/scripts/ci/format.ts --python", "format:extra:check": "tsx ./eng/scripts/ci/format.ts --python --check", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", + "regenerate:render-diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package .", + "regenerate:diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package . --open", + "regenerate:vscode-diff": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts render --package . --vscode", + "regenerate:review": "pnpm run regenerate && pnpm run regenerate:diff", + "regenerate:push-assets": "tsx ../../eng/common/scripts/regen-diff/src/cli.ts push-assets --package .", "sync": "tsx ./eng/scripts/sync.ts", "sync:check": "tsx ./eng/scripts/sync.ts --check", "test:python:e2e": "tsx ./eng/scripts/ci/run-tests.ts", diff --git a/packages/typespec-python/regen-diff.config.json b/packages/typespec-python/regen-diff.config.json new file mode 100644 index 0000000000..e2350f8ffc --- /dev/null +++ b/packages/typespec-python/regen-diff.config.json @@ -0,0 +1,7 @@ +{ + "slug": "python", + "title": "typespec-python emitter — generated test diff", + "generatedDir": "tests/generated", + "flavors": ["azure", "unbranded"], + "pruneFilePrefixes": [".tsp-codegen-"] +}