From 07edb450333facbf547e1c707d11731509904d23 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 20 Jul 2026 14:39:37 -0400 Subject: [PATCH 1/2] Fix typespec-java emitter build hanging in parallel repo build Copy-Sources.ps1 and SyncTests.ps1 no longer check out the pinned core/ commit in place. That mutated the shared submodule working tree and raced with the concurrent monorepo build / regen-all-packages-docs, hanging the full pnpm build at the java emitter (while an isolated package build was fine). CoreCommit.ps1 now reads the pinned sources via git archive into a temp dir, leaving the core/ checkout untouched. --- ...-core-checkout-race-2026-07-20-14-30-00.md | 7 + packages/typespec-java/Copy-Sources.ps1 | 16 +-- packages/typespec-java/CoreCommit.ps1 | 120 +++++++++++------- .../typespec-java/emitter-tests/SyncTests.ps1 | 12 +- 4 files changed, 96 insertions(+), 59 deletions(-) create mode 100644 .chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md diff --git a/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md b/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md new file mode 100644 index 0000000000..68fd84f7a3 --- /dev/null +++ b/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@azure-tools/typespec-java" +--- + +Fix the emitter build hanging during a full repo `pnpm build`. `Copy-Sources.ps1` no longer checks out the pinned `core/` commit in place (which mutated the shared submodule working tree and raced with the concurrent monorepo build / `regen-all-packages-docs`); it now reads the pinned sources with `git archive`, leaving the `core/` checkout untouched. diff --git a/packages/typespec-java/Copy-Sources.ps1 b/packages/typespec-java/Copy-Sources.ps1 index 664963bf97..c7da32e2ad 100644 --- a/packages/typespec-java/Copy-Sources.ps1 +++ b/packages/typespec-java/Copy-Sources.ps1 @@ -4,10 +4,10 @@ # Azure-flavored `src/options.ts` (excluded from the copy). # # The desired core commit is pinned in core-commit.json; CoreCommit.ps1's -# Enter-CoreCommit temporarily checks it out (when newer than the submodule's -# current checkout) to copy the sources from, and Restore-CoreCommit always puts -# the submodule back to its original SHA (repo-wide `pnpm build` runs this and CI -# checks git status). +# Get-CoreSourceRoot returns a directory to read the emitter/generator sources from +# (extracted from the pinned commit via `git archive` when it is newer than the +# submodule's current checkout), without ever mutating the core/ submodule working +# tree. Remove-CoreSourceRoot cleans up any temporary extraction afterwards. # # Mirrors the "Copy TypeScript code" step of autorest.java's Build-TypeSpec.ps1. @@ -18,9 +18,9 @@ $repoRoot = Resolve-Path (Join-Path $packageRoot ".." "..") $coreRoot = Join-Path $repoRoot "core" . (Join-Path $packageRoot "CoreCommit.ps1") -$originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot +$core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot try { - $emitterRoot = Resolve-Path (Join-Path $coreRoot "packages" "http-client-java" "emitter") + $emitterRoot = Join-Path $core.Root "emitter" Copy-Item -Path (Join-Path $emitterRoot "src") -Destination $packageRoot -Exclude "options.ts" -Recurse -Force Copy-Item -Path (Join-Path $emitterRoot "test") -Destination $packageRoot -Recurse -Force @@ -29,7 +29,7 @@ try { # customization patch (core.patch) to the copy, so building emitter.jar # (Build-Generator.ps1) never mutates the core/ submodule. The patch paths are # relative to this generator folder. - $srcGenerator = Join-Path $coreRoot "packages" "http-client-java" "generator" + $srcGenerator = Join-Path $core.Root "generator" $destGenerator = Join-Path $packageRoot "generator" $patchFile = Join-Path $packageRoot "core.patch" @@ -50,5 +50,5 @@ try { } } finally { - Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha + Remove-CoreSourceRoot $core } diff --git a/packages/typespec-java/CoreCommit.ps1 b/packages/typespec-java/CoreCommit.ps1 index 36d33359c3..feb88b602f 100644 --- a/packages/typespec-java/CoreCommit.ps1 +++ b/packages/typespec-java/CoreCommit.ps1 @@ -1,26 +1,36 @@ -# Shared helpers for running work against the desired `core/` submodule commit. +# Shared helper for reading sources from the desired `core/` submodule commit +# WITHOUT mutating the submodule's working tree. # -# Dot-source this file to import Enter-CoreCommit / Restore-CoreCommit, then wrap -# the work in a try/finally: +# Dot-source this file to import Get-CoreSourceRoot / Remove-CoreSourceRoot, then +# wrap the work in a try/finally so the temporary extraction (if any) is cleaned up: # # . (Join-Path $packageRoot "CoreCommit.ps1") -# $originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot +# $core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot # try { -# # copy/sync from the (possibly pinned) core checkout +# # read sources from under $core.Root (e.g. emitter/, generator/) # } # finally { -# Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha +# Remove-CoreSourceRoot $core # } # # The desired core commit is pinned in /core-commit.json. When that # file is present and its commit is newer than the submodule's current checkout, -# Enter-CoreCommit temporarily checks it out; Restore-CoreCommit puts the submodule -# back to its original SHA (repo-wide `pnpm build` runs these scripts and CI checks -# git status). +# Get-CoreSourceRoot extracts that commit's `packages/http-client-java` subtree into +# a temporary directory via `git archive`. It never checks out a different SHA or +# otherwise touches the submodule's working tree, so it is safe to run while a +# parallel monorepo build (`pnpm build`, `regen-all-packages-docs`, ...) reads the +# core/ submodule concurrently, and `git status` on core always stays clean. +# +# When core-commit.json is absent, or its commit is not newer than the current +# checkout, the sources are read straight from the submodule's current working tree. + +# Path (relative to the core repo root) of the subtree the Java package reads from. +$script:CoreJavaSubtree = "packages/http-client-java" -# Check out the pinned core commit (when newer) and return the submodule's original -# SHA so the caller can restore it in a finally block. -function Enter-CoreCommit { +# Resolve where to read the core Java sources from and return a descriptor: +# @{ Root = ; TempDir = } +# Pass the descriptor to Remove-CoreSourceRoot in a finally block to clean up. +function Get-CoreSourceRoot { param( # Path to the `core/` submodule. [Parameter(Mandatory = $true)] @@ -31,59 +41,77 @@ function Enter-CoreCommit { [string] $PackageRoot ) - # Capture the submodule's current SHA so the caller can restore it. $originSha = (git -C $CoreRoot rev-parse HEAD).Trim() if ($LASTEXITCODE -ne 0) { Write-Error "Failed to read the current SHA of the 'core' submodule at: $CoreRoot`nMake sure it is checked out (git submodule update --init --recursive)." } + $liveRoot = Join-Path $CoreRoot $script:CoreJavaSubtree + # core-commit.json is optional: when present it pins the core commit to use; # when absent we just use the submodule's current checkout. $configPath = Join-Path $PackageRoot "core-commit.json" - $targetSha = $null - if (Test-Path $configPath) { - $targetSha = ((Get-Content -Raw $configPath | ConvertFrom-Json).sha).Trim() - if ([string]::IsNullOrWhiteSpace($targetSha)) { - Write-Error "No 'sha' found in $configPath." - } - } - else { + if (-not (Test-Path $configPath)) { Write-Host "core-commit.json not found; using current core checkout $originSha." + return @{ Root = $liveRoot; TempDir = $null } + } + + $targetSha = ((Get-Content -Raw $configPath | ConvertFrom-Json).sha).Trim() + if ([string]::IsNullOrWhiteSpace($targetSha)) { + Write-Error "No 'sha' found in $configPath." + } + + if ($targetSha -eq $originSha) { + return @{ Root = $liveRoot; TempDir = $null } } - if ($targetSha -and $targetSha -ne $originSha) { - # Make sure the target commit is available locally (it may be newer than - # what has been fetched); ignore failure if it is already present. - git -C $CoreRoot fetch --quiet origin $targetSha 2>$null + # Make sure the target commit is available locally (it may be newer than what + # has been fetched); ignore failure if it is already present. + git -C $CoreRoot fetch --quiet origin $targetSha 2>$null - # Only move forward: check out the target when it is a descendant (newer) - # of the current checkout. `merge-base --is-ancestor A B` exits 0 when A is - # an ancestor of B, i.e. B is newer than A. - git -C $CoreRoot merge-base --is-ancestor $originSha $targetSha - if ($LASTEXITCODE -eq 0) { - Write-Host "Checking out newer core commit $targetSha (was $originSha)" - git -C $CoreRoot checkout --quiet $targetSha - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to checkout core commit $targetSha." - } + # Only move forward: use the target when it is a descendant (newer) of the + # current checkout. `merge-base --is-ancestor A B` exits 0 when A is an ancestor + # of B, i.e. B is newer than A. + git -C $CoreRoot merge-base --is-ancestor $originSha $targetSha + if ($LASTEXITCODE -ne 0) { + Write-Host "core-commit.json SHA $targetSha is not newer than current $originSha; using current checkout." + return @{ Root = $liveRoot; TempDir = $null } + } + + # The target is newer: extract only its packages/http-client-java subtree into a + # temp directory with `git archive`. This never modifies the submodule's working + # tree, so it stays safe under a parallel monorepo build that reads core/. + Write-Host "Reading core sources from pinned commit $targetSha (was $originSha) without checking it out" + $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("typespec-java-core-" + [System.Guid]::NewGuid().ToString("N")) + New-Item -ItemType Directory -Path $tempDir | Out-Null + $tarPath = Join-Path $tempDir "core.tar" + try { + git -C $CoreRoot archive -o $tarPath "${targetSha}:$script:CoreJavaSubtree" + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to archive core commit ${targetSha}:$script:CoreJavaSubtree." } - else { - Write-Host "core-commit.json SHA $targetSha is not newer than current $originSha; using current checkout." + tar -x -f $tarPath -C $tempDir + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to extract core archive $tarPath." } } - - return $originSha + catch { + Remove-Item $tempDir -Recurse -Force -ErrorAction SilentlyContinue + throw + } + Remove-Item $tarPath -Force + return @{ Root = $tempDir; TempDir = $tempDir } } -# Restore the submodule to the SHA captured by Enter-CoreCommit. -function Restore-CoreCommit { +# Clean up the temporary extraction created by Get-CoreSourceRoot (no-op when the +# submodule's current checkout was read directly). +function Remove-CoreSourceRoot { param( [Parameter(Mandatory = $true)] - [string] $CoreRoot, - - [Parameter(Mandatory = $true)] - [string] $OriginSha + $CoreSource ) - git -C $CoreRoot checkout --quiet $OriginSha + if ($CoreSource.TempDir) { + Remove-Item $CoreSource.TempDir -Recurse -Force -ErrorAction SilentlyContinue + } } diff --git a/packages/typespec-java/emitter-tests/SyncTests.ps1 b/packages/typespec-java/emitter-tests/SyncTests.ps1 index 2ff63eaefb..0e8c1c8305 100644 --- a/packages/typespec-java/emitter-tests/SyncTests.ps1 +++ b/packages/typespec-java/emitter-tests/SyncTests.ps1 @@ -10,8 +10,10 @@ # hand-written tests and specs synced here are committed. # # The desired core commit is pinned in ../core-commit.json; CoreCommit.ps1's -# Enter-CoreCommit transiently checks it out (when newer) and Restore-CoreCommit -# restores the submodule afterwards. +# Get-CoreSourceRoot returns a directory to read the test sources from (extracted +# from the pinned commit via `git archive` when it is newer than the submodule's +# current checkout), without mutating the core/ submodule working tree. +# Remove-CoreSourceRoot cleans up any temporary extraction afterwards. $ErrorActionPreference = "Stop" @@ -20,9 +22,9 @@ $packageRoot = Resolve-Path (Join-Path $scriptRoot "..") $coreRoot = Resolve-Path (Join-Path $scriptRoot ".." ".." ".." "core") . (Join-Path $packageRoot "CoreCommit.ps1") -$originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot +$core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot try { - $testRoot = Resolve-Path (Join-Path $coreRoot "packages" "http-client-java" "generator" "http-client-generator-test") + $testRoot = Join-Path $core.Root "generator" "http-client-generator-test" $localSrc = Join-Path $scriptRoot "src" $localTsp = Join-Path $scriptRoot "tsp" @@ -67,7 +69,7 @@ try { Write-Host "Synced dependency and override versions from $corePackageJsonPath" } finally { - Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha + Remove-CoreSourceRoot $core } # Point the local @azure-tools/typespec-java dependency at the .tgz produced by From bee9364aa268c6b6ed1f4a517be74e4841f8889d Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 20 Jul 2026 14:55:09 -0400 Subject: [PATCH 2/2] Skip copying unused ~38MB generator test modules in emitter build The emitter.jar reactor only builds http-client-generator, -mgmt and -core. http-client-generator-test and http-client-generator-clientcore-test live in a never-activated Maven test profile and are not needed by the build (nor by publish), so skip copying them. Cuts the generator copy from ~17s to ~2s. --- ...build-core-checkout-race-2026-07-20-14-30-00.md | 2 +- packages/typespec-java/Copy-Sources.ps1 | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md b/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md index 68fd84f7a3..2e4a6dfb28 100644 --- a/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md +++ b/.chronus/changes/fix-java-emitter-build-core-checkout-race-2026-07-20-14-30-00.md @@ -4,4 +4,4 @@ packages: - "@azure-tools/typespec-java" --- -Fix the emitter build hanging during a full repo `pnpm build`. `Copy-Sources.ps1` no longer checks out the pinned `core/` commit in place (which mutated the shared submodule working tree and raced with the concurrent monorepo build / `regen-all-packages-docs`); it now reads the pinned sources with `git archive`, leaving the `core/` checkout untouched. +Fix the emitter build hanging during a full repo `pnpm build`. `Copy-Sources.ps1` no longer checks out the pinned `core/` commit in place (which mutated the shared submodule working tree and raced with the concurrent monorepo build / `regen-all-packages-docs`); it now reads the pinned sources with `git archive`, leaving the `core/` checkout untouched. It also no longer copies the ~38MB of `http-client-generator-test` / `http-client-generator-clientcore-test` sources that are not part of the emitter.jar build, cutting the generator copy time from ~17s to ~2s. diff --git a/packages/typespec-java/Copy-Sources.ps1 b/packages/typespec-java/Copy-Sources.ps1 index c7da32e2ad..607b8db41a 100644 --- a/packages/typespec-java/Copy-Sources.ps1 +++ b/packages/typespec-java/Copy-Sources.ps1 @@ -33,11 +33,23 @@ try { $destGenerator = Join-Path $packageRoot "generator" $patchFile = Join-Path $packageRoot "core.patch" + # The emitter.jar reactor (generator/pom.xml) only builds http-client-generator, + # http-client-generator-mgmt and http-client-generator-core. The two test modules + # below live in a separate (never-activated) Maven `test` profile and hold ~38MB + # of generated Java test sources that are not part of the emitter build, so skip + # copying them -- it is the bulk of the copy time. (The Azure emitter-tests + # project syncs http-client-generator-test straight from core via SyncTests.ps1, + # not from this copy.) + $excludedGeneratorModules = @("http-client-generator-test", "http-client-generator-clientcore-test") + Write-Host "Copy generator sources from core" if (Test-Path $destGenerator) { Remove-Item $destGenerator -Recurse -Force } - Copy-Item -Path $srcGenerator -Destination $destGenerator -Recurse -Force + New-Item -ItemType Directory -Path $destGenerator | Out-Null + Get-ChildItem -Path $srcGenerator -Force | + Where-Object { $excludedGeneratorModules -notcontains $_.Name } | + ForEach-Object { Copy-Item -Path $_.FullName -Destination $destGenerator -Recurse -Force } Write-Host "Apply Azure customization patch to copied generator" # Run from the repo root with --directory so git apply resolves the patch's