diff --git a/.github/workflows/images-release.yml b/.github/workflows/images-release.yml index 4c2df0c..6e9903b 100644 --- a/.github/workflows/images-release.yml +++ b/.github/workflows/images-release.yml @@ -429,6 +429,7 @@ jobs: brew tap-new mesh-llm/packaging-validation tap_formula="$(brew --repository)/Library/Taps/mesh-llm/homebrew-packaging-validation/Formula/mesh-llm.rb" cp artifacts/homebrew/Formula/mesh-llm.rb "$tap_formula" + brew trust --formula mesh-llm/packaging-validation/mesh-llm brew audit --strict --online mesh-llm/packaging-validation/mesh-llm brew install --formula mesh-llm/packaging-validation/mesh-llm brew test mesh-llm/packaging-validation/mesh-llm @@ -484,8 +485,8 @@ jobs: if-no-files-found: error retention-days: 14 - node-sdk-preflight: - name: Assemble and install-test Node SDK npm package + node-sdk-package: + name: Assemble and dry-run publish Node SDK npm package if: always() && needs.plan.outputs.npm_enabled == 'true' && needs.node-sdk-addon.result == 'success' needs: [plan, node-sdk-addon] runs-on: ubuntu-24.04 @@ -526,16 +527,15 @@ jobs: --output-dir artifacts/node-sdk-package \ --expected-version "$MESH_VERSION" \ --targets "$targets" - - name: Test, pack, install, and dry-run publish + - name: Test, pack, and dry-run publish timeout-minutes: 5 env: - NPM_PLAN: ${{ needs.plan.outputs.npm_plan }} MESH_VERSION: ${{ needs.plan.outputs.mesh_version }} shell: bash run: | set -euo pipefail npm test --prefix artifacts/node-sdk-package - mkdir -p artifacts/npm consumer + mkdir -p artifacts/npm ( cd artifacts/node-sdk-package npm pack --pack-destination "$GITHUB_WORKSPACE/artifacts/npm" @@ -549,17 +549,6 @@ jobs: if [[ "$MESH_VERSION" == *-* ]]; then dist_tag=next; fi npm publish "./$tarball" --dry-run --access public --tag "$dist_tag" --provenance=false fi - ( - cd consumer - npm init --yes >/dev/null - npm install "../$tarball" - if jq -e '.targets | index("linux-x64")' <<<"$NPM_PLAN" >/dev/null; then - node "$GITHUB_WORKSPACE/scripts/node-sdk-runtime-smoke.cjs" \ - --package-root "$PWD" \ - --expected-version "$MESH_VERSION" \ - --target linux-x64-assembled-package - fi - ) - uses: actions/upload-artifact@v7 with: name: mesh-llm-node-sdk-npm-${{ needs.plan.outputs.mesh_version }} @@ -567,10 +556,75 @@ jobs: if-no-files-found: error retention-days: 14 + node-sdk-preflight: + name: Install and start Node SDK ${{ matrix.name }} + if: always() && needs.plan.outputs.npm_enabled == 'true' && needs.node-sdk-package.result == 'success' + needs: [plan, node-sdk-package] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.npm_matrix) }} + runs-on: ${{ fromJSON(matrix.runner_labels) }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: '24' + package-manager-cache: false + - uses: actions/download-artifact@v8 + with: + name: mesh-llm-node-sdk-npm-${{ needs.plan.outputs.mesh_version }} + path: artifacts/npm + - name: Install and exercise the assembled package on Unix + if: runner.os != 'Windows' + timeout-minutes: 5 + env: + MESH_VERSION: ${{ needs.plan.outputs.mesh_version }} + NODE_SDK_TARGET: ${{ matrix.target }} + shell: bash + run: | + set -euo pipefail + tarballs=(artifacts/npm/*.tgz) + [[ ${#tarballs[@]} -eq 1 && -f "${tarballs[0]}" ]] || { echo "expected one assembled npm tarball" >&2; exit 1; } + tarball="$GITHUB_WORKSPACE/${tarballs[0]}" + mkdir consumer + cd consumer + npm init --yes >/dev/null + npm install "$tarball" + node "$GITHUB_WORKSPACE/scripts/node-sdk-runtime-smoke.cjs" \ + --package-root "$PWD" --expected-version "$MESH_VERSION" \ + --target "$NODE_SDK_TARGET" + - name: Install and exercise the assembled package on Windows + if: runner.os == 'Windows' + timeout-minutes: 5 + env: + MESH_VERSION: ${{ needs.plan.outputs.mesh_version }} + NODE_SDK_TARGET: ${{ matrix.target }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $tarballs = @(Get-ChildItem -LiteralPath artifacts/npm -Filter '*.tgz' -File) + if ($tarballs.Count -ne 1) { throw 'expected one assembled npm tarball' } + $tarball = $tarballs[0].FullName + New-Item -ItemType Directory -Path consumer | Out-Null + Push-Location consumer + try { + npm init --yes | Out-Null + npm install $tarball + node "$env:GITHUB_WORKSPACE/scripts/node-sdk-runtime-smoke.cjs" ` + --package-root (Get-Location).Path --expected-version $env:MESH_VERSION ` + --target $env:NODE_SDK_TARGET + } finally { + Pop-Location + } + publish-node-sdk: name: Publish Node SDK to npm - if: always() && needs.plan.outputs.publish_npm == 'true' && needs.node-sdk-preflight.result == 'success' - needs: [plan, node-sdk-preflight] + if: always() && needs.plan.outputs.publish_npm == 'true' && needs.node-sdk-package.result == 'success' && needs.node-sdk-preflight.result == 'success' + needs: [plan, node-sdk-package, node-sdk-preflight] runs-on: ubuntu-24.04 environment: npm permissions: @@ -958,7 +1012,7 @@ jobs: readiness: name: Packaging readiness manifest if: always() - needs: [plan, upstream, package-image, homebrew, release-assembly, node-sdk-addon, node-sdk-preflight, publish-node-sdk, image-index, promote-images, publish-release-assets] + needs: [plan, upstream, package-image, homebrew, release-assembly, node-sdk-addon, node-sdk-package, node-sdk-preflight, publish-node-sdk, image-index, promote-images, publish-release-assets] runs-on: ubuntu-24.04 steps: - name: Record and enforce required results @@ -975,6 +1029,7 @@ jobs: RELEASE_ASSEMBLY: ${{ needs.release-assembly.result }} NPM_ENABLED: ${{ needs.plan.outputs.npm_enabled }} NPM_ADDONS: ${{ needs.node-sdk-addon.result }} + NPM_PACKAGE: ${{ needs.node-sdk-package.result }} NPM_PREFLIGHT: ${{ needs.node-sdk-preflight.result }} PUBLISH_NPM_REQUESTED: ${{ needs.plan.outputs.publish_npm }} PUBLISH_NPM: ${{ needs.publish-node-sdk.result }} @@ -1000,7 +1055,7 @@ jobs: if [[ "$HOMEBREW_ENABLED" == true ]]; then [[ "$HOMEBREW" == success ]] || return 1; fi if [[ "$ASSEMBLY_ENABLED" == true ]]; then [[ "$RELEASE_ASSEMBLY" == success ]] || return 1; fi if [[ "$NPM_ENABLED" == true ]]; then - [[ "$NPM_ADDONS" == success && "$NPM_PREFLIGHT" == success ]] || return 1 + [[ "$NPM_ADDONS" == success && "$NPM_PACKAGE" == success && "$NPM_PREFLIGHT" == success ]] || return 1 fi if [[ "$PUBLISH_NPM_REQUESTED" == true ]]; then [[ "$PUBLISH_NPM" == success ]] || return 1; fi if [[ "$PUBLISH_IMAGES_REQUESTED" == true ]]; then @@ -1016,12 +1071,12 @@ jobs: --argjson run_id "$GITHUB_RUN_ID" --argjson run_attempt "$GITHUB_RUN_ATTEMPT" \ --arg plan "$PLAN" --arg upstream "$UPSTREAM" --arg package_images "$PACKAGE_IMAGES" \ --arg homebrew "$HOMEBREW" --arg release_assembly "$RELEASE_ASSEMBLY" \ - --arg npm_addons "$NPM_ADDONS" --arg npm_preflight "$NPM_PREFLIGHT" --arg publish_npm "$PUBLISH_NPM" \ + --arg npm_addons "$NPM_ADDONS" --arg npm_package "$NPM_PACKAGE" --arg npm_preflight "$NPM_PREFLIGHT" --arg publish_npm "$PUBLISH_NPM" \ --arg image_index "$IMAGE_INDEX" --arg promote_images "$PROMOTE_IMAGES" --arg publish_assets "$PUBLISH_ASSETS" \ --arg publish_images_requested "$PUBLISH_IMAGES_REQUESTED" \ --arg publish_assets_requested "$PUBLISH_ASSETS_REQUESTED" \ --arg publish_npm_requested "$PUBLISH_NPM_REQUESTED" \ - '{schema:"mesh-packaging-readiness-v1",status:$status,correlation_id:$correlation_id,packaging_run_id:$run_id,packaging_run_attempt:$run_attempt,upstream:{repository:$repository,ref:$ref,sha:$sha,manifest_sha256:$manifest_sha256},requested:{publish_images:$publish_images_requested,publish_release_assets:$publish_assets_requested,publish_npm:$publish_npm_requested},results:{plan:$plan,upstream:$upstream,package_images:$package_images,homebrew:$homebrew,release_assembly:$release_assembly,npm_addons:$npm_addons,npm_preflight:$npm_preflight,publish_npm:$publish_npm,image_index:$image_index,promote_images:$promote_images,publish_release_assets:$publish_assets}}' \ + '{schema:"mesh-packaging-readiness-v1",status:$status,correlation_id:$correlation_id,packaging_run_id:$run_id,packaging_run_attempt:$run_attempt,upstream:{repository:$repository,ref:$ref,sha:$sha,manifest_sha256:$manifest_sha256},requested:{publish_images:$publish_images_requested,publish_release_assets:$publish_assets_requested,publish_npm:$publish_npm_requested},results:{plan:$plan,upstream:$upstream,package_images:$package_images,homebrew:$homebrew,release_assembly:$release_assembly,npm_addons:$npm_addons,npm_package:$npm_package,npm_preflight:$npm_preflight,publish_npm:$publish_npm,image_index:$image_index,promote_images:$promote_images,publish_release_assets:$publish_assets}}' \ | tee packaging-readiness.json [[ "$status" == success ]] - uses: actions/upload-artifact@v7 diff --git a/README.md b/README.md index 7587358..db6b98c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This repository is the packaging and distribution control plane for published [` ```text published upstream tag + immutable tag SHA -> verified host/runtime product bundles -> native packages -> OCI images - -> npm addon lanes -> per-lane fresh install/start -> assembled @mesh-llm/sdk + -> verified npm addon lanes -> assembled @mesh-llm/sdk -> per-platform fresh install/start tarball -> clean install/start ``` diff --git a/TODO.md b/TODO.md index 83276d3..d2676ce 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,23 @@ # Production Readiness TODO +- [ ] Verify Homebrew host integrity and bound forced test-process cleanup. + QA: compare the installed host with its product-manifest digest, prove a + TERM-ignoring child is killed and reaped with bounded waits, pass the formula + tests and full suite, and complete the updated Homebrew release rehearsal. + +- [x] Admit the generated Homebrew validation formula under tap-trust policy. + QA: trust only that formula after generation and before audit, verify the + isolated Homebrew trust record, pass the workflow regression, and complete a + Homebrew audit/install/runtime dry run without disabling tap trust. + Validation: run `41341832014` passed audit, installation, automatic client + readiness, bounded shutdown, and terminal readiness with publication disabled. + +- [ ] Require installed npm lifecycle QA on every selected native platform. + QA: execute one assembled tarball on macOS ARM64, Linux ARM64/x64, and + Windows x64; verify each runner target; reject failed, cancelled, or skipped + required consumers in readiness and publication gates; pass focused workflow + and smoke fixtures, the full TypeScript suite, and a current-artifact dry run. + - [ ] Align packaging with the 0.79.0 producer after the workspace and CI migrations. QA: verify the exact current product schema, runtime ABI/release identity, bundled plugins, and Node SDK source/console/lifecycle contracts; @@ -183,10 +201,11 @@ - [x] Assemble, preflight, and publish the canonical `@mesh-llm/sdk` tarball. QA: local fixture dry runs produce the expected cross-platform tarball, - every addon lane packs and installs into a clean consumer project, and its + every selected native platform installs it into a clean consumer project, and its public `Node` API completes bounded start/status/finally-stop with normal - process exit. The assembled host package repeats the lifecycle proof before - `npm publish --dry-run`; workflow lint proves CI can schedule every lane. + process exit. Assembly performs `npm publish --dry-run`; workflow lint proves + CI can schedule every lane. The later per-platform lifecycle regression and + its current-artifact acceptance are tracked in the active item above. - [x] Document npm packaging and hand ownership off from `mesh-llm`. QA: documentation scans find the canonical repository and workflow, the diff --git a/docs/packaging-readiness-scorecard.md b/docs/packaging-readiness-scorecard.md index bc0eb9c..f8138f7 100644 --- a/docs/packaging-readiness-scorecard.md +++ b/docs/packaging-readiness-scorecard.md @@ -9,7 +9,7 @@ Readiness is evidence-based, not a static percentage. A release is ready only wh | Package correctness | Native metadata, checksum, host/runtime ownership, version/runtime-list checks, and no-device client readiness with bounded shutdown | Full dry-run evidence | | OCI correctness | Image installs the exact package, labels both input digests, checks the final entrypoint, and reaches client readiness without a device | Full dry-run evidence plus separate hardware qualification | | Homebrew | Direct upstream arm64 product, formula-owned runtime, strict audit/install/test, and no-device client readiness | Validate each release before tap update | -| npm SDK | Every addon lane is packed into a fresh project and must complete public `Node` start/status/finally-stop with normal process exit; the assembled package repeats the host-lane proof | Validate the exact public package separately during distribution certification | +| npm SDK | Every selected native platform installs the same assembled tarball into a fresh project, verifies its runner target, and completes public `Node.fromConfig` start/status/finally-stop with normal process exit | Validate the exact public package separately during distribution certification | | Efficiency | One host per OS/architecture is composed with 8 verified runtimes and fans out to 11 package rows; no source builds | Record duration, artifact sizes, and cache behavior | | Publish safety | Dry-run override, job-local write permissions, release environment | Configure/approve environment and observe first publish rehearsal | | Automation | Repository dispatch receiver | Provision upstream fine-grained dispatch credential/App | diff --git a/docs/publishing.md b/docs/publishing.md index e1ed0d9..543a922 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -29,6 +29,15 @@ unknown IDs fail before scheduling. Publication requires the complete matching producer set: images require all native rows, package release assets require all native rows plus Homebrew, and npm requires all addon lanes. +For npm, one assembly job packs the verified addons and tagged JavaScript into +the release tarball. Each selected target then downloads that same artifact, +installs it in a clean project on its native runner, checks the runner's OS and +architecture, and executes bounded `Node.fromConfig` start/status/stop. npm +publication waits for every selected consumer. Readiness records archive +verification as `npm_addons`, assembly as `npm_package`, and the complete native +consumer matrix as `npm_preflight`; required failures, cancellations, and skips +fail the gate. These consumers do not compile upstream sources. + Every enabled native package row preserves its BuildKit statement under an artifact-specific filename, scans the exact package file into SPDX, and verifies that the SPDX file subject names and hashes that same package. The current diff --git a/docs/release-079-compatibility.md b/docs/release-079-compatibility.md index 9ab1bb8..effc5d1 100644 --- a/docs/release-079-compatibility.md +++ b/docs/release-079-compatibility.md @@ -13,6 +13,9 @@ immutable tag and artifact bytes remain the final compatibility inputs. | SDK sources moved to `mesh/sdk/node` | Resolve the SDK directory explicitly and reject missing or ambiguous layouts | | The console verifier now invokes Rust automation | Verify the immutable console assets in this consumer without compiling upstream automation | | Node SDK lifecycle and constructor options changed | Exercise the current public SDK with isolated state and bounded shutdown | +| Addon producers only import their package, and downstream lifecycle QA ran only on Linux x64 | Install the assembled tarball and run the bounded lifecycle on every selected native platform before npm publication | +| Current Homebrew refuses untrusted third-party formulae | Explicitly trust only the generated validation formula before its audit and installation | +| Release admission depends on unchanged host bytes and bounded client shutdown | Compare the installed Homebrew host with its product-manifest digest and bound forced cleanup after a SIGINT timeout | | Unix product archives now contain pinned plugin archives | Preserve safe, regular, top-level plugin archives in native packages and Homebrew; rely on the verified upstream archive checksum and producer's host-pin check | | CUDA native runtimes carry their user-space library closure | Use plain distro runtime bases and remove toolkit package dependencies while retaining NVIDIA driver-injection variables | | Package QA must cover automatic client selection | Exercise `client --auto` with an unreachable loopback discovery relay in native packages, final images, and Homebrew | @@ -73,7 +76,10 @@ publication must exercise the applicable permissions. - [x] Pass the full local suite, exact matrix coverage, shell and workflow checks, and real Docker package reproducibility checks on the final diff. -- [ ] Pass Packaging Precheck on the reviewed compatibility commit. +- [x] Pass Packaging Precheck on the reviewed compatibility commit. Both the + [final PR check](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/41338638204) + and [merged main check](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/41338913553) + passed; the merged implementation is `294a0ae1d7642d0a330e5f34a2a970682d166c82`. - [ ] Run the complete native, Homebrew, and npm matrix against the immutable 0.79.0 RC, with `dry_run=true` and all publication switches disabled. - [ ] Require a successful readiness receipt bound to that RC's SHA and @@ -94,3 +100,66 @@ archive from final image layers. Matrix coverage is 100% for lines, branches, and functions. ShellCheck, actionlint, and the Dockerfile build check passed. These results qualify the packaging implementation; the RC and publication gates above remain open until their real runs finish. + +## Supplemental preview rehearsal + +While 0.79.0 candidate preparation continued, packaging ran +[41339711692](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/41339711692) +against the published October 30 preview. This is infrastructure evidence, not +0.79.0 qualification. The inputs were: + +- Packaging commit: `294a0ae1d7642d0a330e5f34a2a970682d166c82`. +- Upstream tag: `v0.78.0-preview.40959817943.1`. +- Upstream commit: `1253a2a490ab5cf7a7897cb6bb06297f4f286a67`. +- `native-runtimes.json` SHA-256: + `aecff7cb55e384ad17d91143bcec73a4b46b1665ec50bda8f241df573d3aafc8`. +- `dry_run=true`, all validation channels selected, all publication disabled. + +All eight Linux product archives, the shared host-identity gate, all 11 native +package rows, and all 11 final runtime images passed. The package and image +checks include no-driver automatic client readiness and bounded shutdown. + +All four Node addon archives passed verification. The assembled npm package +passed installation and dry-run publication, then failed its lifecycle smoke: +the preview predates `Node.fromConfig`, which current upstream exports and the +current packaging smoke requires. Matching product schemas do not establish +SDK API compatibility. This failure does not justify a fallback in the current +smoke and does not establish a defect in the 0.79.0 candidate. + +The run finished with failed readiness. After its runner queue cleared, +Homebrew refused to load the generated validation formula from an untrusted +tap. Its audit, installation, and runtime test did not execute. The fix trusts +that specific generated formula before loading it, preserving Homebrew's trust +checks for other tap contents. Release assembly was skipped, every publication +job was skipped, and the readiness report retained the exact upstream identity +above. Follow [issue #37](https://github.com/Mesh-LLM/mesh-packaging/issues/37) +for the corrected Homebrew rehearsal and subsequent 0.79.0 evidence. + +## Follow-up validation gates + +[#39](https://github.com/Mesh-LLM/mesh-packaging/pull/39) adds native lifecycle +consumers for the same assembled npm tarball on all four supported targets. +Archive verification, package assembly, and each selected native consumer must +succeed before npm publication or successful readiness. A mismatched runner +target fails before addon loading. + +The generated Homebrew formula receives explicit formula-specific trust. Its +test also checks the installed host against the product-manifest digest before +execution and bounds forced TERM/KILL cleanup after a SIGINT timeout. The +cleanup regression exercises a real child process that ignores TERM. + +Local validation of these follow-up changes passed 208 tests with five expected +local skips. Ruby syntax and Homebrew formula style also passed. SDK subprocess +fixtures now allow startup under host load while retaining bounded execution, +lifecycle assertions, and temporary-state cleanup. The actual RC matrix and +stable publication remain required independently of these local results. + +The formula-specific trust correction passed a Homebrew-only preview rehearsal +in [41341832014](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/41341832014) +at packaging `8a9e054bb938f3f79d4988b3219099d9ed271033`. Audit, installation, +automatic client readiness, bounded shutdown, and terminal readiness passed. +Every publication job was skipped. The receipt retained the preview identity +listed above. The subsequent formula with installed-host digest verification is +being checked in +[41343494547](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/41343494547); +follow issue #37 for its terminal result and actual RC qualification. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 5c66078..4554544 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -31,12 +31,15 @@ - [ ] The arm64 Homebrew formula installs and tests the upstream Metal archive. The exact Cellar binary reaches structured JSON client readiness while alive and completes bounded clean shutdown with isolated runtime and cache state. -- [ ] Every enabled npm addon lane succeeds; the assembled tarball passes - `npm publish --dry-run`, installs in a clean project, and completes public - `Node.create`, `start`, `status`, and `finally`-guarded `stop` with normal +- [ ] Every enabled npm addon lane succeeds; one assembled tarball passes + `npm publish --dry-run`. Every selected target installs that same tarball in + a clean project on its native runner, verifies the runner target, and completes public + `Node.fromConfig`, `start`, `status`, and `finally`-guarded `stop` with normal process exit and verified temporary-state cleanup. - [ ] The final readiness manifest reports success for plan, upstream, native - packages, runtime images, Homebrew, and every enabled npm addon/preflight lane. + packages, runtime images, Homebrew, npm assembly, and every enabled npm + addon/preflight lane. A failed, cancelled, or skipped required preflight + prevents npm publication and readiness success. - [ ] The readiness artifact names the dispatched correlation ID, exact upstream tag commit and native manifest digest, and the upstream release verifies the matching terminal packaging run and attempt. diff --git a/packaging/homebrew/Formula/mesh-llm.rb.template b/packaging/homebrew/Formula/mesh-llm.rb.template index 4976d7d..94e1dc8 100644 --- a/packaging/homebrew/Formula/mesh-llm.rb.template +++ b/packaging/homebrew/Formula/mesh-llm.rb.template @@ -31,9 +31,28 @@ class MeshLlm < Formula writable_paths: ["{{libexec}}"] end + def terminate_test_process(pid) + stopped = %w[TERM KILL].any? do |signal| + Process.kill(signal, pid) + Timeout.timeout(5) { Process.wait(pid) } + true + rescue Errno::ESRCH, Errno::ECHILD + true + rescue Timeout::Error + false + end + flunk "mesh-llm client forced cleanup did not finish" unless stopped + end + test do - assert_match version.to_s, shell_output("#{bin}/mesh-llm --version") + require "digest" + require "json" + require "timeout" + assert_path_exists libexec/"product-manifest.json" + manifest = JSON.parse((libexec/"product-manifest.json").read) + assert_equal manifest.fetch("host").fetch("sha256"), Digest::SHA256.file(bin/"mesh-llm").hexdigest + assert_match version.to_s, shell_output("#{bin}/mesh-llm --version") # `runtime list` prints a warning naming any runtime it rejected and still # exits 0, so assert that the runtime was actually discovered rather than @@ -43,9 +62,6 @@ class MeshLlm < Formula refute_match "malformed native runtime", runtimes refute_match "No local native runtimes found", runtimes - require "json" - require "timeout" - smoke_root = testpath/"client-readiness" log = smoke_root/"client.jsonl" runtime_root = smoke_root/"runtime" @@ -97,16 +113,7 @@ class MeshLlm < Formula rescue Errno::ESRCH, Errno::ECHILD flunk "mesh-llm client exited before bounded SIGINT shutdown" rescue Timeout::Error - begin - Process.kill("TERM", pid) - rescue Errno::ESRCH - nil - end - begin - Process.wait(pid) - rescue Errno::ECHILD - nil - end + terminate_test_process(pid) flunk "mesh-llm client did not stop within 30 seconds of SIGINT" end end diff --git a/scripts/node-sdk-runtime-smoke.cjs b/scripts/node-sdk-runtime-smoke.cjs index 8da43e3..d11a24c 100644 --- a/scripts/node-sdk-runtime-smoke.cjs +++ b/scripts/node-sdk-runtime-smoke.cjs @@ -67,7 +67,11 @@ async function reservePorts() { } } +/** Run one native SDK lifecycle in the supervisor's isolated state directory. */ async function runChild(options) { + const actualTarget = `${process.platform}-${process.arch}` + assert.equal(actualTarget, options.target, + `Node SDK smoke target mismatch: expected ${options.target}, running ${actualTarget}`) const projectRequire = createRequire(path.join(options.packageRoot, 'package.json')) const sdk = projectRequire('@mesh-llm/sdk') assert.equal(sdk.currentMeshVersion(), options.expectedVersion) diff --git a/tests/homebrew-release.test.ts b/tests/homebrew-release.test.ts index 30ae51c..12348ab 100644 --- a/tests/homebrew-release.test.ts +++ b/tests/homebrew-release.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; @@ -55,6 +56,69 @@ test("formula preserves bundled plugin archives for the host's pinned defaults", assert.match(template, /libexec\.install "plugins" if File\.directory\?\("plugins"\)/); }); +test("formula rejects installed host bytes that differ from the product manifest", () => { + const template = readFileSync("packaging/homebrew/Formula/mesh-llm.rb.template", "utf8"); + const checksum = template.indexOf('assert_equal manifest.fetch("host").fetch("sha256"), Digest::SHA256.file(bin/"mesh-llm").hexdigest'); + const startup = template.indexOf('shell_output("#{bin}/mesh-llm --version")'); + assert.ok(checksum >= 0 && checksum < startup); +}); + +test("formula cleanup kills and reaps a child that ignores TERM within bounded waits", () => { + const template = readFileSync("packaging/homebrew/Formula/mesh-llm.rb.template", "utf8"); + const method = template.match(/^ def terminate_test_process\(pid\)\n[\s\S]*?^ end$/m)?.[0]; + assert.ok(method, "formula must provide bounded forced cleanup"); + const result = spawnSync("ruby", ["-e", ` +require "timeout" +${method} +module Timeout + class << self + alias original_timeout timeout + def timeout(seconds, &block) + raise "unbounded cleanup wait" unless seconds == 5 + original_timeout(0.1, &block) + end + end +end +reader, writer = IO.pipe +pid = fork do + reader.close + Signal.trap("TERM", "IGNORE") + writer.write("ready") + writer.close + loop { sleep 1 } +end +writer.close +begin + raise "child not ready" unless reader.read == "ready" + Timeout.original_timeout(2) { terminate_test_process(pid) } + begin + Process.wait(pid, Process::WNOHANG) + raise "child was not reaped" + rescue Errno::ECHILD + end + terminate_test_process(pid) +ensure + reader.close + begin + Process.kill("KILL", pid) + Process.wait(pid) + rescue Errno::ESRCH, Errno::ECHILD + end +end +`], { encoding: "utf8", timeout: 5_000 }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); +}); + +test("Homebrew validation trusts only its generated formula before loading it", () => { + const workflow = readFileSync(".github/workflows/images-release.yml", "utf8"); + const copy = workflow.indexOf('cp artifacts/homebrew/Formula/mesh-llm.rb "$tap_formula"'); + const trust = workflow.indexOf("brew trust --formula mesh-llm/packaging-validation/mesh-llm"); + const audit = workflow.indexOf("brew audit --strict --online mesh-llm/packaging-validation/mesh-llm"); + assert.ok(copy >= 0 && copy < trust && trust < audit); + assert.doesNotMatch(workflow, /brew trust --tap|HOMEBREW_REQUIRE_TAP_TRUST/); +}); + test("formula ships the native runtime past keg relocation and proves it loads", () => { const template = readFileSync("packaging/homebrew/Formula/mesh-llm.rb.template", "utf8"); // Keg relocation rewrites install names and re-signs every Mach-O file, which diff --git a/tests/node-sdk-runtime-smoke.test.ts b/tests/node-sdk-runtime-smoke.test.ts index 972b8dd..2b689f1 100644 --- a/tests/node-sdk-runtime-smoke.test.ts +++ b/tests/node-sdk-runtime-smoke.test.ts @@ -6,11 +6,16 @@ import { spawnSync } from "node:child_process"; import { test } from "node:test"; const smoke = resolve("scripts/node-sdk-runtime-smoke.cjs"); +const nativeTarget = `${process.platform}-${process.arch}`; +// Allow both Node processes to start under host load before testing timeouts. +const fixtureTimeoutMs = 5000; +const fixtureProcessTimeoutMs = 10000; type FixtureBehavior = "normal" | "leak" | "start-failure" | "start-hang" | "stop-hang" | "not-running"; type FixtureEvent = { event: string; smokeRoot?: string }; +/** Create an SDK fixture that records lifecycle calls and can stall or reject them. */ function project( t: { after(callback: () => void): void }, behavior: FixtureBehavior = "normal", @@ -30,7 +35,7 @@ function project( Node: { fromConfig(options) { if (options.mode !== 'client' || options.storage.isolated_config !== true) throw new Error('invalid isolation') if (options.network.auto_join !== false || options.network.publish !== false || !options.network.disable_iroh_relays) throw new Error('external discovery enabled') - if (!options.network.mesh_name.startsWith('packaging-smoke-test-target-')) throw new Error('mesh name is not unique') + if (!options.network.mesh_name.startsWith('packaging-smoke-${nativeTarget}-')) throw new Error('mesh name is not unique') if (options.http.api_port < 1 || options.http.console_port < 1 || options.http.api_port === options.http.console_port) throw new Error('ports are not isolated') const path = require('node:path') const smokeRoot = path.dirname(options.admission.owner_key) @@ -72,9 +77,9 @@ test("runtime smoke starts, reads status, stops, and exits normally", (t) => { const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", - "--timeout-ms", "3000", - ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + "--target", nativeTarget, + "--timeout-ms", String(fixtureTimeoutMs), + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); assert.match(result.stdout, /runtime smoke passed/); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ @@ -88,9 +93,9 @@ test("runtime smoke fails a native addon that prevents normal process exit", (t) const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", - "--timeout-ms", "400", - ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + "--target", nativeTarget, + "--timeout-ms", String(fixtureTimeoutMs), + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /did not exit normally/); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ @@ -104,9 +109,9 @@ test("runtime smoke attempts stop when start partially initializes and rejects", const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", - "--timeout-ms", "3000", - ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + "--target", nativeTarget, + "--timeout-ms", String(fixtureTimeoutMs), + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /start partially initialized/); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ @@ -119,8 +124,8 @@ test("runtime smoke rejects an inactive node even when status returns an object" const fixture = project(t, "not-running"); const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", "--timeout-ms", "3000", - ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + "--target", nativeTarget, "--timeout-ms", String(fixtureTimeoutMs), + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /must report a running client/); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), ["create", "start", "status", "stop"]); @@ -132,10 +137,10 @@ test("runtime smoke handles SIGTERM by stopping a partially started Node and rem const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", - "--timeout-ms", "400", + "--target", nativeTarget, + "--timeout-ms", String(fixtureTimeoutMs), "--shutdown-grace-ms", "1500", - ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /did not exit normally/); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ @@ -150,14 +155,14 @@ test("runtime smoke bounds SIGTERM cleanup before SIGKILL and parent temp cleanu const result = spawnSync(process.execPath, [smoke, "--package-root", fixture.root, "--expected-version", "1.2.3", - "--target", "test-target", - "--timeout-ms", "400", + "--target", nativeTarget, + "--timeout-ms", String(fixtureTimeoutMs), "--shutdown-grace-ms", "200", - ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); const elapsed = performance.now() - startedAt; assert.notEqual(result.status, 0); assert.match(result.stderr, /did not exit normally/); - assert.ok(elapsed < 2000, `forced cleanup took ${elapsed}ms`); + assert.ok(elapsed < fixtureProcessTimeoutMs, `forced cleanup took ${elapsed}ms`); assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ "create", "start", "status", "stop", ]); @@ -165,11 +170,27 @@ test("runtime smoke bounds SIGTERM cleanup before SIGKILL and parent temp cleanu }); test("runtime smoke validates required arguments", () => { - const result = spawnSync(process.execPath, [smoke], { encoding: "utf8" }); + const result = spawnSync(process.execPath, [smoke], { encoding: "utf8", timeout: fixtureProcessTimeoutMs }); assert.notEqual(result.status, 0); assert.match(result.stderr, /--expected-version is required/); }); +test("runtime smoke refuses a foreign native target before loading the package", (t) => { + const fixture = project(t); + const wrongTarget = nativeTarget === "linux-x64" ? "darwin-arm64" : "linux-x64"; + writeFileSync(resolve(fixture.root, "node_modules/@mesh-llm/sdk/index.js"), ` + require('node:fs').writeFileSync(process.env.SMOKE_MARKER, 'package was loaded') + throw new Error('native package must not load on a mismatched worker') + `); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, "--expected-version", "1.2.3", + "--target", wrongTarget, "--timeout-ms", String(fixtureTimeoutMs), + ], { encoding: "utf8", timeout: fixtureProcessTimeoutMs, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + assert.notEqual(result.status, 0); + assert.ok(result.stderr.includes(`Node SDK smoke target mismatch: expected ${wrongTarget}, running ${nativeTarget}`), result.stderr); + assert.equal(existsSync(fixture.marker), false, "target admission must run before native loading or lifecycle calls"); +}); + test("runtime smoke supervises graceful termination before forced termination", () => { const source = readFileSync(smoke, "utf8"); assert.match(source, /process\.once\('SIGTERM', terminate\)/); @@ -184,5 +205,6 @@ test("release workflow consumes verified upstream addons and starts the assemble assert.doesNotMatch(workflow, /npm run build:native|cargo build/); assert.doesNotMatch(workflow, /run: scripts\/verify-sdk-console-assets\.sh/); assert.match(workflow, /scripts\/verify-node-sdk-source\.ts --source-root upstream-source/); - assert.match(workflow, /--target linux-x64-assembled-package/); + assert.doesNotMatch(workflow, /--target linux-x64-assembled-package|\.targets \| index\("linux-x64"\)/); + assert.match(workflow, /runs-on: \$\{\{ fromJSON\(matrix\.runner_labels\) \}\}/); }); diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index b02c1fe..ad70f64 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; +import { readFileSync, rmSync } from "node:fs"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; @@ -104,16 +104,17 @@ test("published release prerelease metadata is required and passed to the planne assert.match(release, /--upstream-prerelease "\$UPSTREAM_PRERELEASE"/); }); -test("terminal readiness records exact release identity on success and failure", () => { +test("terminal readiness records exact release identity and rejects every required npm failure", (t) => { const step = section(release, " - name: Record and enforce required results", " - uses: actions/upload-artifact@v7"); const script = step.match(/ run: \|\n([\s\S]*)/)?.[1].replace(/^ {10}/gm, ""); assert.ok(script, "readiness shell is missing"); const cwd = mkdtempSync(resolve(tmpdir(), "packaging-readiness-")); + t.after(() => rmSync(cwd, { recursive: true, force: true })); const env = { ...process.env, PLAN: "success", UPSTREAM: "success", NATIVE_ENABLED: "true", PACKAGE_IMAGES: "success", HOMEBREW_ENABLED: "true", HOMEBREW: "success", ASSEMBLY_ENABLED: "true", RELEASE_ASSEMBLY: "success", - NPM_ENABLED: "true", NPM_ADDONS: "success", NPM_PREFLIGHT: "success", + NPM_ENABLED: "true", NPM_ADDONS: "success", NPM_PACKAGE: "success", NPM_PREFLIGHT: "success", PUBLISH_NPM_REQUESTED: "true", PUBLISH_NPM: "success", PUBLISH_IMAGES_REQUESTED: "true", IMAGE_INDEX: "success", PROMOTE_IMAGES: "success", PUBLISH_ASSETS_REQUESTED: "true", PUBLISH_ASSETS: "success", @@ -132,6 +133,23 @@ test("terminal readiness records exact release identity on success and failure", assert.equal(success.upstream.sha, env.MESH_SHA); assert.equal(success.upstream.manifest_sha256, env.MANIFEST_SHA256); assert.deepEqual(success.requested, { publish_images: "true", publish_release_assets: "true", publish_npm: "true" }); + assert.equal(success.results.npm_package, "success"); + for (const [environment, resultField] of [ + ["NPM_ADDONS", "npm_addons"], ["NPM_PACKAGE", "npm_package"], ["NPM_PREFLIGHT", "npm_preflight"], + ]) { + for (const outcome of ["failure", "cancelled", "skipped"]) { + const result = run({ [environment]: outcome, PUBLISH_NPM_REQUESTED: "false", PUBLISH_NPM: "skipped" }); + assert.notEqual(result.status, 0, `${environment}=${outcome} must block npm readiness`); + const receipt = JSON.parse(readFileSync(resolve(cwd, "packaging-readiness.json"), "utf8")); + assert.equal(receipt.status, "failure"); + assert.equal(receipt.results[resultField], outcome); + } + } + const disabled = run({ + NPM_ENABLED: "false", NPM_ADDONS: "skipped", NPM_PACKAGE: "skipped", NPM_PREFLIGHT: "skipped", + PUBLISH_NPM_REQUESTED: "false", PUBLISH_NPM: "skipped", + }); + assert.equal(disabled.status, 0, disabled.stderr); assert.notEqual(run({ PUBLISH_NPM: "failure" }).status, 0); const failure = JSON.parse(readFileSync(resolve(cwd, "packaging-readiness.json"), "utf8")); assert.equal(failure.status, "failure"); @@ -330,16 +348,46 @@ test("every native build receives immutable source epoch and every image selects }); test("Node packaging consumes safe upstream addon artifacts without compiling", () => { - const addons = section(release, " node-sdk-addon:", " node-sdk-preflight:"); + const addons = section(release, " node-sdk-addon:", " node-sdk-package:"); assert.match(addons, /scripts\/upstream-node-addon\.ts/); assert.match(addons, /releases\/download\/\$MESH_REF/); assert.doesNotMatch(addons, /rust-toolchain|sccache|build:native|cargo/); }); test("Node prerelease validation uses a non-latest npm dist-tag", () => { + const assembly = section(release, " node-sdk-package:", " node-sdk-preflight:"); + assert.match(assembly, /if \[\[ "\$MESH_VERSION" == \*-\* \]\]; then dist_tag=next; fi/); + assert.match(assembly, /npm publish "\.\/\$tarball" --dry-run --access public --tag "\$dist_tag"/); +}); + +test("every selected native npm target installs the same assembled tarball before publication", () => { + const assembly = section(release, " node-sdk-package:", " node-sdk-preflight:"); const preflight = section(release, " node-sdk-preflight:", " publish-node-sdk:"); - assert.match(preflight, /if \[\[ "\$MESH_VERSION" == \*-\* \]\]; then dist_tag=next; fi/); - assert.match(preflight, /npm publish "\.\/\$tarball" --dry-run --access public --tag "\$dist_tag"/); + const publisher = section(release, " publish-node-sdk:", " image-index:"); + const readiness = section(release, " readiness:"); + const artifactName = "name: mesh-llm-node-sdk-npm-${{ needs.plan.outputs.mesh_version }}"; + assert.equal((assembly.match(/npm pack --pack-destination/g) ?? []).length, 1); + assert.ok(assembly.includes(artifactName)); + assert.match(preflight, /needs: \[plan, node-sdk-package\]/); + assert.match(preflight, /needs\.node-sdk-package\.result == 'success'/); + assert.match(preflight, /fail-fast: false/); + assert.match(preflight, /matrix: \$\{\{ fromJSON\(needs\.plan\.outputs\.npm_matrix\) \}\}/); + assert.match(preflight, /runs-on: \$\{\{ fromJSON\(matrix\.runner_labels\) \}\}/); + assert.match(preflight, /NODE_SDK_TARGET: \$\{\{ matrix\.target \}\}/); + assert.ok(preflight.includes(artifactName)); + assert.doesNotMatch(preflight, /npm pack|scripts\/node-sdk-package\.ts|continue-on-error|linux-x64-assembled-package|\.targets \| index\("linux-x64"\)/); + assert.match(preflight, /shell: bash/); + assert.match(preflight, /shell: pwsh/); + assert.equal((preflight.match(/node-sdk-runtime-smoke\.cjs/g) ?? []).length, 2); + assert.match(preflight, /--target "\$NODE_SDK_TARGET"/); + assert.match(preflight, /--target "?\$env:NODE_SDK_TARGET"?/); + assert.equal((preflight.match(/npm install /g) ?? []).length, 2); + assert.ok(publisher.includes(artifactName)); + assert.match(publisher, /needs: \[plan, node-sdk-package, node-sdk-preflight\]/); + assert.match(publisher, /needs\.node-sdk-preflight\.result == 'success'/); + assert.match(readiness, /needs: \[[^\n]*node-sdk-package[^\n]*node-sdk-preflight/); + assert.match(readiness, /NPM_PACKAGE: \$\{\{ needs\.node-sdk-package\.result \}\}/); + assert.match(readiness, /NPM_PREFLIGHT: \$\{\{ needs\.node-sdk-preflight\.result \}\}/); }); test("readiness requires only explicitly enabled or requested components", () => {