diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb6dece0..e5b26988 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,15 @@ on: # Qualify every review branch, including stacked acquisition repairs. pull_request: {} workflow_dispatch: + inputs: + build_candidates: + description: Build the sixteen unsigned candidate artifacts without creating a tag or release + type: boolean + default: false + expected_source: + description: Exact 40-character source commit required when build_candidates is true + type: string + default: '' permissions: contents: read @@ -33,6 +42,18 @@ jobs: - uses: actions/setup-python@v6 with: python-version-file: .python-version + - name: Verify explicit build-only source selection + if: github.event_name == 'workflow_dispatch' && inputs.build_candidates + env: + EXPECTED_SOURCE: ${{ inputs.expected_source }} + run: | + python - <<'PYTHON' + import os, re, subprocess + expected = os.environ['EXPECTED_SOURCE'] + actual = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() + if re.fullmatch('[0-9a-f]{40}', expected) is None or expected != actual: + raise SystemExit('build-only dispatch requires the exact checked-out source commit') + PYTHON - name: Run checksum-pinned actionlint with shell and Python checks shell: bash run: | @@ -105,16 +126,16 @@ jobs: - run: cargo test --locked --manifest-path rust/Cargo.toml -p pumas-library --no-default-features --features hf-client - run: cargo test --locked --manifest-path rust/Cargo.toml -p pumas-rpc --no-default-features - name: Build inference-disabled release backend - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) run: cargo build --locked --manifest-path rust/Cargo.toml -p pumas-rpc --release --no-default-features - name: Verify inference-disabled release backend - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) run: python3 scripts/release/smoke-rpc.py rust/target/release/pumas-rpc --inference-disabled headless-rpc: name: Headless archive (${{ matrix.headless_os }}) needs: lint-workflows - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) strategy: fail-fast: false matrix: @@ -154,19 +175,22 @@ jobs: headless-inference: name: Inference headless candidate (${{ matrix.target }}) needs: lint-workflows - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) strategy: fail-fast: false matrix: include: - os: ubuntu-24.04 target: linux-x86_64 + binary: pumas-rpc check: linux - os: windows-2025 target: windows-x86_64 + binary: pumas-rpc.exe check: windows - os: macos-15 target: macos-arm64 + binary: pumas-rpc check: macos runs-on: ${{ matrix.os }} timeout-minutes: 150 @@ -197,6 +221,11 @@ jobs: ORT_SKIP_DOWNLOAD: '1' CARGO_BUILD_JOBS: '1' run: python scripts/release/headless_inference_ci.py --target ${{ matrix.target }} --runtime-dir "${{ runner.temp }}/pumas-ort" --output-dir "${{ runner.temp }}/inference-candidate" --build-id "ci-${{ github.run_id }}-${{ github.run_attempt }}" + - name: Create bounded untrained native fixture + run: python scripts/release/synthetic_onnx_fixture.py --output-dir "${{ runner.temp }}/native-synthetic-fixture" + - name: Verify packaged native untrained model lifecycle + timeout-minutes: 5 + run: python scripts/release/verify-packaged-onnx.py "${{ runner.temp }}/inference-candidate/extracted/${{ matrix.binary }}" "${{ runner.temp }}/native-synthetic-fixture" > "${{ runner.temp }}/inference-candidate/native-synthetic-lifecycle.log" - name: Verify required inference archive run: node scripts/release/check-artifacts.mjs "${{ runner.temp }}/inference-candidate" headless-inference-${{ matrix.check }} - uses: actions/upload-artifact@v7.0.1 @@ -222,7 +251,7 @@ jobs: review-inference-linux: name: Review Linux inference candidate needs: lint-workflows - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.build_candidates) runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -293,7 +322,7 @@ jobs: review-audio-dependencies: name: Review confined audio dependencies needs: lint-workflows - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.build_candidates) runs-on: ubuntu-24.04 timeout-minutes: 60 env: @@ -405,11 +434,11 @@ jobs: - run: pnpm --dir frontend check:types - run: pnpm --dir frontend test:run - name: Build library-only renderer - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) run: pnpm --dir frontend build:library-only # Restore the default renderer before publishing the packaging input. - name: Build default renderer - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) run: pnpm --dir frontend build - run: pnpm --dir electron lint - run: pnpm --dir electron validate @@ -421,7 +450,7 @@ jobs: - run: pnpm --dir electron test:desktop-contract-conformance - run: pnpm --dir frontend test:desktop-contract - uses: actions/upload-artifact@v7.0.1 - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) with: name: frontend-dist path: frontend/dist/ @@ -454,6 +483,9 @@ jobs: - run: python -m pip install -r torch-server/requirements-dev.txt - run: python -m ruff check torch-server scripts/release - run: python -m ruff format --check torch-server scripts/release + - name: Test native packaged release contracts + working-directory: scripts/release + run: python -m unittest -v test_inference_release test_release_metadata - name: Test native headless owner root identity working-directory: scripts/release run: python -m unittest -v test_headless_inference.OwnerContractTests @@ -578,7 +610,7 @@ jobs: torch-native-e2e: name: Torch native RPC E2E (${{ matrix.os }}) needs: lint-workflows - if: github.event_name == 'workflow_dispatch' || (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) + if: (github.event_name == 'workflow_dispatch' && !inputs.build_candidates) || (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) strategy: fail-fast: false matrix: @@ -615,11 +647,15 @@ jobs: build-rust: name: Rust release (${{ matrix.os }}) needs: lint-workflows - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-15] + include: + - os: ubuntu-24.04 + ort_target: linux-x86_64 + - os: macos-15 + ort_target: macos-arm64 runs-on: ${{ matrix.os }} timeout-minutes: 120 steps: @@ -634,6 +670,10 @@ jobs: env: RUSTFLAGS: -D warnings run: cargo check --locked --manifest-path rust/Cargo.toml --workspace --exclude pumas_rustler --all-targets + - name: Provision official desktop CPU runtime + run: python scripts/release/onnx_runtime_stage.py --target ${{ matrix.ort_target }} --download --output-dir "${{ runner.temp }}/desktop-ort" + - name: Probe desktop native CPU runtime + run: python scripts/release/onnx_runtime_probe.py --target ${{ matrix.ort_target }} --runtime-dir "${{ runner.temp }}/desktop-ort" --output "${{ runner.temp }}/desktop-native-loader.json" - name: Build release backend run: cargo build --locked --manifest-path rust/Cargo.toml -p pumas-rpc --release - name: Test release workspace @@ -641,7 +681,7 @@ jobs: - name: Verify backend startup from isolated staging shell: bash run: | - python scripts/release/stage-rpc.py + python scripts/release/stage-rpc.py --target ${{ matrix.ort_target }} --runtime-dir "${{ runner.temp }}/desktop-ort" python scripts/release/smoke-rpc.py electron/resources/bin - uses: actions/upload-artifact@v7.0.1 with: @@ -652,7 +692,7 @@ jobs: build-electron: name: Desktop package (${{ matrix.platform }}) needs: [build-rust, build-frontend] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) strategy: fail-fast: false matrix: @@ -716,7 +756,7 @@ jobs: build-electron-no-inference: name: Desktop package without inference (${{ matrix.platform }}) needs: [headless-rpc, build-frontend] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) strategy: fail-fast: false matrix: @@ -796,7 +836,7 @@ jobs: windows-release: name: Rust release (windows-2025) needs: [lint-workflows, build-frontend] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) runs-on: windows-2025 timeout-minutes: 150 steps: @@ -837,9 +877,13 @@ jobs: env: RUSTFLAGS: -D warnings run: cargo check --locked --manifest-path rust/Cargo.toml --workspace --exclude pumas_rustler --all-targets + - name: Provision official desktop CPU runtime + run: python scripts/release/onnx_runtime_stage.py --target windows-x86_64 --download --output-dir "${{ runner.temp }}/desktop-ort" + - name: Probe desktop native CPU runtime + run: python scripts/release/onnx_runtime_probe.py --target windows-x86_64 --runtime-dir "${{ runner.temp }}/desktop-ort" --output "${{ runner.temp }}/desktop-native-loader.json" - name: Build Windows backend run: cargo build --locked --manifest-path rust/Cargo.toml -p pumas-rpc --release - - run: python scripts/release/stage-rpc.py + - run: python scripts/release/stage-rpc.py --target windows-x86_64 --runtime-dir "${{ runner.temp }}/desktop-ort" - run: python scripts/release/smoke-rpc.py electron/resources/bin - name: Package Windows installers env: @@ -860,7 +904,7 @@ jobs: windows-no-inference: name: Desktop package without inference (win) needs: [lint-workflows, build-frontend, headless-rpc] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) runs-on: windows-2025 timeout-minutes: 60 steps: @@ -920,7 +964,7 @@ jobs: verify-launcher: name: Linux release startup needs: [build-rust, build-frontend] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -954,7 +998,7 @@ jobs: release-candidate: name: Assemble release candidate needs: [rust-quality, headless, headless-rpc, headless-inference, torch-quality, build-electron, build-electron-no-inference, windows-release, windows-no-inference, verify-launcher] - if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + if: (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates) runs-on: ubuntu-24.04 timeout-minutes: 10 steps: @@ -971,10 +1015,15 @@ jobs: pattern: headless-* path: candidate-input - run: node scripts/release/check-artifacts.mjs candidate-input required release-candidate + - uses: actions/setup-python@v6 + with: + python-version-file: .python-version + - name: Describe every final candidate artifact and metadata byte + run: python scripts/release/write-release-metadata.py release-candidate --build-id "github-${{ github.run_id }}-${{ github.run_attempt }}" - uses: actions/upload-artifact@v7.0.1 with: name: release-candidate path: release-candidate/ if-no-files-found: error - # Publication is separate: installer behavior, notices, SBOMs and - # provenance remain release acceptance obligations in RELEASING.md. + # Metadata is an unsigned artifact-level inventory. Native workflows, + # trained models and maintainer license review remain acceptance gates. diff --git a/RELEASING.md b/RELEASING.md index f7064436..7c61164b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,13 +3,33 @@ ## Prepare and rehearse before tagging The `Build` workflow runs on pull requests, `main`, version tags, and manual -`workflow_dispatch`. Pull requests, ordinary branch pushes, and branch-targeted -manual runs execute source quality and contract tests only. Fully optimized -backends, renderer bundles, installers, archives, artifact uploads, startup -smoke, and release-candidate assembly run only for a `v*` version tag (including -a manual dispatch explicitly targeting that tag). Complete the local release QA -before creating the tag; CI then qualifies the exact tagged commit and assembles -its candidate artifacts. +`workflow_dispatch`. Ordinary PR/main validation runs source/contract checks; +PR/ordinary manual runs also produce a separately labeled Linux inference review +archive. +The complete optimized sixteen-artifact matrix runs for a `v*` version tag or an +explicit manual `build_candidates=true` dispatch. Manual candidate builds require +`expected_source` to equal the exact checked-out 40-character commit; a missing or +moved source fails before packaging. The default is false. Candidate mode omits +the separate review jobs and non-shipped Torch E2E; those retain their ordinary +manual/tag behavior. All outputs remain +unsigned, unverified candidates. The workflow has read-only contents permissions +and never creates a tag or GitHub release. + +For a build-only rehearsal, first have the parent review the exact source and +run action. Set `PUMAS_REHEARSAL_SHA` to that reviewed full commit, then dispatch +this branch (or the accepted branch containing this workflow): + +```bash +gh workflow run build.yml --repo MrScripty/Pumas-Library \ + --ref release/v080-package-qualification-20261014 \ + -F build_candidates=true -f expected_source="$PUMAS_REHEARSAL_SHA" +``` + +Do not dispatch this merely because a PR is green. Complete native artifact and +trained-model acceptance before the maintainer's later tag/publication decision. +The same existing package jobs and final inventory checker serve rehearsals and +tags; branch rehearsal does not imply a prerelease or final release. A tag must +match source manifests exactly. Current `0.8.0-rc.1` must never be tagged `v0.8.0`. 1. Update `CHANGELOG.md` and the version's release notes. Keep an empty `Unreleased` section for subsequent changes. @@ -67,7 +87,10 @@ python3 -m ruff check torch-server scripts/release python3 -m ruff format --check torch-server scripts/release python3 -m unittest discover -s torch-server/tests ./launcher.sh --build-release -python3 scripts/release/stage-rpc.py +pumas_ort_work=$(mktemp -d) +python3 scripts/release/onnx_runtime_stage.py --target linux-x86_64 --download --output-dir "$pumas_ort_work/runtime" +python3 scripts/release/onnx_runtime_probe.py --target linux-x86_64 --runtime-dir "$pumas_ort_work/runtime" --output "$pumas_ort_work/loader.json" +python3 scripts/release/stage-rpc.py --target linux-x86_64 --runtime-dir "$pumas_ort_work/runtime" python3 scripts/release/smoke-rpc.py electron/resources/bin smoke_root=$(mktemp -d) mkdir -p "$smoke_root/library/shared-resources/models" "$smoke_root/config" @@ -132,13 +155,15 @@ Electron/Chromium notices alongside the combined notice. For Linux, use `python3 scripts/release/verify-deb-install.py CANDIDATE_DEB` for isolated native install/remove checks. Run `scripts/release/verify-packaged-onnx.py` with the extracted RPC and a real local Nomic fixture to verify import, load, gateway embeddings and -unload. Generate per-installer SPDX, local provenance and final checksums with -`write-linux-metadata.py`; see its attribution report for scope and limitations. +unload. `write-linux-metadata.py` remains a separate optional extracted-file +inventory for the two full Linux installers. It does not generate the complete +sixteen-artifact publication metadata; do not mix its output directory into a +fresh final candidate assembly. CI-built releases must use the CI build identity and actual final files. ## Inference headless candidates -The tag-only `headless-inference` matrix produces Linux x86_64, Windows x86_64 +The tag or explicit build-only `headless-inference` matrix produces Linux x86_64, Windows x86_64 and macOS ARM64 archives through `headless_inference_ci.py` and the existing strict `headless_inference_build.py` producer. Candidate assembly requires all three in addition to the thirteen existing assets. A missing or failed native @@ -148,7 +173,7 @@ Pull requests and manual workflow runs also run the bounded Linux-only the exact PR head (or manual run SHA), uses the same strict release producer, extracts the archive, and compares authenticated core/RPC startup identities. Review archives and diagnostic evidence are retained for seven days under -separate `review-inference-*` artifact names; they cannot satisfy the tag +separate `review-inference-*` artifact names; they cannot satisfy the full matrix's release-candidate inventory. This job grants neither pretrained-model acceptance nor Windows/macOS qualification, and publishes no tag or release. The Linux review job also imports, loads, executes and unloads a deterministic @@ -203,12 +228,17 @@ archive, and runs authenticated owner startup/graceful shutdown. Archive artifac and diagnostic evidence are uploaded separately. These remain unsigned, `unverified_candidate` outputs; startup never qualifies model execution. -For Linux real-model acceptance, `verify-packaged-onnx.py RPC NOMIC_DIRECTORY` -now requires the pinned CPU ORT files beside RPC. The child receives an allowlist +For native real-model acceptance, `verify-packaged-onnx.py RPC NOMIC_DIRECTORY` +requires the selected target's pinned CPU ORT files beside RPC. The child receives an allowlist environment and isolated home/config/cache/registry/current directory, with no -ambient ORT, loader, Pumas, Python or proxy overrides. After finite embedding -inference, `/proc/PID/maps` must identify the packaged loader by path, device and -inode, and current bytes must still match the reviewed pin. Missing runtime, +ambient ORT, loader, Pumas, Python or proxy overrides. Windows retains the actual +OS directory needed for system DLLs and uses a new process group plus graceful +CTRL_BREAK shutdown; Unix uses SIGINT. Fixture copying uses Python on all hosts. +After finite embedding inference, Linux `/proc/PID/maps` must identify the packaged +loader by path, device and inode. Windows PSAPI and macOS lsof observe the actual +process's native module paths; current packaged bytes must match the reviewed pin. +Windows/macOS mapped-file-object identity is not established by these path +observations. The receipt records that narrower scope and copied fixture hashes. Missing runtime, ambient mappings, model errors and forced shutdown fail acceptance. This observes the mapped ORT file; it is not a filesystem sandbox, protection against concurrent hostile same-user mutation, or a complete audit of every loaded system dependency. @@ -224,6 +254,43 @@ This development environment has not executed a v0.8 release build, pretrained O startup, signing/notarization, final SBOM or security qualification. Preserve those gates before the maintainer's publication decision. +## Complete candidate metadata + +After collecting the sixteen exact artifacts into a fresh directory, run: + +```bash +node scripts/release/check-artifacts.mjs candidate-input required release-candidate +python3 scripts/release/write-release-metadata.py release-candidate --build-id local-candidate-rehearsal +``` + +CI invokes the same generator with its actual run/attempt build identity. It +requires unchanged tracked source and current default/S3 attribution inputs. It +writes sixteen SPDX 2.3 package inventories, aggregate unsigned in-toto provenance, +project license, combined default/S3/native notices, and SHA-256 checksums over +all other final files. SPDX `filesAnalyzed=false` is intentional: these describe +exact artifact bytes and referenced attribution/runtime inputs, not exhaustive +extracted-file or native transitive component analysis. Native package inspection, +current audits and maintainer license/notice acceptance remain separate gates. +The aggregate disclosure includes every pinned native notice representation, +including Windows CRLF bytes. Embedded notice identity continues to use each +producer's accepted default/S3 collection and exact target runtime notice bytes; +inspect those actual package contents separately. +Reassemble a fresh set after changing artifacts or metadata; do not retain stale +SPDX/provenance or treat installer-only checksums as final publication coverage. + +## Reviewed trained fixture + +The documented fixture is Nomic's public `nomic-ai/nomic-embed-text-v1.5`, using +`onnx/model_fp16.onnx` plus its matching tokenizer/config files. The +[fixture investigation](docs/plans/v080-packaging/reports/nomic-fixture.md) records +the official revision, publisher license declaration, exact weight identity and +remaining byte-verification limits. A provided directory is copied into an +isolated library and never modified. The lifecycle receipt records the copied +bytes; it does not itself authenticate their model provenance or establish model +quality. No model is bundled in release artifacts or silently downloaded by this +verifier. The inference matrix additionally executes an explicitly untrained +fixture on each native target; that result cannot accept the trained fixture. + ## macOS candidate verification The macOS ARM64 job mounts the DMG read-only, copies the application with `ditto`, @@ -248,7 +315,9 @@ Before tagging or publishing, satisfy the remaining artifact-plan obligations: - Separately provision and verify ONNX Runtime for full inference packages. Cargo never downloads it. Stage the selected 1.24.2 distribution's native library closure beside RPC, with trusted archive/library hashes and notices; - `stage-rpc.py` copies already staged native inputs, not a build download. + `stage-rpc.py --target TARGET --runtime-dir DIRECTORY` accepts only named pinned + native members and notices from that explicitly provisioned directory; it never + searches ambient developer outputs for runtime libraries. Prove real ONNX execution from the extracted package with developer runtime paths removed. An SDK-free build/health check does not qualify inference. - Extract each exact installer and verify renderer, RPC, native dependencies, diff --git a/docs/plans/v080-packaging/execution-ledger.md b/docs/plans/v080-packaging/execution-ledger.md new file mode 100644 index 00000000..7037f137 --- /dev/null +++ b/docs/plans/v080-packaging/execution-ledger.md @@ -0,0 +1,106 @@ +# Execution ledger + +## 2026-10-14 — scoped admission + +Accepted checkpoint/main is `055222931100eeedc43307271dbf7c504787888d`; +worktree clean, original readiness branch and receipts retained. Created the +separate scoped branch. Read current release policy, artifact plan, affected +staging/verifier/metadata tools and current canonical standards before source edits. +The parent explicitly admits this implementation under `docs/plans/v080-packaging/plan.md`. +The design is one serial preparation slice; independent review remains parent-owned. + +Confirmed full desktop jobs do not provision ORT, the existing verifier is +Linux-specific (copy command, environment, mapping, signal), and the old metadata +helper covers only two full Linux installers. Existing tag-only jobs can be +reused behind one explicit manual build-only boolean rather than redesigned. + +Official Nomic model card declares Apache-2.0; verified public weight pointer +at commit `ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2`: FP16 is 273,859,028 bytes, +SHA-256 `cf5b5a86edb00f895561803cfc04729090a958340b8ca2ad76c143f565f6bb04`. +Executor metadata request to HF failed with proxy tunnel 403 before any model +bytes. Browser public-source observations are metadata evidence only; no model +download or inference is claimed. + +Before editing the no-inference packager, admitted its directly affected config: +it inherits full native-resource filters, so a stale developer staging directory +could otherwise copy an unused runtime into that variant. Restrict that variant's +backend resource entry to the two platform RPC filenames. This preserves the +existing backend/variant contract and prevents the same packaging failure family. + +Before extending dispatch guards, bounded the new build-only mode to the actual +sixteen-artifact plan. Separate review archives/dependency probes would duplicate +its native producer or qualify unavailable speech. Existing manual Torch E2E +provisions a non-shipped sidecar cohort and is not an assembly dependency. Omit +those three jobs only in the new candidate mode; retain ordinary PR/manual/tag +semantics. This avoids unrelated runtime downloads without redesigning jobs or +changing release artifact/acceptance support. + +## 2026-10-14 — implementation and local verification + +Implemented explicit official ORT staging/probing in full desktop backend jobs, +including exact pinned notices and no-inference resource exclusion. Staging +validates all native inputs before replacing its generated output. No-inference +configuration retains its other shared resources and variant names. + +Extended the existing positional packaged verifier with Python file copying, +isolated native environments, Windows process-group/CTRL_BREAK shutdown and +unconditional disposal after failed signals. Linux retains device/inode mapping +proof. Windows PSAPI and macOS lsof record the narrower native module path/current +byte observation under a distinct result label; mapped-file-object identity and +complete system closure stay unqualified. Microsoft PSAPI function signatures +were read from official documentation. Old macOS CI job 115744179824 reports +`macos-15-arm64` and Rust 1.92.0 aarch64; that resolves runner architecture for +design selection, not acceptance of the new implementation. + +Reused the existing native inference matrix for explicit untrained model +lifecycle execution in candidate/tag runs. Ordinary native QA now runs the source +contracts including actual Windows/macOS OS-observer self-process checks. This +Linux executor's skip is explicit. No trained weights are downloaded by CI here. + +The complete candidate metadata generator creates sixteen artifact-level SPDX +inventories, sixteen provenance subjects, combined source/native notices, project +license and checksums covering every other generated final file. SPDX does not +claim extracted-file or exhaustive native component analysis. The old two-Linux +extracted-file helper remains usable separately. The synthetic 16-file test +produces 35 checksummed files; those bytes are non-installable test fixtures. + +Validation: + +- Python focused suite: **20 passed, 1 native Windows/macOS-only skip**, 0 failures. + Corrupt staging preserves a prior sentinel; missing/ambient modules, forced + stop/signal errors, absent/extra/empty/symlink artifacts and stale attribution + cannot produce accepted output. Native OS module enumeration is not mocked + when its self-process test runs on the selected host. +- Full Node release suite: **37 passed**, 0 failures/skips. Initial run was + 32 passed/1 failed: its old no-inference assertion required the full native + resource filters. Replaced that obsolete expectation with exact RPC-only + exclusion plus preservation of all other shared resources. Added actual + source-guard execution against the real Git commit and wrong/missing inputs. +- Ruff 0.15.2 (the existing dev requirement): release-script check and formatting + passed. Checksum-pinned actionlint 1.7.8 passed with shellcheck/pyflakes disabled + locally because those plugins are unavailable; CI retains its full plugin gate. +- Both canonical default/S3 notice inventories regenerated. Notice texts remain + byte-identical; only the Electron package source hash changed. Freshness, + dependency ownership and version alignment (0.8.0-rc.1) passed. +- Main remained `055222931100eeedc43307271dbf7c504787888d`; no open Pumas PR was + reported before creating this slice's draft. Both lockfiles and the two earlier + candidate/qualification receipts retain their exact hashes. + +Logs and hashes are indexed in [the receipt](reports/verification.json). +Full native artifacts, trained execution, CI outcome and independent cumulative +review are not invented. Parent owns review of the concrete candidate and exact +build-only dispatch; this task has created no tag, release or full rehearsal run. + +During staged self-review, strengthened native notice coverage by role and every +target pin. That exposed a real representation difference: Linux/macOS LICENSE +and third-party texts use LF, Windows uses CRLF. The stricter first run had +18 passed/1 error/1 skip, retained in the receipt's earlier attempts. Verified +that CRLF reconstruction has exactly the Windows published sizes and digests +(1,094 and 331,175 bytes). The generator now emits every distinct exact pinned +representation, rejects a project license substituted for third-party terms, +and refuses any unrepresented platform's notice pin. No legal wording is guessed +or edited; each emitted variant is authenticated by its own role/target digest. +The complete suite then passed as above. One intermediate Ruff invocation used +the wrong working directory and failed before formatting/tests; the corrected +root invocation passed. The 09:22 environment warning did not lose work: a fresh +09:23:29 UTC git/date/read check succeeded, followed by these actual tests. diff --git a/docs/plans/v080-packaging/issues.md b/docs/plans/v080-packaging/issues.md new file mode 100644 index 00000000..457528c1 --- /dev/null +++ b/docs/plans/v080-packaging/issues.md @@ -0,0 +1,11 @@ +# Packaging issues + +| Finding | Disposition / owner / next evidence | +| --- | --- | +| Full desktop ORT provisioning implicit/missing | Implemented and contract-tested: official pin stage/probe and native notices. Actual final desktop runs pending. | +| Packaged lifecycle verifier Linux-only | Portable implementation and controlled refusal/disposal tests passed; actual native packaged inference remains pending. Windows/macOS module-path evidence is explicitly narrower than Linux device/inode proof. | +| Metadata only describes two Linux full installers | Complete artifact-level SPDX/provenance/checksum generator tested on all sixteen planned identities; actual final bytes remain pending. Extracted-file helper retained separately. | +| Full matrix requires a tag | Default-false, exact-source build-only dispatch reuses jobs, omits review-only/non-shipped runtime jobs, and cannot publish. No run/tag executed here; parent reviews exact action. | +| HF metadata transfer denied by executor proxy | Blocked transfer, confirmed 403 and no bytes; public browser metadata can inform identity, no bypass or inference claim | +| Trained Nomic tokenizer/config byte identities and terms | Inspect exact upstream revision; missing evidence remains unavailable, no substituted weights or private user data | +| Independent cumulative review | Parent-owned after concrete candidate; do not equate self-review or tests with independence | diff --git a/docs/plans/v080-packaging/plan.md b/docs/plans/v080-packaging/plan.md new file mode 100644 index 00000000..c261e1b4 --- /dev/null +++ b/docs/plans/v080-packaging/plan.md @@ -0,0 +1,131 @@ +# v0.8 native package qualification + +**Status:** Verifying. **Acceptance:** partial. **Operation:** verify the implemented +slice admitted by the parent's explicit 2026-10-14 packaging continuation. +**Current phase:** local verification complete; CI and independent parent review. +**Next slice:** inspect exact-candidate checks and the parent's cumulative review. + +## Objective and boundaries + +From accepted main `055222931100eeedc43307271dbf7c504787888d`, make official CPU +ORT provisioning explicit for full desktops, support actual packaged ONNX lifecycle +verification on Linux x86_64/Windows x86_64/macOS ARM64, and generate complete +artifact-level release metadata for all sixteen planned files. Reuse existing +jobs for an opt-in, build-only manual rehearsal. Outputs remain unsigned, +unverified candidates; no tag, release publication, trained-model acceptance by +mock, ASR, vision, SDK, product API or schema change is authorized here. + +Isolation: `release/v080-package-qualification-20261014`, rooted at accepted main; +one serial integration owner, draft PR to main, preserve history and branch until +parent adoption. Existing checkpoint/receipts and user inference work are read-only. + +## Binding design and owners + +- `onnx-runtime-pins.json` stays the sole runtime identity authority. Full desktop + jobs explicitly stage/probe the official 1.24.2 archive, pass its directory to + RPC staging, and include its license/notices. No-inference packages retain their + exact inference-disabled backend and do not gain native runtime payloads. +- Extend `packaged_onnx_support.py` and the existing lifecycle verifier rather + than add a second harness. Use Python copying, native OS module observations, + isolated child environments and platform-correct graceful shutdown. Linux + retains mapped device/inode checks; Windows/macOS record native module paths + and current pinned file bytes with their narrower identity limits explicit. + Native source contract tests do not prove packaged model execution. +- Reuse existing artifact-plan identity checks before generating a per-artifact + SPDX package inventory for all sixteen final files, aggregate unsigned in-toto + provenance, combined accepted source/native notices, and checksums over every + other final file. `filesAnalyzed=false` is explicit for archive-level SPDX; + no exhaustive extracted-file/native transitive component SBOM is claimed. + Preserve the old Linux extracted-file helper and its historical receipts. +- Add a default-false manual `build_candidates` input and extend existing tag + job guards to this explicit build-only dispatch. No publication credentials or + tag writes; ordinary PR/main checks stay bounded. Candidate mode omits the + separate review archives/dependency probes and excluded Torch sidecar E2E; + ordinary manual/tag behavior is retained. Parent reviews the exact + SHA/dispatch before any full rehearsal. Add native untrained lifecycle execution + to the reused inference matrix, without promoting it to trained acceptance. +- Investigate exactly the documented Nomic v1.5 FP16 fixture and immutable public + source/license/file identities. The executor's first HF metadata request was + proxy-denied (403); no model bytes transferred. Do not bypass that restriction, + substitute another model or download the whole repository. A supplied fixture + receives byte hashes in its lifecycle receipt; missing authoritative material + remains visible rather than guessed. No private library scan. + +## Acceptance claims + +| ID | Observable criterion | Kind / environment / mode | Status | +| --- | --- | --- | --- | +| PK1 | Full desktop staging refuses absent/drifting official runtime and includes verified native files/notices; no-inference configuration excludes them | contract / representative source fixtures plus required-real final packages / automated | source regressions satisfied; actual final packages pending approved run | +| PK2 | The shared verifier supports each native OS module observer, isolated startup, import/load/finite embedding/unload and graceful stop; actual native results remain separately identified | system / required-real selected native target / automated | implementation and Linux contract tests satisfied; native/model results pending | +| PK3 | Sixteen exact artifact identities produce sixteen SPDX inventories, provenance subjects and final-byte checksum coverage, with closure/assurance limits explicit | release-artifact / controlled complete fixture and later real final files / automated | complete synthetic inventory satisfied; real artifact set pending | +| PK4 | Manual build-only dispatch reuses the matrix with read-only contents permissions and creates no tag/release; PR/main packaging remains off | contract / workflow analysis and focused dispatch fixtures / automated | source guard/opt-in/no-publish checks and actionlint satisfied; dispatch not executed | +| PK5 | Exact Nomic source/license/revision/weight identity, bounded download feasibility and unavailable inputs have authoritative evidence | contract / actual public upstream and executor / manual | research satisfied; trained bytes/config hashes and execution unavailable | +| PK6 | Focused Python/Node, formatting/lint, workflow validation and independent cumulative review have retained outcomes; commit/push/draft PR are concrete | integration / available local/CI and parent review / either | local checks satisfied; CI and independent parent review pending | + +## Milestone, write set and gates + +One coherent milestone: release-tool preparation, **Verifying**. Write set: +`.github/workflows/build.yml`; `scripts/release/stage-rpc.py`, +`packaged_onnx_support.py`, `verify-packaged-onnx.py`, +`verify_synthetic_packaged_onnx.py`, `write-release-metadata.py`, +`test_inference_release.py`, `test_release_metadata.py`, and focused release +workflow/packaging fixtures; `electron/package.json` and +`electron/electron-builder.no-inference.cjs` resource filters and directly +derived default/S3 attribution inventories; `RELEASING.md`; this plan directory. +Add a directly affected file only with an explicit ledger disposition before edit. +No Cargo or pnpm resolution updates. Preserve legacy verifier positional CLI and +receipt fields consumed by synthetic qualification. + +Gates: regression negatives must decide absent/drifted runtime, ambient module, +unsupported host, forced shutdown, missing/duplicate/stale artifact, incomplete +metadata/checksum, and unintended packaging dispatch. Run the focused existing +release suites and native-source tests, Ruff, Node release tests, actionlint or +available workflow checks. Retain failed attempts and platform not-run boundaries. +Use an independent parent review of the final material diff; no self-review is +called independent. Actual full matrix/trained/native model qualification may +remain pending with exact runnable procedures/owners; source fixtures cannot +close those release gates. + +## Composed-design review (applicable) + +1. Runtime pins/staging own dependency bytes; lifecycle support owns observation + and process disposal; metadata owns final-file descriptions; workflow owns + explicit dispatch. Maintainer owns acceptance/publication, parent owns run/tag + approval. Their contracts are referenced, not merged into one authority. +2. Necessary interleaving is stage/probe before native execution and final + collection before hashes. Model data never enters source/package payloads. +3. Callers provide target, verified runtime directory, binary/fixture or final + artifact directory; helpers hide OS module enumeration and metadata encoding. +4. Runtime pin updates stay with pins/staging; new supported targets touch the + artifact plan/native observer; release file additions touch the plan and its + generator consumer; dispatch policy stays in workflow and its focused check. +5. Stable values are target IDs, exact file digests and existing artifact-plan + identities. OS observation and signal details stay behind lifecycle support. +6. Each helper has independently testable/refusable inputs. Metadata cannot grant + runtime/model acceptance; workflow success cannot grant publication. +7. Deleting native support would spread platform branches across verifiers; + deleting the final metadata generator would leave sixteen partial/manual + descriptions. No registry, generic plugin, second lifecycle framework or + duplicate artifact-plan authority is introduced. +8. Inherent complexity is three native observers plus sixteen artifact identities; + existing deep helpers contain it. Retain one lifecycle path and existing + producers, not a replacement release workflow. + +## Standards, blockers and re-plan triggers + +Canonical Coding-Standards main is unchanged at `188beda1fa477d21d576c233dd8c7f4c4c267d23`. +Planning/Implementation/Verification/Oracles/Platforms/Proportionality, +Build/Release/Commit and Architecture/Dependencies/Security/Licensing apply. +No repository AGENTS.md or relevant `.agents/skills` exists. The application +standards router is unavailable; manual canonical reads confer no executable +standards certification. + +Blockers: executor HF transfer denied; Windows/macOS final package execution and +the full build-only dispatch require their actual native jobs/run approval; +independent review is parent-owned once the candidate is concrete. Replan for +new dependency resolution, production runtime/schema change, unverifiable model +authority, new permanent composition, publication permissions or artifact-plan +support expansion. Stop at preparation plus exact remaining release gates. + +[Ledger](execution-ledger.md) · [Issues](issues.md) · +[Verification receipt](reports/verification.json) · [Nomic investigation](reports/nomic-fixture.md) diff --git a/docs/plans/v080-packaging/reports/nomic-fixture.md b/docs/plans/v080-packaging/reports/nomic-fixture.md new file mode 100644 index 00000000..2ae5bbce --- /dev/null +++ b/docs/plans/v080-packaging/reports/nomic-fixture.md @@ -0,0 +1,65 @@ +# Bounded public Nomic fixture investigation + +Observed 2026-10-14. No model bytes were downloaded or executed in this task. + +## Authoritative identity + +The existing Rust optional real-fixture tests name `nomic-embed-text-v1.5` and +default to `onnx/model_fp16.onnx`. The packaged verifier uses the same layout. +The publisher's [official model card](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) +declares Apache-2.0 and documents task-prefixed inputs and Matryoshka dimensions. +Use the publisher's own repository; no third-party conversion, remote Python +model code, private model directory or unrelated model family is selected here. + +Immutable fixture revision: +[`ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2`](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/tree/ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2). +The publisher's [commit/pointer](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/commit/ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2) +records: + +| Required file | Immutable upstream identity / observation | +| --- | --- | +| `onnx/model_fp16.onnx` | 273,859,028 bytes; SHA-256 `cf5b5a86edb00f895561803cfc04729090a958340b8ca2ad76c143f565f6bb04`; public Git-LFS pointer, not a locally computed digest | +| `tokenizer.json` | [Same revision](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/blob/ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2/tokenizer.json), approximately 711 kB; local SHA-256 unavailable | +| `config.json`, `tokenizer_config.json`, `special_tokens_map.json` | Same immutable repository revision; root listing reports approximately 2.06 kB, 1.19 kB and 695 bytes. Exact transferred bytes/digests are unavailable here. The config page also returned browser HTTP 429. | + +The entire repository is approximately 2.22 GB and includes alternative weights. +Only the selected approximately 275 MB file set is relevant. No whole-repository +clone, Safetensors, quantized alternatives, vision/ASR weights, model execution +scripts or training dependencies are necessary for this fixture. + +## Feasibility and stopping condition + +The first executor request was the public HF model metadata endpoint, not a +weight request. Its HTTP proxy rejected the tunnel with **403** before any bytes +transferred. No alternate host, proxy bypass, private credentials, Git mirror, +or download retry was attempted. Public browser pages supplied the observations +above; they do not transfer a usable model into the executor. The raw API was +also unavailable through the browser. Thus bounded trained-model execution is +**unavailable in this executor**, rather than failed inference. + +On a native host with authorized HF access, fetch only those five paths from the +exact revision. Bound the weight to 273,859,028 bytes and the four small files to +a reviewed small-file budget (2 MiB each is sufficient for the observed sizes). +Verify the weight SHA-256 against the upstream pointer and record SHA-256/size +for every tokenizer/config byte before using the directory. Retain the immutable +source reference and publisher license declaration with the evidence. Do not +execute repository Python or substitute a different fixture after a refusal. + +Run the existing positional verifier against each exact extracted native RPC: + +```bash +python scripts/release/verify-packaged-onnx.py EXTRACTED_RPC NOMIC_DIRECTORY +``` + +It records the copied model/config/tokenizer hashes, import/load, 256 finite +embedding values, selected packaged native runtime, unload and clean disposal. +Its model filename is copied to `model.onnx`; compare that receipt hash with the +upstream FP16 pointer. A successful receipt is a lifecycle result for those exact +bytes, not a quality benchmark or provenance authority. Windows/macOS need their +own actual result; a Linux receipt cannot substitute. The model is test input, +not a redistributed release asset. Maintainer publication/license acceptance +remains separate; no fixture license text or model data was invented or bundled. + +The investigation stops at the verified public identity and exact transfer +blocker. Future evidence must come from an authorized transfer and actual native +execution, not more mocks or untrained graph runs. diff --git a/docs/plans/v080-packaging/reports/verification.json b/docs/plans/v080-packaging/reports/verification.json new file mode 100644 index 00000000..b9ec61a6 --- /dev/null +++ b/docs/plans/v080-packaging/reports/verification.json @@ -0,0 +1,161 @@ +{ + "schema_version": 1, + "date": "2026-10-14", + "scope": "release-tool preparation; not full native or trained-model acceptance", + "base_commit": "055222931100eeedc43307271dbf7c504787888d", + "branch": "release/v080-package-qualification-20261014", + "candidate_commit": "The containing commit is identified in the draft PR; this record does not self-reference its own Git SHA.", + "environment": { + "system": "Linux-6.18.44-x86_64-with-glibc2.41", + "python": "3.12.14", + "repository_python_pin": "3.12.3", + "node": "v24.19.0", + "repository_node_pin": "24.15.0", + "ruff": "0.15.2", + "actionlint": "1.7.8" + }, + "checks": { + "node_release": { + "passed": 37, + "failed": 0, + "skipped": 0 + }, + "python_focused": { + "passed": 20, + "failed": 0, + "skipped": 1, + "skip": "required native Windows/macOS observer self-process test on Linux" + }, + "ruff_check": "passed scripts/release", + "ruff_format": "passed scripts/release", + "actionlint": "passed; shellcheck/pyflakes plugins disabled locally, full CI gate unchanged", + "default_s3_attribution": "passed, regenerated inventories, source notice text unchanged", + "version_alignment": "passed 0.8.0-rc.1", + "dependency_ownership": "passed", + "git_diff_check": "passed" + }, + "logs": { + "/tmp/pumas-v080-packaging-node-final.log": { + "sha256": "b294421536e68dd57840e6156ba63737f806b4de88109f09e7e5234ca7a0e344", + "bytes": 3319, + "retention": "task-local, not release assets" + }, + "/tmp/pumas-v080-packaging-python-final.log": { + "sha256": "0bee95c5adbcf3e11286ea1a65e1a1ea5f5bf5a5a2558a5222b18b01109c9c36", + "bytes": 3686, + "retention": "task-local, not release assets" + }, + "/tmp/pumas-v080-packaging-dispatch.log": { + "sha256": "bcaca246df52a2e7d3a6eb7ef4e802a9611b95cf3d525ec3a1226c20163c199a", + "bytes": 1614, + "retention": "task-local, not release assets" + }, + "/tmp/pumas-v080-packaging-actionlint.log": { + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "bytes": 0, + "retention": "task-local, not release assets" + }, + "/tmp/pumas-v080-packaging-notices-default.log": { + "sha256": "b42e19964ae0229994d6d5b7f704e299d88d71f2e2bf818c0a3a8130f86c91d5", + "bytes": 84, + "retention": "task-local, not release assets" + }, + "/tmp/pumas-v080-packaging-notices-s3.log": { + "sha256": "719c689e15128b44264d76555819239179e9ac435b65a2679ea059cdba628f87", + "bytes": 79, + "retention": "task-local, not release assets" + } + }, + "preserved": { + "rust/Cargo.lock": "91fb9823de2f0346211b4f68ca400f03a628890e55dca90e0803e2dd7ad47e42", + "pnpm-lock.yaml": "78cac9b50e2f28d293420ffb8b9d57889a723669a3969b3a670dd3e6cbfdda18", + "docs/plans/v080-readiness/reports/candidate.json": "617b7fb1ec18c04274d7ad56d346d0a8e6d62b2b3358b1b1f18a139715fdb814", + "docs/plans/v080-qualification/reports/qualification.json": "12578a001da624a1a08819ae44efdcefb83a1b42f6059d9e1a1eed498345a0a6" + }, + "notice_inventory": { + "": { + "sha256": "a2b59c47a02da0fe8720aa4b6e13f3c771bc1cd7bdf47c3d24f8a44ac2d3b1ad", + "notice_sha256": "5e5e273c6d4a24563d91de8d9d349413481da992d714438d80e884d61eb13dfc" + }, + "-s3": { + "sha256": "0b61269264f6fd0cf24ab180a9f3f9fbcff3b92c39a79721359799bd32582183", + "notice_sha256": "abf171b832a35b70752979877aaab3b9ad3fdbf2e9cb2a034e2bce3a832134df" + } + }, + "native_results": { + "linux_contracts": "passed controlled file/process assertions, not trained inference", + "windows_macos_observer": "not run in Linux; registered in existing native QA", + "native_packaged_untrained": "added to reused full inference matrix; not run here", + "native_packaged_trained": "not run" + }, + "metadata": { + "controlled_fixture_artifacts": 16, + "spdx_documents": 16, + "provenance_subjects": 16, + "checksummed_final_files": 35, + "filesAnalyzed": false, + "assurance": "unsigned, unverified candidate; extracted/native component inspection separate", + "native_notice_variants": "Exact Unix LF and Windows CRLF bytes authenticated separately against all target/role pins; no notice role may be substituted or dropped." + }, + "model": { + "repository": "nomic-ai/nomic-embed-text-v1.5", + "revision": "ac6fcd72429d86ff25c17895e47a9bfcfc50c1b2", + "file": "onnx/model_fp16.onnx", + "bytes": 273859028, + "upstream_pointer_sha256": "cf5b5a86edb00f895561803cfc04729090a958340b8ca2ad76c143f565f6bb04", + "license": "Apache-2.0 publisher declaration; model not redistributed", + "local_bytes": "not obtained", + "small_file_local_hashes": "unavailable", + "executor_transfer": "metadata request denied by proxy tunnel HTTP 403; no model bytes, retry or bypass", + "report": "nomic-fixture.md" + }, + "dispatch": { + "manual_build_candidates_default": false, + "expected_source": "exact checked-out lowercase 40-character commit required", + "creates_tag": false, + "publishes_release": false, + "executed": false, + "owner": "parent reviews exact source/action first" + }, + "independent_review": { + "state": "pending", + "owner": "parent cumulative material review; self-review not independent" + }, + "ci": { + "state": "not yet observed; inspect draft PR exact-head run" + }, + "remaining_release_gates": [ + "Approved full native build-only rehearsal and artifact/user-workflow inspection", + "Reviewed trained fixture bytes and native packaged execution", + "Maintainer final notices/security/metadata acceptance", + "Separate final version alignment and parent tag/publication decision" + ], + "earlier_attempts": [ + { + "kind": "node release suite", + "passed": 32, + "failed": 1, + "disposition": "Old no-inference full-resource inheritance assertion replaced with intended exact RPC-only filter and preservation of other resources.", + "log": "/tmp/pumas-v080-packaging-node.log", + "log_sha256": "00322acb2698f72dd68cd99676589e7dcce84e2ae516facacd37fd9df8e73f78" + }, + { + "kind": "stronger native notice role check", + "passed": 18, + "failed": 1, + "skipped": 1, + "disposition": "Exposed Windows CRLF notice representations; now every target/role exact size and SHA-256 must be present in generated byte variants.", + "log": "/tmp/pumas-v080-packaging-python-windows-notice-gap.log", + "log_sha256": "4d50a8ae20dd306fc70de0abbaf6a889066a3f4e0dec6481a1589738f63c27b4" + }, + { + "kind": "Ruff setup invocation", + "result": "failed before formatting/tests due wrong working directory", + "disposition": "Corrected root invocation passed." + } + ], + "environment_recheck": { + "time": "2026-10-14T09:23:29Z", + "actual_executor": "git/date/read check succeeded after disconnect notification; focused tests then completed successfully" + } +} diff --git a/docs/release-attribution/0.8.0-rc.1-s3/inventory.json b/docs/release-attribution/0.8.0-rc.1-s3/inventory.json index 6da6a040..90ae02e6 100644 --- a/docs/release-attribution/0.8.0-rc.1-s3/inventory.json +++ b/docs/release-attribution/0.8.0-rc.1-s3/inventory.json @@ -9233,7 +9233,7 @@ "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-linux-cpu-rpc-restart-acceptance/runtime.json": "d8b9826efc0aa5fedd0353f5227b1001cfe6bda96287ff3136b81b513c71e104", "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-macos-cpu-rpc-restart-acceptance/runtime.json": "a280a955f040983398474af0c34f92e9172fc3c69ef22f7422ac990ed0554e26", "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-windows-cpu-rpc-restart-acceptance/runtime.json": "3e2f6e67e3a4492c01232778191871e9f39543f02996349f055272f2d19fbb89", - "electron/package.json": "b5b510c5d32948485b1e2ef616b11e707e462f58ec83d2776a5428725c102b65", + "electron/package.json": "d22363bc876a8abcf8eb459bd8e1fb169e5e717baa391db7f4ddce896297327b", "frontend/package.json": "8e24aa7eceed7cdb9279a77d42a18e7183abefae32586ea4cdcac48a042729ae", "pnpm-lock.yaml": "78cac9b50e2f28d293420ffb8b9d57889a723669a3969b3a670dd3e6cbfdda18", "rust/Cargo.lock": "91fb9823de2f0346211b4f68ca400f03a628890e55dca90e0803e2dd7ad47e42", diff --git a/docs/release-attribution/0.8.0-rc.1/inventory.json b/docs/release-attribution/0.8.0-rc.1/inventory.json index 956699db..ee79399b 100644 --- a/docs/release-attribution/0.8.0-rc.1/inventory.json +++ b/docs/release-attribution/0.8.0-rc.1/inventory.json @@ -8273,7 +8273,7 @@ "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-linux-cpu-rpc-restart-acceptance/runtime.json": "d8b9826efc0aa5fedd0353f5227b1001cfe6bda96287ff3136b81b513c71e104", "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-macos-cpu-rpc-restart-acceptance/runtime.json": "a280a955f040983398474af0c34f92e9172fc3c69ef22f7422ac990ed0554e26", "docs/plans/torch-cross-platform-runtime-management/reports/v2.14.0-windows-cpu-rpc-restart-acceptance/runtime.json": "3e2f6e67e3a4492c01232778191871e9f39543f02996349f055272f2d19fbb89", - "electron/package.json": "b5b510c5d32948485b1e2ef616b11e707e462f58ec83d2776a5428725c102b65", + "electron/package.json": "d22363bc876a8abcf8eb459bd8e1fb169e5e717baa391db7f4ddce896297327b", "frontend/package.json": "8e24aa7eceed7cdb9279a77d42a18e7183abefae32586ea4cdcac48a042729ae", "pnpm-lock.yaml": "78cac9b50e2f28d293420ffb8b9d57889a723669a3969b3a670dd3e6cbfdda18", "rust/Cargo.lock": "91fb9823de2f0346211b4f68ca400f03a628890e55dca90e0803e2dd7ad47e42", diff --git a/electron/electron-builder.no-inference.cjs b/electron/electron-builder.no-inference.cjs index 4bfff6b5..49092ae1 100644 --- a/electron/electron-builder.no-inference.cjs +++ b/electron/electron-builder.no-inference.cjs @@ -12,6 +12,9 @@ const base = require('./package.json').build; module.exports = { ...base, appId: 'com.pumas.library.no-inference', + extraResources: base.extraResources.map(resource => resource.from === 'resources/bin' + ? { ...resource, filter: ['pumas-rpc', 'pumas-rpc.exe'] } + : resource), appImage: { ...base.appImage, artifactName: 'Pumas.Library-no-inference-${version}.${ext}', diff --git a/electron/package.json b/electron/package.json index 8291bade..b47ce79b 100644 --- a/electron/package.json +++ b/electron/package.json @@ -71,7 +71,9 @@ "*.dll", "*.so", "*.so.*", - "*.dylib" + "*.dylib", + "ONNX-LICENSE.txt", + "ONNX-NOTICES.txt" ] }, { diff --git a/scripts/release/check-ci-release-gating.test.mjs b/scripts/release/check-ci-release-gating.test.mjs index 54d5b811..2c7e4709 100644 --- a/scripts/release/check-ci-release-gating.test.mjs +++ b/scripts/release/check-ci-release-gating.test.mjs @@ -7,7 +7,7 @@ import { spawnSync } from 'node:child_process'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const workflow = fs.readFileSync(path.join(root, '.github/workflows/build.yml'), 'utf8'); -const versionTagGuard = "github.ref_type == 'tag' && startsWith(github.ref_name, 'v')"; +const versionTagGuard = "(github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) || (github.event_name == 'workflow_dispatch' && inputs.build_candidates)"; const escapedVersionTagGuard = versionTagGuard.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'); test('ONNX graph gate rejects transitive download unification and linked runtime mode', () => { @@ -45,7 +45,7 @@ function job(name) { return workflow.slice(start, end); } -test('release-producing jobs run only for version tags', () => { +test('candidate-producing jobs require a version tag or explicit build-only dispatch', () => { for (const name of [ 'headless-rpc', 'headless-inference', @@ -61,6 +61,47 @@ test('release-producing jobs run only for version tags', () => { } }); +test('manual candidate dispatch is opt-in, source-pinned and cannot publish', () => { + assert.match(workflow, /build_candidates:\n description: .*\n type: boolean\n default: false/); + assert.match(job('lint-workflows'), /EXPECTED_SOURCE: \$\{\{ inputs.expected_source \}\}/); + assert.match(job('lint-workflows'), /re.fullmatch\('\[0-9a-f\]\{40\}', expected\).*expected != actual/); + assert.match(workflow, /permissions:\n contents: read/); + assert.doesNotMatch(workflow, /contents: write|gh release|git tag|git push|create-release|publish: always/); + assert.match(job('release-candidate'), /write-release-metadata.py release-candidate --build-id/); + for (const name of ['review-inference-linux', 'review-audio-dependencies', 'torch-native-e2e']) { + assert.ok(job(name).includes("github.event_name == 'workflow_dispatch' && !inputs.build_candidates")); + } +}); + +test('manual source guard accepts only its exact Git commit', () => { + const python = process.env.PYTHON ?? (process.platform === 'win32' ? 'python' : 'python3'); + const head = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(); + const code = job('lint-workflows').match(/python - <<'PYTHON'\n([\s\S]*?) PYTHON/)[1] + .replace(/^ /gm, ''); + const different = (head[0] === 'a' ? 'b' : 'a') + head.slice(1); + for (const [source, accepted] of [[head, true], ['', false], ['not-a-commit', false], [different, false]]) { + const result = spawnSync(python, ['-c', code], { + cwd: root, encoding: 'utf8', timeout: 10_000, env: { ...process.env, EXPECTED_SOURCE: source }, + }); + assert.equal(result.status === 0, accepted, result.stderr); + } +}); + +test('full desktop native runtimes are provisioned and probed before staging', () => { + for (const name of ['build-rust', 'windows-release']) { + const block = job(name); + assert.ok(block.includes('onnx_runtime_stage.py')); + assert.ok(block.includes('onnx_runtime_probe.py')); + assert.ok(block.indexOf('onnx_runtime_probe.py') < block.indexOf('stage-rpc.py')); + assert.match(block, /stage-rpc.py --target .* --runtime-dir/); + } + const inference = job('headless-inference'); + assert.match(inference, /binary: pumas-rpc.exe/); + assert.match(inference, /name: Verify packaged native untrained model lifecycle/); + assert.match(inference, /verify-packaged-onnx.py .*matrix.binary/); + assert.match(job('torch-quality'), /python -m unittest -v test_inference_release test_release_metadata/); +}); + test('shared quality jobs remain available to pull requests and ordinary pushes', () => { for (const name of ['lint-workflows', 'rust-quality', 'headless', 'build-frontend', 'torch-quality']) { assert.doesNotMatch(job(name), /^ if:/m, `${name} must not be job-gated to version tags`); @@ -69,7 +110,7 @@ test('shared quality jobs remain available to pull requests and ordinary pushes' test('review inference candidate is bounded, source-pinned and never publishes a release', () => { const review = job('review-inference-linux'); - assert.match(review, /^ if: github\.event_name == 'pull_request' \|\| github\.event_name == 'workflow_dispatch'$/m); + assert.ok(review.includes(" if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.build_candidates)\n")); assert.match(review, /^ needs: lint-workflows$/m); assert.match(review, /^ runs-on: ubuntu-24\.04$/m); assert.match(review, /^ timeout-minutes: 60$/m); @@ -181,7 +222,7 @@ test('release build commands never enable test-support or all-features', () => { test('review audio dependency qualification requires bounded exact-source confinement', () => { const audio = job('review-audio-dependencies'); - assert.match(audio, /if: github.event_name == 'pull_request' \|\| github.event_name == 'workflow_dispatch'/); + assert.ok(audio.includes("if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.build_candidates)")); assert.match(audio, /runs-on: ubuntu-24\.04/); assert.match(audio, /timeout-minutes: 60/); assert.match(audio, /persist-credentials: false/); diff --git a/scripts/release/check-no-inference-config.test.mjs b/scripts/release/check-no-inference-config.test.mjs index 088b46e3..394d4d9a 100644 --- a/scripts/release/check-no-inference-config.test.mjs +++ b/scripts/release/check-no-inference-config.test.mjs @@ -44,5 +44,13 @@ test('no-inference packaging config matches the artifact plan', () => { assert.equal(config.productName, base.productName); assert.equal(config.beforePack, base.beforePack); assert.deepEqual(config.files, base.files); - assert.deepEqual(config.extraResources, base.extraResources); + assert.deepEqual( + config.extraResources.filter(resource => resource.from !== 'resources/bin'), + base.extraResources.filter(resource => resource.from !== 'resources/bin'), + ); + assert.deepEqual(config.extraResources.find(resource => resource.from === 'resources/bin').filter, + ['pumas-rpc', 'pumas-rpc.exe']); + const full = base.extraResources.find(resource => resource.from === 'resources/bin').filter; + assert.ok(full.includes('ONNX-LICENSE.txt')); + assert.ok(full.includes('ONNX-NOTICES.txt')); }); diff --git a/scripts/release/headless-inference.test.mjs b/scripts/release/headless-inference.test.mjs index 0cf4d5c2..23fb8d10 100644 --- a/scripts/release/headless-inference.test.mjs +++ b/scripts/release/headless-inference.test.mjs @@ -22,3 +22,10 @@ test('pinned native staging and packaged environment admission', () => { ['-m', 'unittest', '-v', 'test_inference_release'], { cwd: directory, stdio: 'pipe', timeout: 30_000 }); }); + +test('complete artifact metadata coverage and final-byte checksums', () => { + const directory = fileURLToPath(new URL('.', import.meta.url)); + execFileSync(process.env.PYTHON ?? (process.platform === 'win32' ? 'python' : 'python3'), + ['-m', 'unittest', '-v', 'test_release_metadata'], + { cwd: directory, stdio: 'pipe', timeout: 30_000 }); +}); diff --git a/scripts/release/packaged_onnx_support.py b/scripts/release/packaged_onnx_support.py index 6975f2eb..efb5762c 100644 --- a/scripts/release/packaged_onnx_support.py +++ b/scripts/release/packaged_onnx_support.py @@ -1,16 +1,32 @@ -"""Linux packaged inference acceptance: pinned bytes and isolated child environment.""" +"""Native packaged inference: pinned bytes, OS observations and child disposal.""" +import ctypes import json import os from pathlib import Path +import platform +import signal +import subprocess +import sys import headless_inference as package from onnx_runtime_stage import PINS -def verify_runtime(directory): +def native_target(): + system, machine = platform.system(), platform.machine().lower() + if system == "Linux" and machine in {"x86_64", "amd64"}: + return "linux-x86_64" + if system == "Windows" and machine in {"x86_64", "amd64"}: + return "windows-x86_64" + if system == "Darwin" and machine in {"arm64", "aarch64"}: + return "macos-arm64" + raise ValueError("packaged inference requires a supported native host") + + +def verify_runtime(directory, target=None): directory = Path(directory).resolve() - selected = json.loads(PINS.read_text())["targets"]["linux-x86_64"] + selected = json.loads(PINS.read_text())["targets"][target or native_target()] for name, item in selected["files"].items(): path = directory / name package.require( @@ -27,11 +43,16 @@ def child_environment(root): root = Path(root).resolve() # An allowlist prevents all ambient ORT/Pumas/Python/loader/proxy overrides, # including ones introduced after this verifier was written. - return { - "PATH": "/usr/bin:/bin", + environment = { + "PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "LANG": "C.UTF-8", "HOME": str(root / "home"), "TMPDIR": str(root / "tmp"), + "TMP": str(root / "tmp"), + "TEMP": str(root / "tmp"), + "USERPROFILE": str(root / "home"), + "APPDATA": str(root / "config"), + "LOCALAPPDATA": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config"), "XDG_DATA_HOME": str(root / "data"), "XDG_CACHE_HOME": str(root / "cache"), @@ -40,13 +61,132 @@ def child_environment(root): "PUMAS_REGISTRY_DB_PATH": str(root / "registry.db"), "ORT_SKIP_DOWNLOAD": "1", } + if sys.platform == "win32": + # Read the OS directory from the OS, never from an ambient override. + buffer = ctypes.create_unicode_buffer(32768) + package.require( + ctypes.windll.kernel32.GetWindowsDirectoryW(buffer, len(buffer)) > 0, + "Windows directory unavailable", + ) + environment.update( + SystemRoot=buffer.value, + WINDIR=buffer.value, + PATH=str(Path(buffer.value) / "System32"), + ) + return environment + + +def windows_modules(pid): + """Enumerate the owned process's mapped modules through the native PSAPI ABI.""" + from ctypes import wintypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel.OpenProcess.restype = wintypes.HANDLE + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + kernel.K32EnumProcessModulesEx.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + wintypes.DWORD, + ] + kernel.K32GetModuleFileNameExW.argtypes = [ + wintypes.HANDLE, + wintypes.HMODULE, + wintypes.LPWSTR, + wintypes.DWORD, + ] + kernel.K32GetModuleFileNameExW.restype = wintypes.DWORD + handle = kernel.OpenProcess(0x410, False, pid) + package.require(bool(handle), "cannot observe owned Windows process") + try: + modules = (wintypes.HMODULE * 4096)() + required = wintypes.DWORD() + package.require( + kernel.K32EnumProcessModulesEx( + handle, modules, ctypes.sizeof(modules), ctypes.byref(required), 3 + ) + and 0 < required.value <= ctypes.sizeof(modules), + "bounded Windows module enumeration failed", + ) + paths = [] + for module in modules[: required.value // ctypes.sizeof(wintypes.HMODULE)]: + name = ctypes.create_unicode_buffer(32768) + length = kernel.K32GetModuleFileNameExW(handle, module, name, len(name)) + package.require(0 < length < len(name) - 1, "mapped module path unavailable") + paths.append(Path(name.value).resolve()) + return paths + finally: + kernel.CloseHandle(handle) + + +def native_modules(pid, target): + package.require(target == native_target(), "module observation requires matching host") + if target == "windows-x86_64": + return windows_modules(pid) + package.require(target == "macos-arm64", "native module observer unavailable") + observed = subprocess.run( + ["/usr/sbin/lsof", "-a", "-p", str(pid), "-d", "txt", "-F0n"], + capture_output=True, + check=True, + timeout=15, + ).stdout + package.require(len(observed) <= 1024 * 1024, "module observation exceeds bound") + return [ + Path(field[1:].decode()).resolve() + for item in observed.split(b"\0") + if (field := item.lstrip(b"\n")).startswith(b"n/") + ] + + +def stop_process(process): + if process.poll() is None: + try: + process.send_signal(signal.CTRL_BREAK_EVENT if os.name == "nt" else signal.SIGINT) + process.wait(timeout=20) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + raise RuntimeError("Packaged backend required forced shutdown") + except BaseException: + # A failed signal (including unavailable Windows console control) + # must fail acceptance and still dispose the process we own. + if process.poll() is None: + process.kill() + process.wait(timeout=10) + raise + package.require(process.returncode == 0, "packaged backend did not shut down cleanly") def loaded_runtime(pid, directory): - """Observe Linux mappings while the model is loaded; not a full dependency audit.""" + """Observe the loaded packaged ORT; never claim a full dependency audit.""" directory = Path(directory).resolve() - selected = verify_runtime(directory) - expected = directory / "libonnxruntime.so" + target = native_target() + selected = verify_runtime(directory, target) + expected = directory / package.TARGETS[target]["runtime_entry"] + if target != "linux-x86_64": + observed = [ + path + for path in native_modules(pid, target) + if path.name.lower().startswith(("onnxruntime", "libonnxruntime")) + ] + package.require(expected in observed, "packaged ONNX runtime was not observed loaded") + for path in observed: + package.require( + path.parent == directory + and path.name.casefold() in {name.casefold() for name in selected["files"]}, + "ambient ONNX runtime mapping detected", + ) + verify_runtime(directory, target) + return { + "loader_sha256": package.sha256(expected), + "mapping": "packaged-module-path-observed", + "native_module_paths": [str(path) for path in observed], + "identity_observation": "native-module-path-and-current-pinned-bytes", + "mapped_file_object_identity": "unavailable on this platform", + "system_dependency_closure": "unqualified", + } stat = expected.stat() matched = False for line in Path(f"/proc/{pid}/maps").read_text().splitlines(): @@ -75,5 +215,6 @@ def loaded_runtime(pid, directory): return { "loader_sha256": package.sha256(expected), "mapping": "packaged-file-identity-observed", + "identity_observation": "mapped-device-inode-and-current-pinned-bytes", "system_dependency_closure": "unqualified", } diff --git a/scripts/release/stage-rpc.py b/scripts/release/stage-rpc.py index 8741d731..1efef669 100644 --- a/scripts/release/stage-rpc.py +++ b/scripts/release/stage-rpc.py @@ -1,25 +1,54 @@ #!/usr/bin/env python3 -"""Stage the freshly built RPC and adjacent native runtime dependencies.""" +"""Stage a full desktop RPC with explicitly provisioned official CPU ORT bytes.""" +import argparse +import hashlib +import json import os from pathlib import Path import shutil -root = Path(__file__).resolve().parents[2] -source = root / "rust/target/release" -target = root / "electron/resources/bin" -binary = "pumas-rpc.exe" if os.name == "nt" else "pumas-rpc" -if not (source / binary).is_file(): - raise SystemExit(f"Missing release backend: {source / binary}") -# This is a generated staging directory, never a runtime/library root. -if target.exists(): - shutil.rmtree(target) -target.mkdir(parents=True) -shutil.copy2(source / binary, target / binary) -# ONNX Runtime is the backend's redistributable native runtime. A developer's -# target directory can also contain unrelated NIF/UniFFI outputs; never ship -# those merely because they share a dynamic-library suffix. -for pattern in ("onnxruntime*.dll", "libonnxruntime.so*", "libonnxruntime*.dylib"): - for library in source.glob(pattern): - shutil.copy2(library, target / library.name) -print(f"Staged backend and native libraries in {target}") +from headless_inference import require +from onnx_runtime_stage import PINS +from packaged_onnx_support import native_target + +ROOT = Path(__file__).resolve().parents[2] + + +def stage_rpc(source, target, runtime, target_id): + require(target_id == native_target(), "desktop runtime requires matching native host") + selected = json.loads(PINS.read_text())["targets"][target_id] + binary = "pumas-rpc.exe" if os.name == "nt" else "pumas-rpc" + require((source / binary).is_file(), "release backend is missing") + payloads = {} + for name, pin in {**selected["files"], **selected["notices"]}.items(): + path = runtime / name + require(path.is_file() and not path.is_symlink(), "official runtime member is missing") + require(path.stat().st_size == pin["bytes"], "official runtime member size differs") + data = path.read_bytes() + require(hashlib.sha256(data).hexdigest() == pin["sha256"], "official runtime pin differs") + payloads[name] = data + # Validate all inputs before replacing this generated staging directory. + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True) + shutil.copy2(source / binary, target / binary) + for name, data in payloads.items(): + (target / name).write_bytes(data) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime-dir", type=Path, required=True) + parser.add_argument("--target", required=True) + args = parser.parse_args() + stage_rpc( + ROOT / "rust/target/release", + ROOT / "electron/resources/bin", + args.runtime_dir, + args.target, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/test_inference_release.py b/scripts/release/test_inference_release.py index f01d5be3..40162e56 100644 --- a/scripts/release/test_inference_release.py +++ b/scripts/release/test_inference_release.py @@ -1,6 +1,7 @@ """Synthetic archives and real subprocess environment checks; no model inference.""" import hashlib +import importlib.util import copy import io import json @@ -21,8 +22,114 @@ import synthetic_onnx_fixture as synthetic import verify_synthetic_packaged_onnx as synthetic_check +STAGE_SPEC = importlib.util.spec_from_file_location( + "desktop_stage", Path(__file__).with_name("stage-rpc.py") +) +desktop = importlib.util.module_from_spec(STAGE_SPEC) +STAGE_SPEC.loader.exec_module(desktop) + class ReleaseTests(unittest.TestCase): + def test_desktop_staging_uses_only_pinned_payloads_and_preserves_old_stage_on_refusal(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, runtime, target = root / "source", root / "runtime", root / "stage" + source.mkdir() + runtime.mkdir() + target.mkdir() + binary = "pumas-rpc.exe" if os.name == "nt" else "pumas-rpc" + (source / binary).write_bytes(b"test RPC, not executable") + (source / "ambient.dll").write_bytes(b"must never copy") + payloads = {"native.dll": b"test native", "ONNX-LICENSE.txt": b"test notice"} + selected = {"files": {}, "notices": {}} + for name, data in payloads.items(): + (runtime / name).write_bytes(data) + selected["notices" if name.endswith(".txt") else "files"][name] = { + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + pins = root / "pins.json" + pins.write_text(json.dumps({"targets": {"test-native": selected}})) + (target / "sentinel").write_bytes(b"prior usable stage") + with ( + patch.object(desktop, "PINS", pins), + patch.object(desktop, "native_target", return_value="test-native"), + ): + (runtime / "native.dll").write_bytes(b"corrupt0000") + with self.assertRaises(ValueError): + desktop.stage_rpc(source, target, runtime, "test-native") + self.assertEqual((target / "sentinel").read_bytes(), b"prior usable stage") + (runtime / "native.dll").write_bytes(payloads["native.dll"]) + desktop.stage_rpc(source, target, runtime, "test-native") + self.assertEqual({p.name for p in target.iterdir()}, {binary, *payloads}) + for name, data in payloads.items(): + self.assertEqual((target / name).read_bytes(), data) + + def test_native_target_rejects_unsupported_host_instead_of_assuming_linux(self): + with ( + patch.object(support.platform, "system", return_value="Linux"), + patch.object(support.platform, "machine", return_value="aarch64"), + ): + with self.assertRaisesRegex(ValueError, "supported native host"): + support.native_target() + + @unittest.skipIf(sys.platform == "linux", "native Windows/macOS OS observer") + def test_real_native_module_observer_can_inspect_its_own_process(self): + paths = support.native_modules(os.getpid(), support.native_target()) + self.assertTrue(paths) + self.assertTrue(any(path.is_file() for path in paths)) + + def test_nonlinux_observer_requires_packaged_runtime_and_refuses_ambient_runtime(self): + for target, name in ( + ("windows-x86_64", "onnxruntime.dll"), + ("macos-arm64", "libonnxruntime.dylib"), + ): + with self.subTest(target=target), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + runtime = root / name + runtime.write_bytes(b"test-only observed mapping") + with ( + patch.object(support, "native_target", return_value=target), + patch.object(support, "verify_runtime", return_value={"files": {name: {}}}), + ): + for paths in ( + [], + [Path("/ambient") / name], + [runtime, Path("/ambient") / name], + ): + with patch.object(support, "native_modules", return_value=paths): + with self.assertRaises(ValueError): + support.loaded_runtime(123, root) + with patch.object(support, "native_modules", return_value=[runtime]): + observed = support.loaded_runtime(123, root) + self.assertEqual( + observed["loader_sha256"], + hashlib.sha256(runtime.read_bytes()).hexdigest(), + ) + self.assertEqual( + observed["mapped_file_object_identity"], "unavailable on this platform" + ) + self.assertEqual(observed["mapping"], "packaged-module-path-observed") + + def test_forced_shutdown_and_nonzero_exit_never_pass(self): + process = Mock(returncode=-9) + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired("owned RPC", 20), -9] + with self.assertRaisesRegex(RuntimeError, "forced shutdown"): + support.stop_process(process) + process.kill.assert_called_once() + process = Mock(returncode=1) + process.poll.return_value = 1 + with self.assertRaisesRegex(ValueError, "shut down cleanly"): + support.stop_process(process) + process = Mock() + process.poll.return_value = None + process.send_signal.side_effect = OSError("test-only unavailable console") + with self.assertRaises(OSError): + support.stop_process(process) + process.kill.assert_called_once() + process.wait.assert_called_once_with(timeout=10) + def test_synthetic_fixture_is_deterministic_untrained_and_refuses_overwrite(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) / "fixture" diff --git a/scripts/release/test_release_metadata.py b/scripts/release/test_release_metadata.py new file mode 100644 index 00000000..61028a20 --- /dev/null +++ b/scripts/release/test_release_metadata.py @@ -0,0 +1,176 @@ +"""Complete synthetic artifact inventories; no actual installer/model acceptance.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import shutil +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "release_metadata", Path(__file__).with_name("write-release-metadata.py") +) +metadata = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(metadata) + + +class MetadataTests(unittest.TestCase): + def fixture(self, root): + repository, output = root / "repo", root / "candidate" + output.mkdir() + for name, data in { + "package.json": b'{"version":"0.8.0-rc.1"}', + "rust/Cargo.lock": b"locked Rust fixture", + "pnpm-lock.yaml": b"locked JS fixture", + }.items(): + path = repository / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + for name in ( + "LICENSE", + "scripts/release/artifact-plan.json", + "scripts/release/onnx-runtime-pins.json", + "scripts/release/licenses/onnxruntime-LICENSE", + "scripts/release/licenses/onnxruntime-ThirdPartyNotices.txt", + ): + path = repository / name + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, path) + for profile in ("", "-s3"): + base = repository / f"docs/release-attribution/0.8.0-rc.1{profile}" + base.mkdir(parents=True) + notices = b"test-only collected attribution fixture" + profile.encode() + (base / "THIRD-PARTY-NOTICES.txt").write_bytes(notices) + (base / "inventory.json").write_text( + json.dumps( + { + "input_sha256": { + "package.json": hashlib.sha256( + (repository / "package.json").read_bytes() + ).hexdigest() + }, + "notices_sha256": hashlib.sha256(notices).hexdigest(), + "packages": [ + { + "name": "test-source" + profile, + "version": "1.0", + "scope": "test-only input", + "license_declared": "MIT", + "source": "NOASSERTION", + } + ], + } + ) + ) + plan = json.loads((repository / "scripts/release/artifact-plan.json").read_text()) + names = [ + entry["filename"].replace("{version}", "0.8.0-rc.1") for entry in plan["artifacts"] + ] + self.assertEqual(len(names), 16) + for name in names: + (output / name).write_bytes(b"test-only non-installable archive: " + name.encode()) + return repository, output, set(names) + + def test_every_final_artifact_and_metadata_byte_is_described(self): + with tempfile.TemporaryDirectory() as temporary: + repository, output, names = self.fixture(Path(temporary)) + metadata.write_metadata( + output, + "test-build", + {"head": "a" * 40, "tree": "b" * 40, "builder": "test-only"}, + repository, + ) + documents = list(output.glob("*.spdx.json")) + self.assertEqual(len(documents), 16) + for path in documents: + document = json.loads(path.read_text()) + artifact = output / document["name"] + self.assertFalse(document["packages"][0]["filesAnalyzed"]) + self.assertEqual( + document["packages"][0]["checksums"][0]["checksumValue"], + hashlib.sha256(artifact.read_bytes()).hexdigest(), + ) + provenance = json.loads(next(output.glob("*.provenance.jsonl")).read_text()) + self.assertEqual({entry["name"] for entry in provenance["subject"]}, names) + self.assertTrue( + provenance["predicate"]["buildDefinition"]["internalParameters"]["unsigned"] + ) + checksums = dict( + line.split(" ", 1)[::-1] + for line in (output / "checksums-sha256.txt").read_text().splitlines() + ) + self.assertEqual( + set(checksums), {p.name for p in output.iterdir()} - {"checksums-sha256.txt"} + ) + self.assertEqual(len(checksums), 35) + for name, sha in checksums.items(): + self.assertEqual(sha, hashlib.sha256((output / name).read_bytes()).hexdigest()) + combined = (output / "THIRD-PARTY-NOTICES-0.8.0-rc.1.txt").read_bytes() + native = repository / "scripts/release/licenses" + for name in ("onnxruntime-LICENSE", "onnxruntime-ThirdPartyNotices.txt"): + lf = (native / name).read_bytes().replace(b"\r\n", b"\n") + self.assertIn(lf, combined) + self.assertIn(lf.replace(b"\n", b"\r\n"), combined) + with self.assertRaisesRegex(ValueError, "fresh artifact-plan"): + metadata.write_metadata(output, "test-build", {}, repository) + + def test_missing_extra_empty_and_symlink_artifacts_fail_before_metadata(self): + for mutation in ("missing", "extra", "empty", "symlink"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary: + repository, output, names = self.fixture(Path(temporary)) + artifact = output / sorted(names)[0] + if mutation == "missing": + artifact.unlink() + elif mutation == "extra": + (output / "stale.exe").write_bytes(b"stale") + elif mutation == "empty": + artifact.write_bytes(b"") + else: + if not hasattr(Path, "symlink_to"): + self.skipTest("filesystem has no symlink support") + artifact.unlink() + try: + artifact.symlink_to(output / sorted(names)[1]) + except OSError: + # Native Windows runners may lack symlink privilege; + # the other required mutation cases still execute. + self.skipTest("native filesystem denied symlink creation") + with self.assertRaises(ValueError): + metadata.write_metadata(output, "test", {}, repository) + self.assertFalse(list(output.glob("*.spdx.json"))) + + def test_stale_attribution_fails_before_metadata(self): + with tempfile.TemporaryDirectory() as temporary: + repository, output, _ = self.fixture(Path(temporary)) + (repository / "package.json").write_text('{"version":"0.8.0-rc.1","changed":true}') + with self.assertRaisesRegex(ValueError, "attribution inputs changed"): + metadata.write_metadata(output, "test", {}, repository) + self.assertFalse(list(output.glob("*.spdx.json"))) + + def test_native_license_cannot_substitute_for_third_party_terms(self): + with tempfile.TemporaryDirectory() as temporary: + repository, output, _ = self.fixture(Path(temporary)) + base = repository / "scripts/release/licenses" + shutil.copyfile( + base / "onnxruntime-LICENSE", base / "onnxruntime-ThirdPartyNotices.txt" + ) + with self.assertRaisesRegex(ValueError, "native notice differs"): + metadata.write_metadata(output, "test", {}, repository) + self.assertFalse(list(output.glob("*.spdx.json"))) + + def test_unrepresented_platform_notice_cannot_be_silently_dropped(self): + with tempfile.TemporaryDirectory() as temporary: + repository, output, _ = self.fixture(Path(temporary)) + path = repository / "scripts/release/onnx-runtime-pins.json" + pins = json.loads(path.read_text()) + pins["targets"]["windows-x86_64"]["notices"]["ONNX-NOTICES.txt"]["sha256"] = "f" * 64 + path.write_text(json.dumps(pins)) + with self.assertRaisesRegex(ValueError, "native notice differs"): + metadata.write_metadata(output, "test", {}, repository) + self.assertFalse(list(output.glob("*.spdx.json"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/verify-packaged-onnx.py b/scripts/release/verify-packaged-onnx.py index bf4a279d..2a0f70de 100644 --- a/scripts/release/verify-packaged-onnx.py +++ b/scripts/release/verify-packaged-onnx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify a Linux packaged RPC with a supplied real Nomic ONNX fixture. +"""Verify a native packaged RPC with a supplied Nomic-layout ONNX fixture. Usage: verify-packaged-onnx.py /path/to/packaged/pumas-rpc /path/to/nomic-model The fixture is copied into an isolated library and is never modified. @@ -8,7 +8,8 @@ import json import math import re -import signal +import os +import shutil import subprocess import sys import tempfile @@ -16,7 +17,7 @@ import urllib.request from pathlib import Path -from packaged_onnx_support import child_environment, loaded_runtime, verify_runtime +from packaged_onnx_support import child_environment, loaded_runtime, stop_process, verify_runtime from headless_inference import require, sha256 binary = Path(sys.argv[1]).resolve() @@ -36,147 +37,137 @@ "special_tokens_map.json", ]: if (fixture / name).exists(): - subprocess.run( - ["cp", "--reflink=auto", str(fixture / name), str(model / name)], check=True - ) - subprocess.run( - ["cp", "--reflink=auto", str(fixture / "onnx/model_fp16.onnx"), str(model / "model.onnx")], - check=True, - ) - log = (root / "rpc.log").open("w+") - p = subprocess.Popen( - [str(binary), "--launcher-root", str(root), "--port", "0"], - stdout=log, - stderr=subprocess.STDOUT, - cwd=root, - env=child_environment(root), - ) - try: - for _ in range(450): - text = (root / "rpc.log").read_text() - m = re.search(r"RPC_PORT=(\d+)", text) - if m: - break - if p.poll() is not None: + shutil.copyfile(fixture / name, model / name) + shutil.copyfile(fixture / "onnx/model_fp16.onnx", model / "model.onnx") + fixture_hashes = {path.name: sha256(path) for path in model.iterdir() if path.is_file()} + with (root / "rpc.log").open("w+") as log: + p = subprocess.Popen( + [str(binary), "--launcher-root", str(root), "--port", "0"], + stdout=log, + stderr=subprocess.STDOUT, + cwd=root, + env=child_environment(root), + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0, + ) + try: + for _ in range(450): + text = (root / "rpc.log").read_text() + m = re.search(r"RPC_PORT=(\d+)", text) + if m: + break + if p.poll() is not None: + raise RuntimeError(text) + time.sleep(0.1) + else: raise RuntimeError(text) - time.sleep(0.1) - else: - raise RuntimeError(text) - base = "http://127.0.0.1:" + m[1] + base = "http://127.0.0.1:" + m[1] - def post(path, data): - with http.open( - urllib.request.Request( - base + path, - data=json.dumps(data).encode(), - headers={"Content-Type": "application/json"}, - ), - timeout=120, - ) as r: - return json.load(r) + def post(path, data): + with http.open( + urllib.request.Request( + base + path, + data=json.dumps(data).encode(), + headers={"Content-Type": "application/json"}, + ), + timeout=120, + ) as r: + return json.load(r) - def rpc(method, params={}): - x = post("/rpc", {"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) - if "error" in x: - raise RuntimeError((method, x)) - return x["result"] + def rpc(method, params={}): + x = post("/rpc", {"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + if "error" in x: + raise RuntimeError((method, x)) + return x["result"] - result = rpc( - "import_model_in_place", - { - "model_dir": str(model), - "official_name": "Nomic fixture", - "family": "nomic_bert", - "model_type": "embedding", - }, - ) - print("import", json.dumps(result)[:1000], flush=True) - require(result.get("success") is True, f"Import failed: {result}") - model_id = result["model_id"] - profile = "onnx-verification" - print( - "profile", - rpc( - "upsert_runtime_profile", + result = rpc( + "import_model_in_place", { - "profile": { - "profile_id": profile, - "provider": "onnx_runtime", - "provider_mode": "onnx_serve", - "management_mode": "managed", - "name": "Installer verification", + "model_dir": str(model), + "official_name": "Nomic fixture", + "family": "nomic_bert", + "model_type": "embedding", + }, + ) + print("import", json.dumps(result)[:1000], flush=True) + require(result.get("success") is True, f"Import failed: {result}") + model_id = result["model_id"] + profile = "onnx-verification" + print( + "profile", + rpc( + "upsert_runtime_profile", + { + "profile": { + "profile_id": profile, + "provider": "onnx_runtime", + "provider_mode": "onnx_serve", + "management_mode": "managed", + "name": "Installer verification", + } + }, + ), + flush=True, + ) + print("launch", rpc("launch_runtime_profile", {"profile_id": profile}), flush=True) + loaded = rpc( + "serve_model", + { + "request": { + "model_id": model_id, + "config": { + "provider": "onnx_runtime", + "profile_id": profile, + "device_mode": "auto", + "keep_loaded": True, + "model_alias": "verification-nomic", + }, } }, - ), - flush=True, - ) - print("launch", rpc("launch_runtime_profile", {"profile_id": profile}), flush=True) - loaded = rpc( - "serve_model", - { - "request": { - "model_id": model_id, - "config": { + ) + require(loaded.get("loaded") is True, f"Load failed: {loaded}") + embedding = post( + "/v1/embeddings", + { + "model": "verification-nomic", + "input": ["search_query: hello world"], + "dimensions": 256, + }, + ) + vector = embedding["data"][0]["embedding"] + require( + len(vector) == 256 + and all(type(x) in (int, float) and math.isfinite(x) for x in vector), + "Embedding must contain 256 finite numeric values", + ) + runtime_evidence = loaded_runtime(p.pid, binary.parent) + unloaded = rpc( + "unserve_model", + { + "request": { + "model_id": model_id, "provider": "onnx_runtime", "profile_id": profile, - "device_mode": "auto", - "keep_loaded": True, "model_alias": "verification-nomic", - }, - } - }, - ) - require(loaded.get("loaded") is True, f"Load failed: {loaded}") - embedding = post( - "/v1/embeddings", - { - "model": "verification-nomic", - "input": ["search_query: hello world"], - "dimensions": 256, - }, - ) - vector = embedding["data"][0]["embedding"] - require( - len(vector) == 256 - and all(type(x) in (int, float) and math.isfinite(x) for x in vector), - "Embedding must contain 256 finite numeric values", - ) - runtime_evidence = loaded_runtime(p.pid, binary.parent) - unloaded = rpc( - "unserve_model", - { - "request": { - "model_id": model_id, - "provider": "onnx_runtime", - "profile_id": profile, - "model_alias": "verification-nomic", - } - }, - ) - require(unloaded.get("unloaded") is True, f"Unload failed: {unloaded}") - print( - json.dumps( - { - "binary": str(binary), - "binary_sha256": sha256(binary), - "runtime": runtime_evidence, - "import": "passed", - "load": "passed", - "embedding_dimensions": len(vector), - "finite": True, - "unload": "passed", - } - ), - flush=True, - ) - finally: - if p.poll() is None: - p.send_signal(signal.SIGINT) - try: - p.wait(timeout=20) - except subprocess.TimeoutExpired: - p.kill() - p.wait(timeout=10) - raise RuntimeError("Packaged backend required forced shutdown") - log.close() - require(p.returncode == 0, f"Packaged backend exited with {p.returncode}") + } + }, + ) + require(unloaded.get("unloaded") is True, f"Unload failed: {unloaded}") + print( + json.dumps( + { + "binary": str(binary), + "binary_sha256": sha256(binary), + "fixture_sha256": fixture_hashes, + "fixture_acceptance": "caller-owned; lifecycle does not establish model provenance", + "runtime": runtime_evidence, + "import": "passed", + "load": "passed", + "embedding_dimensions": len(vector), + "finite": True, + "unload": "passed", + } + ), + flush=True, + ) + finally: + stop_process(p) diff --git a/scripts/release/write-release-metadata.py b/scripts/release/write-release-metadata.py new file mode 100644 index 00000000..d4c112ff --- /dev/null +++ b/scripts/release/write-release-metadata.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Describe every final candidate artifact; never grant publication acceptance.""" + +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def write_metadata(directory, build_id, source, repository=ROOT): + directory, repository = Path(directory), Path(repository) + version = json.loads((repository / "package.json").read_text())["version"] + plan = json.loads((repository / "scripts/release/artifact-plan.json").read_text()) + artifacts = { + entry["filename"].replace("{version}", version): entry for entry in plan["artifacts"] + } + # The existing checker has already collected the complete flat artifact set. + # Reject missing/foreign files here too before emitting partial descriptions. + present = {p.name for p in directory.iterdir() if p.name != "checksums-sha256.txt"} + if present != set(artifacts): + raise ValueError("metadata requires the exact fresh artifact-plan inventory") + for name in artifacts: + path = directory / name + if path.is_symlink() or not path.is_file() or path.stat().st_size == 0: + raise ValueError("metadata requires regular nonempty final artifacts") + inventories, notices = {}, [] + for profile in ("", "-s3"): + base = repository / f"docs/release-attribution/{version}{profile}" + inventory = json.loads((base / "inventory.json").read_text()) + for name, expected in inventory["input_sha256"].items(): + if digest(repository / name) != expected: + raise ValueError("release attribution inputs changed") + text = base / "THIRD-PARTY-NOTICES.txt" + if digest(text) != inventory["notices_sha256"]: + raise ValueError("release attribution text changed") + inventories[profile] = inventory + notices.append(text.read_bytes()) + pins = json.loads((repository / "scripts/release/onnx-runtime-pins.json").read_text()) + for name, member in ( + ("onnxruntime-LICENSE", "ONNX-LICENSE.txt"), + ("onnxruntime-ThirdPartyNotices.txt", "ONNX-NOTICES.txt"), + ): + path = repository / "scripts/release/licenses" / name + expected = { + (target["notices"][member]["bytes"], target["notices"][member]["sha256"]) + for target in pins["targets"].values() + } + # Upstream Windows uses CRLF for these same texts. Emit only exact + # byte representations independently authenticated by every target pin; + # never assume one target's notice covers another target's notice role. + lf = path.read_bytes().replace(b"\r\n", b"\n") + variants = { + (len(data), hashlib.sha256(data).hexdigest()): data + for data in (lf, lf.replace(b"\n", b"\r\n")) + } + if not expected <= variants.keys(): + raise ValueError("native notice differs from reviewed runtime authority") + notices.extend(variants[key] for key in sorted(expected)) + created = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + subjects = [] + targets = {entry["id"]: entry for entry in plan["desktop_targets"] + plan["headless_targets"]} + for filename, artifact in sorted(artifacts.items()): + sha = digest(directory / filename) + subjects.append({"name": filename, "digest": {"sha256": sha}}) + profile = "-s3" if artifact.get("variant") == "headless-inference" else "" + packages = [ + { + "name": "Pumas Library", + "SPDXID": "SPDXRef-Pumas", + "versionInfo": version, + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseDeclared": "MIT", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [{"algorithm": "SHA256", "checksumValue": sha}], + "sourceInfo": f"Artifact {artifact['id']}; target {artifact['target']}; build {build_id}", + } + ] + relationships = [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Pumas", + } + ] + if artifact.get("variant", "full") in {"full", "headless-inference"}: + runtime = next( + entry + for entry in pins["targets"].values() + if entry["target"] == targets[artifact["target"]]["rust_target"] + ) + packages.append( + { + "SPDXID": "SPDXRef-ORT", + "name": "onnxruntime-cpu", + "versionInfo": pins["version"], + "downloadLocation": runtime["url"], + "filesAnalyzed": False, + "licenseDeclared": "MIT", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + {"algorithm": "SHA256", "checksumValue": runtime["archive_sha256"]} + ], + "sourceInfo": "Reviewed native runtime input archive; package inspection and execution acceptance remain separate.", + } + ) + relationships.append( + { + "spdxElementId": "SPDXRef-Pumas", + "relationshipType": "OTHER", + "relatedSpdxElement": "SPDXRef-ORT", + "comment": "Pinned runtime input for this artifact target.", + } + ) + for index, dependency in enumerate(inventories[profile]["packages"]): + ident = f"SPDXRef-Dependency-{index}" + packages.append( + { + "SPDXID": ident, + "name": dependency["name"], + "versionInfo": dependency["version"], + "downloadLocation": dependency["source"], + "filesAnalyzed": False, + "licenseDeclared": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "sourceInfo": dependency["scope"] + + "; declares " + + str(dependency["license_declared"]), + } + ) + relationships.append( + { + "spdxElementId": "SPDXRef-Pumas", + "relationshipType": "OTHER", + "relatedSpdxElement": ident, + "comment": "Attribution input closure; includes build-only/other-target packages. Not runtime inclusion proof.", + } + ) + document = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": filename, + "documentNamespace": "https://pumas-library.local/spdx/" + sha, + "creationInfo": {"created": created, "creators": ["Tool: pumas-release-metadata"]}, + "comment": "Artifact-level inventory bound to final bytes. Extracted files and native transitive components are not exhaustively analyzed. Unsigned candidate; maintainer inspection/license review remains required.", + "packages": packages, + "relationships": relationships, + } + with (directory / (filename + ".spdx.json")).open("x") as stream: + stream.write(json.dumps(document, indent=2) + "\n") + shutil.copyfile(repository / "LICENSE", directory / "LICENSE.txt") + (directory / f"THIRD-PARTY-NOTICES-{version}.txt").write_bytes(b"\n\n".join(notices)) + provenance = { + "_type": "https://in-toto.io/Statement/v1", + "subject": subjects, + "predicateType": "https://slsa.dev/provenance/v1", + "predicate": { + "buildDefinition": { + "buildType": "https://github.com/MrScripty/Pumas-Library/candidate-assembly/v1", + "externalParameters": {"build_id": build_id, "version": version}, + "internalParameters": { + "unsigned": True, + "qualification": "unverified_candidate", + "assurance": "self-reported assembly; no SLSA level or model/license acceptance claimed", + "source_tree": source["tree"], + "artifact_plan_sha256": digest( + repository / "scripts/release/artifact-plan.json" + ), + "rust_lock_sha256": digest(repository / "rust/Cargo.lock"), + "pnpm_lock_sha256": digest(repository / "pnpm-lock.yaml"), + "runtime_pins_sha256": digest( + repository / "scripts/release/onnx-runtime-pins.json" + ), + "attribution_input_profiles": { + profile or "default": { + "input_sha256": inventory["input_sha256"], + "notices_sha256": inventory["notices_sha256"], + } + for profile, inventory in inventories.items() + }, + }, + "resolvedDependencies": [ + { + "uri": "git+https://github.com/MrScripty/Pumas-Library", + "digest": {"gitCommit": source["head"]}, + } + ], + }, + "runDetails": {"builder": {"id": source["builder"]}}, + }, + } + (directory / f"pumas-library-{version}.provenance.jsonl").write_text( + json.dumps(provenance) + "\n" + ) + final = sorted(p for p in directory.iterdir() if p.name != "checksums-sha256.txt") + if any(p.is_symlink() or not p.is_file() for p in final): + raise ValueError("final metadata inventory must contain regular files only") + (directory / "checksums-sha256.txt").write_text( + "".join(f"{digest(path)} {path.name}\n" for path in final) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--build-id", required=True) + args = parser.parse_args() + + def git(*arguments): + return subprocess.check_output(["git", *arguments], cwd=ROOT, text=True).strip() + + if git("status", "--porcelain", "--untracked-files=no"): + raise ValueError("candidate metadata requires unchanged tracked source inputs") + source = { + "head": git("rev-parse", "HEAD"), + "tree": git("rev-parse", "HEAD^{tree}"), + "builder": "https://github.com/" + + os.environ.get("GITHUB_WORKFLOW_REF", "MrScripty/Pumas-Library/local-candidate-assembly"), + } + write_metadata(args.directory, args.build_id, source) + + +if __name__ == "__main__": + main()